{"models":[{"modelId":"claude-fable-5-2","providerId":"anthropic","slug":"claude-fable-5-2","name":"Claude Fable 5.2","provider":"Anthropic","description":"Claude Fable 5.2 is a reasoning model from Anthropic in the Claude Fable family. 37 benchmarks count toward its index, in 6 categories.","logoUrl":null,"releaseDate":"2026-11-09","contextWindowTokens":1000000,"modalities":["text","image","file"],"openWeights":false,"details":{"apiId":"anthropic/claude-fable-5.2","maxOutputTokens":128000,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.19999999999999998,"weightsUrl":null},"reasoning":true,"overallScore":85.61873265280894,"overallScoreRange":{"low":81.96040102808132,"high":89.27706427753657},"badges":["best"],"capabilityScores":{"agentic":82.23028473118158,"coding":86.20014821675882,"reasoning":86.51502971742858,"multimodal":82.0459960782477,"knowledge":87.07746091056276,"multilingual":null,"instruction":null,"math":79.33389128512624},"capabilityCounts":{"math":4,"reasoning":10,"knowledge":4,"coding":8,"agentic":10,"multimodal":1},"benchmarks":[{"benchmarkId":"vals-ai:proof_bench:overall","benchmarkName":"ProofBench v1.1","category":"math","domain":null,"eligible":true,"ranked":true,"score":100,"unit":"percent","family":"vals-ai:proof_bench:overall","index":81.68109068983412,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/proof_bench","measuredAt":"2026-11-09","stderr":0,"costPerTest":0.786101,"latency":170.863},{"benchmarkId":"epoch-ai:epoch-frontiermath-tier-4-v2-private:overall","benchmarkName":"FrontierMath-Tier-4-v2-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":100,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tier-4-v2-private:overall","index":87.77330837649896,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":99.72222222222223,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":66.93688599272527,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0.2777777777777778,"costPerTest":null,"latency":null},{"benchmarkId":"arc-prize:v1-public-eval:overall","benchmarkName":"ARC-AGI-1 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":99,"unit":"percent","family":"arc-prize:v1-public-eval:overall","index":null,"aboveFit":false,"run":"max effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.7087507799999995,"latency":null},{"benchmarkId":"arc-prize:v2-public-eval:overall","benchmarkName":"ARC-AGI-2 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":98.75,"unit":"percent","family":"arc-prize:v2-public-eval:overall","index":null,"aboveFit":false,"run":"max effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":2.6861016,"latency":null},{"benchmarkId":"arc-prize:v1-semi-private:overall","benchmarkName":"ARC-AGI-1 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":97.5,"unit":"percent","family":"arc-prize:v1-semi-private:overall","index":71.81162890019444,"aboveFit":false,"run":"max effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.9209223200000004,"latency":null},{"benchmarkId":"gmmlu","benchmarkName":"Global MMLU","category":"multilingual","score":96,"unit":"percent","family":"gmmlu","index":null,"aboveFit":false,"run":null,"source":"Singh et al.","sourceUrl":"https://arxiv.org/abs/2412.03304","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tiers-1-3-v2-private:overall","benchmarkName":"FrontierMath-Tiers-1-3-v2-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":95.08771929824562,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tiers-1-3-v2-private:overall","index":80.94428008144664,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.2824623769055388,"costPerTest":null,"latency":null},{"benchmarkId":"milu","benchmarkName":"Multi-task Indic Language Understanding Benchmark","category":"multilingual","score":94.1,"unit":"percent","family":"milu","index":null,"aboveFit":false,"run":null,"source":"Verma et al.","sourceUrl":"https://arxiv.org/abs/2411.02538","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-simpleqa-verified:overall","benchmarkName":"SimpleQA Verified","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":94,"unit":"percent","family":"epoch-ai:epoch-simpleqa-verified:overall","index":95,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0.7513751157474973,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":93.87626262626263,"unit":"percent","family":"gpqa-diamond","index":63.30984364225107,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.6253431037635206,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:vibe-code:overall","benchmarkName":"Vibe Code Bench v1.1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":93.771,"unit":"percent","family":"vals-ai:vibe-code:overall","index":76.08357264386466,"aboveFit":false,"run":"OpenHands","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","measuredAt":"2026-11-09","stderr":1.152,"costPerTest":22.13226,"latency":2437.279},{"benchmarkId":"swePro","benchmarkName":"SWE-bench Pro","category":"coding","score":93,"unit":"percent","family":"swePro","index":88.03911192439594,"aboveFit":false,"run":null,"source":"Xiang Deng, Jeff Da, Edwin Pan, Yannis Yiming He, Charles Ide, Kanak Garg, Niklas Lauffer, Andrew Park, Nitin Pasari, Chetan Rane, Karmini Sampath, Maya Krishnan, Srivatsa Kundurthy, Sean Hendryx, Zifan Wang, Chen Bo Calvin Zhang, Noah Jacobson, Bing Liu, Brad Kenstler","sourceUrl":"https://arxiv.org/abs/2509.16941","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"arc-prize:v2-semi-private:overall","benchmarkName":"ARC-AGI-2 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":92.5,"unit":"percent","family":"arc-prize:v2-semi-private:overall","index":81.65847524253698,"aboveFit":false,"run":"max effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":2.8135210000000006,"latency":null},{"benchmarkId":"chartographyWithTools","benchmarkName":"Chartography with image and code tools","category":"multimodal","score":91.6,"unit":"percent","family":"chartographyWithTools","index":null,"aboveFit":false,"run":null,"source":"Surge AI and Anthropic","sourceUrl":"https://surgehq.ai/blog/chartography","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-ebr-bench:overall","benchmarkName":"EBR-bench","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":90.47619047619048,"unit":"percent","family":"epoch-ai:epoch-ebr-bench:overall","index":93.94878012176618,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.5058465048420853,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-furniture-assembly:overall","benchmarkName":"Furniture Assembly","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":90,"unit":"percent","family":"epoch-ai:epoch-furniture-assembly:overall","index":88.61939307028528,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":3.905667329424715,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-mystery-game-puzzles:overall","benchmarkName":"Mystery Game Puzzles","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":90,"unit":"percent","family":"epoch-ai:epoch-mystery-game-puzzles:overall","index":95,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":3.015113445777636,"costPerTest":null,"latency":null},{"benchmarkId":"osWorld21Offline","benchmarkName":"OSWorld 2.1 offline subset (82 tasks)","category":"agentic","score":89.3,"unit":"percent","family":"osWorld21Offline","index":86.11378227407562,"aboveFit":false,"run":null,"source":"Mengqi Yuan, Zilong Zhou, Xinzhuang Xiong, Weiming Wu, Jiayang Sun, Jiamin Song, Kaiqian Cui, Bowen Wang, Haoyuan Wu, Yitong Li, Dunjie Lu, Haikong Lu, Qi Zhen, Xinyuan Wang, Jiaqi Deng, Yuhao Yang, Cheng Chen, Boyuan Zheng, Alex Su, Xiao Yu, Hao Zou, Saaket Agashe, Xing Han Lu, Manpreet Kaur, Zhengyang Qi, Vincent Sunn Chen, Frederic Sala, Dayiheng Liu, Junyang Lin, Zhou Yu, Yu Su, Siva Reddy, Xin Eric Wang, Peng Qi, Tianbao Xie, Tao Yu","sourceUrl":"https://github.com/xlang-ai/OSWorld-V2/releases","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"draco","benchmarkName":"Data Research and Analysis with Complex Operations","category":"agentic","score":87.7,"unit":"percent","family":"draco","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"programBench","benchmarkName":"ProgramBench: Can Language Models Rebuild Programs From Scratch?","category":"coding","score":85.6,"unit":"percent","family":"programBench","index":94.86485803850373,"aboveFit":false,"run":null,"source":"John Yang, Kilian Lieret, Jeffrey Ma, Parth Thakkar, Dmitrii Pedchenko, Sten Sootla, Emily McMilin, Pengcheng Yin, Rui Hou, Gabriel Synnaeve, Diyi Yang, Ofir Press","sourceUrl":"https://programbench.com/static/paper.pdf","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:biomysterybench:overall","benchmarkName":"BioMysteryBench","category":null,"domain":"Science","eligible":false,"ranked":true,"score":83.704,"unit":"percent","family":"vals-ai:biomysterybench:overall","index":null,"aboveFit":false,"run":"Terminus 2","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/biomysterybench","measuredAt":"2026-11-09","stderr":0.741,"costPerTest":1.681583,"latency":248.169},{"benchmarkId":"benchCad21WithTools","benchmarkName":"BenchCAD 2.1 Core, with tools","category":"multimodal","score":83.5,"unit":"percent","family":"benchCad21WithTools","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"officeQa","benchmarkName":"OfficeQA","category":"multimodal","score":83.4,"unit":"percent","family":"officeQa","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":83.3,"unit":"percent","family":"lcr","index":64.4485572404562,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"wandr","benchmarkName":"WANDR (Wide ANd Deep Research)","category":"agentic","score":82.1,"unit":"percent","family":"wandr","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"omniscienceAccuracy","benchmarkName":"Artificial Analysis Omniscience Accuracy","category":"knowledge","score":79.1,"unit":"percent","family":"omniscienceAccuracy","index":95,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-chess-puzzles:overall","benchmarkName":"Chess Puzzles","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":79,"unit":"percent","family":"epoch-ai:epoch-chess-puzzles:overall","index":93.94687614199654,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":4.093601807403323,"costPerTest":null,"latency":null},{"benchmarkId":"aaAutomationBench","benchmarkName":"Artificial Analysis AutomationBench","category":"agentic","score":76.4,"unit":"percent","family":"aaAutomationBench","index":85.3398034215638,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/automationbench-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"benchCad21","benchmarkName":"BenchCAD 2.1 Core, without tools","category":"multimodal","score":75.4,"unit":"percent","family":"benchCad21","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"terminalBenchScience","benchmarkName":"Terminal-Bench-Science 0.1","category":"agentic","score":74.6,"unit":"percent","family":"terminalBenchScience","index":87.0146108770067,"aboveFit":false,"run":null,"source":"Terminal-Bench-Science Team","sourceUrl":"https://www.terminal-bench-science.ai/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"deepSwe","benchmarkName":"DeepSWE","category":"agentic","score":74.3,"unit":"percent","family":"deepSwe","index":75.090746473617,"aboveFit":false,"run":null,"source":"Datacurve AI","sourceUrl":"https://deepswe.datacurve.ai/blog","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":74.3,"unit":"percent","family":"hle","index":95,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hle","benchmarkName":"Humanity's Last Exam","category":"knowledge","score":74.2,"unit":"percent","family":"hle","index":95,"aboveFit":false,"run":null,"source":"Center for AI Safety, Scale AI, and thousands of expert contributors","sourceUrl":"https://lastexam.ai/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"chartography","benchmarkName":"Chartography without tools","category":"multimodal","score":73.4,"unit":"percent","family":"chartography","index":null,"aboveFit":false,"run":null,"source":"Surge AI and Anthropic","sourceUrl":"https://surgehq.ai/blog/chartography","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hleNoTools","benchmarkName":"Humanity's Last Exam without tools","category":"knowledge","score":73.2,"unit":"percent","family":"hle","index":95,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"officeQaPro","benchmarkName":"OfficeQA Pro","category":"multimodal","score":72.9,"unit":"percent","family":"officeQaPro","index":82.0459960782477,"aboveFit":false,"run":null,"source":"OfficeQA Pro authors","sourceUrl":"https://arxiv.org/abs/2603.08655","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaHealthBenchProfessionalRaw","benchmarkName":"Artificial Analysis HealthBench Professional raw score","category":"knowledge","score":71.1,"unit":"percent","family":"aaHealthBenchProfessionalRaw","index":null,"aboveFit":false,"run":null,"source":"OpenAI (benchmark); Artificial Analysis (evaluation)","sourceUrl":"https://artificialanalysis.ai/evaluations/healthbench-professional","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"frontierSweV2","benchmarkName":"FrontierSWE v2","category":"coding","score":69.9,"unit":"percent","family":"frontierSweV2","index":86.92236148594588,"aboveFit":false,"run":null,"source":"Proximal","sourceUrl":"https://www.frontierswe.com/blog/v2","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaSciCode","benchmarkName":"Artificial Analysis SciCode","category":"coding","score":67.7,"unit":"percent","family":"scicode-family","index":84.0996890547887,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/scicode","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaHealthBenchProfessional","benchmarkName":"Artificial Analysis HealthBench Professional length-adjusted score","category":"knowledge","score":67.7,"unit":"percent","family":"aaHealthBenchProfessional","index":null,"aboveFit":false,"run":null,"source":"OpenAI (benchmark); Artificial Analysis (evaluation)","sourceUrl":"https://artificialanalysis.ai/evaluations/healthbench-professional","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaBriefcaseRubric","benchmarkName":"AA-Briefcase v1.1, rubric pass rate","category":"agentic","score":66.1,"unit":"percent","family":"aaBriefcaseRubric","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"arc-prize:v3-semi-private:overall","benchmarkName":"ARC-AGI-3 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":65.66315458367342,"unit":"percent","family":"arc-prize:v3-semi-private:overall","index":93.20341537513639,"aboveFit":false,"run":"max effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:terminal-bench-4:overall","benchmarkName":"Terminal-Bench 4.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":65.152,"unit":"percent","family":"terminal-bench-4","index":85.45269998005702,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-4","measuredAt":"2026-11-09","stderr":2.314,"costPerTest":14.564414,"latency":3307.186},{"benchmarkId":"roboDojoSim","benchmarkName":"RoboDojo Sim","category":"agentic","score":64.9,"unit":"percent","family":"roboDojoSim","index":null,"aboveFit":false,"run":null,"source":"Anthropic (reported run)","sourceUrl":"https://www.anthropic.com/claude-fable-and-mythos-5-2","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"physicianBench","benchmarkName":"PhysicianBench","category":"knowledge","score":64.4,"unit":"percent","family":"physicianBench","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"stemProfessorBench","benchmarkName":"Anthropic STEM Professor Bench","category":"reasoning","score":61.8,"unit":"percent","family":"stemProfessorBench","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"cursorBench40","benchmarkName":"cursorBench40","category":"coding","score":60,"unit":"percent","family":"cursorBench40","index":83.06688868564902,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaTerminalBench4","benchmarkName":"Artificial Analysis Terminal-Bench v4.0","category":"agentic","score":59.6,"unit":"percent","family":"aaTerminalBench4","index":79.53215389821452,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/terminalbench-v4-0","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaHealthBenchLengthAdjusted","benchmarkName":"Artificial Analysis HealthBench length-adjusted score","category":"knowledge","score":59.6,"unit":"percent","family":"aaHealthBenchLengthAdjusted","index":null,"aboveFit":false,"run":null,"source":"OpenAI (benchmark); Artificial Analysis (evaluation)","sourceUrl":"https://artificialanalysis.ai/evaluations/healthbench","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaHealthBench","benchmarkName":"Artificial Analysis HealthBench raw score","category":"knowledge","score":58.8,"unit":"percent","family":"aaHealthBench","index":null,"aboveFit":false,"run":null,"source":"OpenAI (benchmark); Artificial Analysis (evaluation)","sourceUrl":"https://artificialanalysis.ai/evaluations/healthbench","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"bug-hunt-bench","benchmarkName":"Bug Hunt Bench","category":"coding","score":58,"unit":"percent","family":"bug-hunt-bench","index":88.09627588681721,"aboveFit":false,"run":null,"source":"Pawel Huryn","sourceUrl":"https://bughunt.productcompass.pm/method","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mka","benchmarkName":"Multimodal Knowledge Agent Benchmark","category":"agentic","score":56.6,"unit":"percent","family":"mka","index":83.61608311271794,"aboveFit":false,"run":null,"source":"Google DeepMind, in partnership with Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mka","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:terminal-bench-science:overall","benchmarkName":"Terminal-Bench-Science","category":null,"domain":"Science","eligible":false,"ranked":true,"score":55.714,"unit":"percent","family":"vals-ai:terminal-bench-science:overall","index":null,"aboveFit":false,"run":"Mini-SWE-agent","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-science","measuredAt":"2026-11-09","stderr":5.98,"costPerTest":23.13708,"latency":5702.569},{"benchmarkId":"vals-ai:mysterymechanism:overall","benchmarkName":"MysteryMechanism","category":null,"domain":"Science","eligible":false,"ranked":true,"score":51.802,"unit":"percent","family":"vals-ai:mysterymechanism:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mysterymechanism","measuredAt":"2026-11-11","stderr":3.361,"costPerTest":1.552812,"latency":292.059},{"benchmarkId":"automationBench","benchmarkName":"AutomationBench","category":"agentic","score":51.4,"unit":"percent","family":"automationBench","index":79.85794195307733,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"frontierCode2","benchmarkName":"FrontierCode 2.0","category":"coding","score":49.1,"unit":"percent","family":"frontierCode2","index":88.42842801410542,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:legal_research:overall","benchmarkName":"Legal Research Bench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":46.154,"unit":"percent","family":"vals-ai:legal_research:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_research","measuredAt":"2026-11-09","stderr":3.465,"costPerTest":3.956441,"latency":416.198},{"benchmarkId":"aaMlcr","benchmarkName":"Medical Long Context Reasoning (MLCR-AA)","category":"reasoning","score":43.9,"unit":"percent","family":"aaMlcr","index":87.51317108191381,"aboveFit":false,"run":null,"source":"Wisedocs and Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mlcr-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"rliAutomationRate","benchmarkName":"Remote Labor Index (RLI)","category":"agentic","score":41.3,"unit":"percent","family":"rliAutomationRate","index":null,"aboveFit":false,"run":null,"source":"Center for AI Safety; Scale AI","sourceUrl":"https://scale.com/leaderboard/rli","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":35.4,"unit":"percent","family":"critpt","index":95,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lmarena:agent-steerability:overall","benchmarkName":"Agent Arena steerability","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":32.95927840209205,"unit":"index","family":"agent-arena","index":93.62849522699193,"aboveFit":false,"run":"max effort","source":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset","measuredAt":"2026-11-11","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:tax_agent_bench:overall","benchmarkName":"Tax Agent Bench","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":32.092,"unit":"percent","family":"vals-ai:tax_agent_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench","measuredAt":"2026-11-09","stderr":3.367,"costPerTest":3.309035,"latency":299.743},{"benchmarkId":"aaGdpPdf","benchmarkName":"Artificial Analysis GDP.pdf","category":"agentic","score":24.2,"unit":"percent","family":"aaGdpPdf","index":71.16703153048233,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdp-pdf","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lmarena:agent-task-outcome:overall","benchmarkName":"Agent Arena task outcome","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":21.9869149957691,"unit":"index","family":"agent-arena","index":84.60749235501521,"aboveFit":false,"run":"max effort","source":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset","measuredAt":"2026-11-11","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-spireclimb-slay-the-spire-1-ascension-20-heart:overall","benchmarkName":"SpireClimb (Slay the Spire 1, Ascension 20 Heart)","category":null,"domain":"Other","eligible":false,"ranked":true,"score":21.666666666666668,"unit":"percent","family":"epoch-ai:epoch-spireclimb-slay-the-spire-1-ascension-20-heart:overall","index":null,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":5.363439040648223,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-spireclimb:overall","benchmarkName":"SpireClimb","category":null,"domain":"Other","eligible":false,"ranked":true,"score":19.617117117117118,"unit":"percent","family":"epoch-ai:epoch-spireclimb:overall","index":null,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":3.4858275906250853,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:programbench:overall","benchmarkName":"ProgramBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":19,"unit":"percent","family":"vals-ai:programbench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/programbench","measuredAt":"2026-11-09","stderr":2.781,"costPerTest":46.822029,"latency":7528.7},{"benchmarkId":"epoch-ai:epoch-spireclimb-slay-the-spire-2-ascension-10:overall","benchmarkName":"SpireClimb (Slay the Spire 2, Ascension 10)","category":null,"domain":"Other","eligible":false,"ranked":true,"score":17.56756756756757,"unit":"percent","family":"epoch-ai:epoch-spireclimb-slay-the-spire-2-ascension-10:overall","index":null,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":4.453930581352028,"costPerTest":null,"latency":null},{"benchmarkId":"aaHarveyLabV11","benchmarkName":"Harvey LAB-AA v1.1","category":"agentic","score":7.2,"unit":"percent","family":"aaHarveyLabV11","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:hlab:overall","benchmarkName":"HLAB-Vals-1.1","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":5,"unit":"percent","family":"vals-ai:hlab:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/hlab","measuredAt":"2026-11-09","stderr":1.715,"costPerTest":19.366751,"latency":1954.199}],"price":{"inputPerMillion":8,"outputPerMillion":40,"batch":{"inputPerMillion":4,"outputPerMillion":20},"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":64,"timeToFirstToken":4.9345,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":90,"benchmarkCount":37,"categoryCount":6,"confidence":"medium","state":"available"},"hosts":[{"host":"amazon-bedrock","name":"Amazon Bedrock","kind":"cloud","url":"https://docs.aws.amazon.com/bedrock/latest/userguide/models-supported.html","inputPerMillion":8,"outputPerMillion":40,"cacheReadPerMillion":0.2,"contextTokens":1000000,"maxOutputTokens":128000,"deprecated":false},{"host":"anthropic","name":"Anthropic","kind":"maker","url":"https://docs.anthropic.com/en/docs/about-claude/models","inputPerMillion":8,"outputPerMillion":40,"cacheReadPerMillion":0.2,"contextTokens":1000000,"maxOutputTokens":128000,"deprecated":false},{"host":"digitalocean","name":"DigitalOcean","kind":"cloud","url":"https://docs.digitalocean.com/products/gradient-ai-platform/details/models/","inputPerMillion":8,"outputPerMillion":40,"cacheReadPerMillion":0.2,"contextTokens":1000000,"maxOutputTokens":128000,"deprecated":false},{"host":"google-vertex","name":"Google Vertex AI","kind":"cloud","url":"https://cloud.google.com/vertex-ai/generative-ai/docs/models","inputPerMillion":8,"outputPerMillion":40,"cacheReadPerMillion":0.2,"contextTokens":1000000,"maxOutputTokens":128000,"deprecated":false},{"host":"azure","name":"Microsoft Azure","kind":"cloud","url":"https://learn.microsoft.com/en-us/azure/ai-services/openai/concepts/models","inputPerMillion":8,"outputPerMillion":40,"cacheReadPerMillion":0.2,"contextTokens":1000000,"maxOutputTokens":128000,"deprecated":false},{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":8,"outputPerMillion":40,"cacheReadPerMillion":0.2,"contextTokens":1000000,"maxOutputTokens":128000,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:arc-prize-foundation","name":"ARC Prize Foundation","url":"https://arcprize.org","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:lmarena","name":"LMArena","url":"https://huggingface.co","license":"CC BY 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gemini-4-carbon","providerId":"google","slug":"gemini-4-carbon","name":"Gemini 4 Carbon","provider":"Google","description":"Gemini 4 Carbon is a reasoning model from Google in the Gemini 4 family. 28 benchmarks count toward its index, in 5 categories.","logoUrl":null,"releaseDate":"2026-10-28","contextWindowTokens":1048576,"modalities":["audio","file","image","text","video"],"openWeights":false,"details":{"apiId":"google/gemini-4-carbon","maxOutputTokens":943718,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.09999999999999999,"weightsUrl":null},"reasoning":true,"overallScore":82.6671329264836,"overallScoreRange":{"low":77.611290162204,"high":87.7229756907632},"badges":[],"capabilityScores":{"agentic":83.21498358330359,"coding":80.5453296888075,"reasoning":83.87948754209178,"multimodal":null,"knowledge":79.51820891883091,"multilingual":null,"instruction":null,"math":77.16627990413554},"capabilityCounts":{"math":3,"reasoning":8,"knowledge":3,"coding":4,"agentic":10},"benchmarks":[{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":100,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":67.09849544987334,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0,"costPerTest":null,"latency":null},{"benchmarkId":"arc-prize:v1-public-eval:overall","benchmarkName":"ARC-AGI-1 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":98.5,"unit":"percent","family":"arc-prize:v1-public-eval:overall","index":null,"aboveFit":false,"run":"high effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.2812528100000001,"latency":null},{"benchmarkId":"arc-prize:v2-public-eval:overall","benchmarkName":"ARC-AGI-2 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":97.91666666666666,"unit":"percent","family":"arc-prize:v2-public-eval:overall","index":null,"aboveFit":false,"run":"high effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.7531328166666665,"latency":null},{"benchmarkId":"arc-prize:v1-semi-private:overall","benchmarkName":"ARC-AGI-1 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":97.5,"unit":"percent","family":"arc-prize:v1-semi-private:overall","index":71.81162890019444,"aboveFit":false,"run":"high effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.40691867999999987,"latency":null},{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":96.52777777777779,"unit":"percent","family":"gpqa-diamond","index":65.74174088635657,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.207061012675132,"costPerTest":null,"latency":null},{"benchmarkId":"graphwalksBfs256k1m","benchmarkName":"GraphWalks BFS 256K–1M","category":"reasoning","score":96.2,"unit":"percent","family":"graphwalksBfs256k1m","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tier-4-v2-private:overall","benchmarkName":"FrontierMath-Tier-4-v2-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":95.1219512195122,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tier-4-v2-private:overall","index":85.85696672385271,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":3.4059122057973035,"costPerTest":null,"latency":null},{"benchmarkId":"arc-prize:v2-semi-private:overall","benchmarkName":"ARC-AGI-2 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":92.08333333333333,"unit":"percent","family":"arc-prize:v2-semi-private:overall","index":81.46716911713024,"aboveFit":false,"run":"high effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.6838793166666667,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tiers-1-3-v2-private:overall","benchmarkName":"FrontierMath-Tiers-1-3-v2-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":90.52631578947368,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tiers-1-3-v2-private:overall","index":78.54337753868055,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.7377516319276178,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:vibe-code:overall","benchmarkName":"Vibe Code Bench v1.1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":89.06,"unit":"percent","family":"vals-ai:vibe-code:overall","index":74.38376982850355,"aboveFit":false,"run":"OpenHands","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","measuredAt":"2026-11-09","stderr":2.294,"costPerTest":8.254926,"latency":2172.019},{"benchmarkId":"epoch-ai:epoch-ebr-bench:overall","benchmarkName":"EBR-bench","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":87.61904761904763,"unit":"percent","family":"epoch-ai:epoch-ebr-bench:overall","index":92.63755902095683,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":6.834619092574916,"costPerTest":null,"latency":null},{"benchmarkId":"osWorld21Offline","benchmarkName":"OSWorld 2.1 offline subset (82 tasks)","category":"agentic","score":81.5,"unit":"percent","family":"osWorld21Offline","index":81.18403076043472,"aboveFit":false,"run":null,"source":"Mengqi Yuan, Zilong Zhou, Xinzhuang Xiong, Weiming Wu, Jiayang Sun, Jiamin Song, Kaiqian Cui, Bowen Wang, Haoyuan Wu, Yitong Li, Dunjie Lu, Haikong Lu, Qi Zhen, Xinyuan Wang, Jiaqi Deng, Yuhao Yang, Cheng Chen, Boyuan Zheng, Alex Su, Xiao Yu, Hao Zou, Saaket Agashe, Xing Han Lu, Manpreet Kaur, Zhengyang Qi, Vincent Sunn Chen, Frederic Sala, Dayiheng Liu, Junyang Lin, Zhou Yu, Yu Su, Siva Reddy, Xin Eric Wang, Peng Qi, Tianbao Xie, Tao Yu","sourceUrl":"https://github.com/xlang-ai/OSWorld-V2/releases","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":79.3,"unit":"percent","family":"lcr","index":61.70423513811355,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-chess-puzzles:overall","benchmarkName":"Chess Puzzles","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":78,"unit":"percent","family":"epoch-ai:epoch-chess-puzzles:overall","index":93.18786094684646,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":4.163331998932266,"costPerTest":null,"latency":null},{"benchmarkId":"aaAutomationBench","benchmarkName":"Artificial Analysis AutomationBench","category":"agentic","score":77.8,"unit":"percent","family":"aaAutomationBench","index":86.61752174420346,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/automationbench-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-simpleqa-verified:overall","benchmarkName":"SimpleQA Verified","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":77.7,"unit":"percent","family":"epoch-ai:epoch-simpleqa-verified:overall","index":88.65001153284064,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.3169830843425607,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-furniture-assembly:overall","benchmarkName":"Furniture Assembly","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":77.5,"unit":"percent","family":"epoch-ai:epoch-furniture-assembly:overall","index":82.79219088539732,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":5.371114414765075,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-mystery-game-puzzles:overall","benchmarkName":"Mystery Game Puzzles","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":73,"unit":"percent","family":"epoch-ai:epoch-mystery-game-puzzles:overall","index":95,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":4.4619604333847365,"costPerTest":null,"latency":null},{"benchmarkId":"chartography","benchmarkName":"Chartography without tools","category":"multimodal","score":72.5,"unit":"percent","family":"chartography","index":null,"aboveFit":false,"run":null,"source":"Surge AI and Anthropic","sourceUrl":"https://surgehq.ai/blog/chartography","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"cweBenchV1","benchmarkName":"CWE-bench v1","category":"agentic","score":71,"unit":"percent","family":"cweBenchV1","index":81.98949574812671,"aboveFit":false,"run":null,"source":"Collinear AI","sourceUrl":"https://cwe-bench.com/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaTerminalBench4","benchmarkName":"Artificial Analysis Terminal-Bench v4.0","category":"agentic","score":65.2,"unit":"percent","family":"aaTerminalBench4","index":82.23742128580662,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/terminalbench-v4-0","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"arc-prize:v3-semi-private:overall","benchmarkName":"ARC-AGI-3 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":63.85513762054875,"unit":"percent","family":"arc-prize:v3-semi-private:overall","index":92.43525632809536,"aboveFit":false,"run":"medium effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:terminal-bench-4:overall","benchmarkName":"Terminal-Bench 4.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":63.131,"unit":"percent","family":"terminal-bench-4","index":84.49305110595951,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-4","measuredAt":"2026-11-09","stderr":2.201,"costPerTest":7.452059,"latency":2706.77},{"benchmarkId":"aaBriefcaseRubric","benchmarkName":"AA-Briefcase v1.1, rubric pass rate","category":"agentic","score":61.8,"unit":"percent","family":"aaBriefcaseRubric","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"frontierSweV2","benchmarkName":"FrontierSWE v2","category":"coding","score":61.2,"unit":"percent","family":"frontierSweV2","index":82.72672003145544,"aboveFit":false,"run":null,"source":"Proximal","sourceUrl":"https://www.frontierswe.com/blog/v2","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"cursorBench40","benchmarkName":"cursorBench40","category":"coding","score":60.4,"unit":"percent","family":"cursorBench40","index":83.38879418069476,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"blueprintBench2","benchmarkName":"Blueprint-Bench 2","category":"multimodal","score":58.6,"unit":"percent","family":"blueprintBench2","index":null,"aboveFit":false,"run":null,"source":"Google DeepMind","sourceUrl":"https://x.com/GoogleDeepMind","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"omniscienceAccuracy","benchmarkName":"Artificial Analysis Omniscience Accuracy","category":"knowledge","score":58.3,"unit":"percent","family":"omniscienceAccuracy","index":84.16287433729556,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:legal_research:overall","benchmarkName":"Legal Research Bench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":55.769,"unit":"percent","family":"vals-ai:legal_research:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_research","measuredAt":"2026-11-09","stderr":3.452,"costPerTest":8.743524,"latency":3112.579},{"benchmarkId":"mka","benchmarkName":"Multimodal Knowledge Agent Benchmark","category":"agentic","score":54.3,"unit":"percent","family":"mka","index":81.63480551386019,"aboveFit":false,"run":null,"source":"Google DeepMind, in partnership with Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mka","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"automationBench","benchmarkName":"AutomationBench","category":"agentic","score":53.9,"unit":"percent","family":"automationBench","index":84.6131518891373,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"exploitGym","benchmarkName":"ExploitGym","category":"agentic","score":53.5,"unit":"percent","family":"exploitGym","index":91.45047072765465,"aboveFit":false,"run":null,"source":"Zhun Wang, Nico Schiller, Hongwei Li, Srijiith Sesha Narayana, Milad Nasr, Nicholas Carlini, Xiangyu Qi, Eric Wallace, Elie Bursztein, Luca Invernizzi, Kurt Thomas, Yan Shoshitaishvili, Wenbo Guo, Jingxuan He, Thorsten Holz, Dawn Song","sourceUrl":"https://arxiv.org/abs/2605.11086","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:tax_agent_bench:overall","benchmarkName":"Tax Agent Bench","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":51.785,"unit":"percent","family":"vals-ai:tax_agent_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench","measuredAt":"2026-11-09","stderr":3.604,"costPerTest":6.340026,"latency":2086.243},{"benchmarkId":"frontierCode2","benchmarkName":"FrontierCode 2.0","category":"coding","score":40,"unit":"percent","family":"frontierCode2","index":81.6820347145763,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaGdpPdf","benchmarkName":"Artificial Analysis GDP.pdf","category":"agentic","score":28.4,"unit":"percent","family":"aaGdpPdf","index":76.01246382897048,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdp-pdf","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lmarena:agent-steerability:overall","benchmarkName":"Agent Arena steerability","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":19.063200731516666,"unit":"index","family":"agent-arena","index":82.20374087523268,"aboveFit":false,"run":"high effort","source":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset","measuredAt":"2026-11-11","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lmarena:agent-task-outcome:overall","benchmarkName":"Agent Arena task outcome","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":18.36669695915689,"unit":"index","family":"agent-arena","index":81.63110558253179,"aboveFit":false,"run":"high effort","source":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset","measuredAt":"2026-11-11","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:hlab:overall","benchmarkName":"HLAB-Vals-1.1","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":17.5,"unit":"percent","family":"vals-ai:hlab:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/hlab","measuredAt":"2026-11-09","stderr":2.997,"costPerTest":6.515809,"latency":1720.841},{"benchmarkId":"epoch-ai:epoch-frontiermath-erdos:overall","benchmarkName":"FrontierMath-Erdos","category":"math","domain":null,"eligible":true,"ranked":true,"score":2.94,"unit":"percent","family":"epoch-ai:epoch-frontiermath-erdos:overall","index":null,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.06,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":2,"outputPerMillion":10,"batch":{"inputPerMillion":1,"outputPerMillion":5},"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":31,"timeToFirstToken":8.1185,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":{"elo":1538.2768191996465,"eloUpper":1547.8165663855534,"eloLower":1528.7370720137392,"votes":4282,"rank":1,"rankUpper":null,"rankLower":null,"arenaName":"Gemini 4 Carbon-high","organization":"google","updatedAt":"2026-11-09"},"quality":{"coveragePercent":80,"benchmarkCount":28,"categoryCount":5,"confidence":"medium","state":"available"},"hosts":[{"host":"google-ai-studio","name":"Google AI Studio","kind":"maker","url":"https://ai.google.dev/gemini-api/docs/models","inputPerMillion":2,"outputPerMillion":10,"cacheReadPerMillion":0.1,"contextTokens":1048576,"maxOutputTokens":32768,"deprecated":false},{"host":"google-vertex","name":"Google Vertex AI","kind":"maker","url":"https://cloud.google.com/vertex-ai/generative-ai/docs/models","inputPerMillion":2,"outputPerMillion":10,"cacheReadPerMillion":0.1,"contextTokens":1048576,"maxOutputTokens":65536,"deprecated":false},{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":2,"outputPerMillion":10,"cacheReadPerMillion":0.1,"contextTokens":1048576,"maxOutputTokens":943718,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:arc-prize-foundation","name":"ARC Prize Foundation","url":"https://arcprize.org","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:lmarena","name":"LMArena","url":"https://huggingface.co","license":"CC BY 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"claude-opus-5-5","providerId":"anthropic","slug":"claude-opus-5-5","name":"Claude Opus 5.5","provider":"Anthropic","description":"Claude Opus 5.5 is a reasoning model from Anthropic. 73 benchmarks count toward its index, in 7 categories.","logoUrl":null,"releaseDate":"2026-09-22","contextWindowTokens":1000000,"modalities":["text","image","file"],"openWeights":false,"details":{"apiId":"anthropic/claude-opus-5.5","maxOutputTokens":128000,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.19999999999999998,"weightsUrl":null},"reasoning":true,"overallScore":78.58507764985569,"overallScoreRange":{"low":76.19252229969426,"high":80.97763300001712},"badges":[],"capabilityScores":{"agentic":74.14897990866976,"coding":81.7650203463107,"reasoning":80.83770072659966,"multimodal":74.06910255645644,"knowledge":79.2078569385896,"multilingual":null,"instruction":59.60287879646549,"math":77.71500008994444},"capabilityCounts":{"math":5,"reasoning":10,"coding":17,"knowledge":6,"multimodal":4,"agentic":30,"instruction":1},"benchmarks":[{"benchmarkId":"vals-ai:proof_bench:overall","benchmarkName":"ProofBench v1.1","category":"math","domain":null,"eligible":true,"ranked":true,"score":100,"unit":"percent","family":"vals-ai:proof_bench:overall","index":81.68109068983412,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/proof_bench","measuredAt":"2026-11-09","stderr":0,"costPerTest":0.924063,"latency":351.035},{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":100,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":67.09849544987334,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0,"costPerTest":null,"latency":null},{"benchmarkId":"arc-prize:v1-public-eval:overall","benchmarkName":"ARC-AGI-1 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":98.25,"unit":"percent","family":"arc-prize:v1-public-eval:overall","index":null,"aboveFit":false,"run":"max effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.38976489000000003,"latency":null},{"benchmarkId":"arc-prize:v2-public-eval:overall","benchmarkName":"ARC-AGI-2 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":97.91666666666666,"unit":"percent","family":"arc-prize:v2-public-eval:overall","index":null,"aboveFit":false,"run":"max effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":1.8968451666666666,"latency":null},{"benchmarkId":"arc-prize:v1-semi-private:overall","benchmarkName":"ARC-AGI-1 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":97.5,"unit":"percent","family":"arc-prize:v1-semi-private:overall","index":71.81162890019444,"aboveFit":false,"run":"max effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.64073264,"latency":null},{"benchmarkId":"livebench:livebench-math:overall","benchmarkName":"LiveBench Mathematics","category":"math","domain":null,"eligible":true,"ranked":true,"score":97.07749999999999,"unit":"percent","family":"livebench:livebench-math:overall","index":75.0736089668336,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Mathematics","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.22107625329815306,"latency":null},{"benchmarkId":"arxivMathAugust2026WithTools","benchmarkName":"ArXivMath August 2026 with tools","category":"math","score":96.9,"unit":"percent","family":"arxivMathAugust2026WithTools","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-coding-2:overall","benchmarkName":"LiveBench Coding 2.0","category":"coding","domain":null,"eligible":true,"ranked":true,"score":96.3,"unit":"percent","family":"livebench:livebench-coding-2:overall","index":77.92442369803838,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Coding","measuredAt":"2026-10-14","stderr":null,"costPerTest":5.030782758620689,"latency":null},{"benchmarkId":"benchCadVision2CodeWithTools","benchmarkName":"BenchCAD Vision2Code voxel IoU with tools","category":"multimodal","score":96.2,"unit":"percent","family":"benchCadVision2CodeWithTools","index":null,"aboveFit":false,"run":null,"source":"Zhang et al. and Anthropic","sourceUrl":"https://arxiv.org/abs/2605.10865","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-mirrorcode:overall","benchmarkName":"MirrorCode","category":"coding","domain":null,"eligible":true,"ranked":true,"score":95.55555555555556,"unit":"percent","family":"epoch-ai:epoch-mirrorcode:overall","index":91.53089000225089,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.5545201491160445,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:ioi:overall","benchmarkName":"IOI","category":"coding","domain":null,"eligible":true,"ranked":true,"score":95.056,"unit":"percent","family":"vals-ai:ioi:overall","index":81.80278590361968,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/ioi","measuredAt":"2026-10-29","stderr":4.944,"costPerTest":4.345967,"latency":1035.191},{"benchmarkId":"epoch-ai:epoch-frontiermath-tier-4-v2-private:overall","benchmarkName":"FrontierMath-Tier-4-v2-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":95,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tier-4-v2-private:overall","index":85.80905818253655,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":4.117547077105885,"costPerTest":null,"latency":null},{"benchmarkId":"gmmlu","benchmarkName":"Global MMLU","category":"multilingual","score":94.3,"unit":"percent","family":"gmmlu","index":null,"aboveFit":false,"run":null,"source":"Singh et al.","sourceUrl":"https://arxiv.org/abs/2412.03304","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"milu","benchmarkName":"Multi-task Indic Language Understanding Benchmark","category":"multilingual","score":93.1,"unit":"percent","family":"milu","index":null,"aboveFit":false,"run":null,"source":"Verma et al.","sourceUrl":"https://arxiv.org/abs/2411.02538","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-reasoning:overall","benchmarkName":"LiveBench Reasoning","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":92.15375,"unit":"percent","family":"livebench","index":80.99882195781015,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Reasoning","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.25223514851485146,"latency":null},{"benchmarkId":"arc-prize:v2-semi-private:overall","benchmarkName":"ARC-AGI-2 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":91.66666666666666,"unit":"percent","family":"arc-prize:v2-semi-private:overall","index":81.27586299172351,"aboveFit":false,"run":"max effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":1.8517409666666667,"latency":null},{"benchmarkId":"vals-ai:medscribe:overall","benchmarkName":"MedScribe","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":91.43,"unit":"percent","family":"vals-ai:medscribe:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medscribe","measuredAt":"2026-10-29","stderr":1.932,"costPerTest":1.154156,"latency":441.582},{"benchmarkId":"vals-ai:time_horizon_index:overall","benchmarkName":"Time Horizon Index: KSP","category":null,"domain":"Other","eligible":false,"ranked":true,"score":91.333,"unit":"percent","family":"vals-ai:time_horizon_index:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/time_horizon_index","measuredAt":"2026-10-07","stderr":0,"costPerTest":921.761327,"latency":0},{"benchmarkId":"vals-ai:sage:overall","benchmarkName":"SAGE","category":null,"domain":"Education","eligible":false,"ranked":true,"score":91.302,"unit":"percent","family":"vals-ai:sage:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/sage","measuredAt":"2026-11-05","stderr":1.382,"costPerTest":0.283134,"latency":88.09},{"benchmarkId":"epoch-ai:epoch-frontiermath-tiers-1-3-v2-private:overall","benchmarkName":"FrontierMath-Tiers-1-3-v2-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":91.22807017543859,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tiers-1-3-v2-private:overall","index":78.91274716064454,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.6786209858442096,"costPerTest":null,"latency":null},{"benchmarkId":"aaHarveyLab","benchmarkName":"Artificial Analysis Harvey LAB-AA v1.0","category":"agentic","score":91.2,"unit":"percent","family":"aaHarveyLab","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"legalAgentBenchHeldoutCriterionPass","benchmarkName":"Legal Agent Benchmark mean criterion-pass rate — Harvey held-out set","category":"agentic","score":91.2,"unit":"percent","family":"legalAgentBenchHeldoutCriterionPass","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"programBench","benchmarkName":"ProgramBench: Can Language Models Rebuild Programs From Scratch?","category":"coding","score":91.2,"unit":"percent","family":"programBench","index":95,"aboveFit":false,"run":null,"source":"John Yang, Kilian Lieret, Jeffrey Ma, Parth Thakkar, Dmitrii Pedchenko, Sten Sootla, Emily McMilin, Pengcheng Yin, Rui Hou, Gabriel Synnaeve, Diyi Yang, Ofir Press","sourceUrl":"https://programbench.com/static/paper.pdf","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"arxivMathAugust2026","benchmarkName":"ArXivMath August 2026 without tools","category":"math","score":91.2,"unit":"percent","family":"arxivMathAugust2026","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":90.59343434343434,"unit":"percent","family":"gpqa-diamond","index":60.29892324478712,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.9434397190192136,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:vibe-code:overall","benchmarkName":"Vibe Code Bench v1.1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":90.294,"unit":"percent","family":"vals-ai:vibe-code:overall","index":74.8290164160976,"aboveFit":false,"run":"OpenHands","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","measuredAt":"2026-11-09","stderr":1.526,"costPerTest":35.774773,"latency":5766.524},{"benchmarkId":"swePro","benchmarkName":"SWE-bench Pro","category":"coding","score":89.9,"unit":"percent","family":"swePro","index":85.39912098271319,"aboveFit":false,"run":null,"source":"Xiang Deng, Jeff Da, Edwin Pan, Yannis Yiming He, Charles Ide, Kanak Garg, Niklas Lauffer, Andrew Park, Nitin Pasari, Chetan Rane, Karmini Sampath, Maya Krishnan, Srivatsa Kundurthy, Sean Hendryx, Zifan Wang, Chen Bo Calvin Zhang, Noah Jacobson, Bing Liu, Brad Kenstler","sourceUrl":"https://arxiv.org/abs/2509.16941","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"bioMysteryBenchHumanSolvable","benchmarkName":"BioMysteryBench Human Solvable","category":"knowledge","score":89.3,"unit":"percent","family":"bioMysteryBenchHumanSolvable","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-coding:overall","benchmarkName":"LiveBench Coding","category":"coding","domain":null,"eligible":true,"ranked":true,"score":89.253,"unit":"percent","family":"livebench:livebench-coding:overall","index":80.35018963292968,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Coding","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.3027102564102564,"latency":null},{"benchmarkId":"chartographyWithTools","benchmarkName":"Chartography with image and code tools","category":"multimodal","score":89,"unit":"percent","family":"chartographyWithTools","index":null,"aboveFit":false,"run":null,"source":"Surge AI and Anthropic","sourceUrl":"https://surgehq.ai/blog/chartography","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaMmmuPro","benchmarkName":"Artificial Analysis MMMU-Pro","category":"multimodal","score":87.7,"unit":"percent","family":"mmmu-pro","index":69.28667067878646,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmmu-pro","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:terminal-bench-2-1:overall","benchmarkName":"Terminal-Bench 2.1","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":87.64,"unit":"percent","family":"terminal-bench-2-1","index":72.39784534976677,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2-1","measuredAt":"2026-10-01","stderr":1.716,"costPerTest":0.496033,"latency":383.079},{"benchmarkId":"draco","benchmarkName":"Data Research and Analysis with Complex Operations","category":"agentic","score":87.4,"unit":"percent","family":"draco","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"osWorld21Offline","benchmarkName":"OSWorld 2.1 offline subset (82 tasks)","category":"agentic","score":87.2,"unit":"percent","family":"osWorld21Offline","index":84.78654148194153,"aboveFit":false,"run":null,"source":"Mengqi Yuan, Zilong Zhou, Xinzhuang Xiong, Weiming Wu, Jiayang Sun, Jiamin Song, Kaiqian Cui, Bowen Wang, Haoyuan Wu, Yitong Li, Dunjie Lu, Haikong Lu, Qi Zhen, Xinyuan Wang, Jiaqi Deng, Yuhao Yang, Cheng Chen, Boyuan Zheng, Alex Su, Xiao Yu, Hao Zou, Saaket Agashe, Xing Han Lu, Manpreet Kaur, Zhengyang Qi, Vincent Sunn Chen, Frederic Sala, Dayiheng Liu, Junyang Lin, Zhou Yu, Yu Su, Siva Reddy, Xin Eric Wang, Peng Qi, Tianbao Xie, Tao Yu","sourceUrl":"https://github.com/xlang-ai/OSWorld-V2/releases","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-design:overall","benchmarkName":"LiveBench Design","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":86.9,"unit":"percent","family":"livebench-2-agentic","index":82.14214373095791,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Design","measuredAt":"2026-10-14","stderr":null,"costPerTest":5.748296,"latency":null},{"benchmarkId":"livebench:livebench-language:overall","benchmarkName":"LiveBench Language","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":86.27,"unit":"percent","family":"livebench:livebench-language:overall","index":72.11368754594334,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Language","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.28351866666666664,"latency":null},{"benchmarkId":"extractBenchLlamaIndex","benchmarkName":"ExtractBench: schema-guided enterprise document extraction (LlamaIndex leaderboard)","category":"multimodal","score":86.06,"unit":"percent","family":"extractBenchLlamaIndex","index":66.2392717924913,"aboveFit":false,"run":null,"source":"Boyang Zhang, Adrian Lyjak, Eli Stewart, Zhaoqi Li and Simon Suo (LlamaIndex)","sourceUrl":"https://arxiv.org/abs/2607.29677","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-long-horizon:overall","benchmarkName":"LiveBench Long Agentic","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":85,"unit":"percent","family":"livebench-2-agentic","index":80.75261465060905,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Long%20Agentic","measuredAt":"2026-10-14","stderr":null,"costPerTest":38.00877222222223,"latency":null},{"benchmarkId":"livebench:livebench-research:overall","benchmarkName":"LiveBench Research","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":85,"unit":"percent","family":"livebench-2-agentic","index":80.75261465060905,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Research","measuredAt":"2026-10-14","stderr":null,"costPerTest":8.81775,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":84.7,"unit":"percent","family":"lcr","index":65.40906997627613,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-furniture-assembly:overall","benchmarkName":"Furniture Assembly","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":83.33333333333334,"unit":"percent","family":"epoch-ai:epoch-furniture-assembly:overall","index":85.51155190501171,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":4.704047221664923,"costPerTest":null,"latency":null},{"benchmarkId":"proteinBinderDesign","benchmarkName":"Anthropic de novo protein-binder design evaluation","category":"knowledge","score":82.6,"unit":"percent","family":"proteinBinderDesign","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"toolathlonVerifiedPass3","benchmarkName":"Toolathlon Verified Pass@3","category":"agentic","score":82.4,"unit":"percent","family":"toolathlonVerifiedPass3","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"osWorld21","benchmarkName":"OSWorld 2.1 (all 108 tasks)","category":"agentic","score":81.8,"unit":"percent","family":"osWorld21","index":78.92883407726703,"aboveFit":false,"run":null,"source":"Mengqi Yuan, Zilong Zhou, Xinzhuang Xiong, Weiming Wu, Jiayang Sun, Jiamin Song, Kaiqian Cui, Bowen Wang, Haoyuan Wu, Yitong Li, Dunjie Lu, Haikong Lu, Qi Zhen, Xinyuan Wang, Jiaqi Deng, Yuhao Yang, Cheng Chen, Boyuan Zheng, Alex Su, Xiao Yu, Hao Zou, Saaket Agashe, Xing Han Lu, Manpreet Kaur, Zhengyang Qi, Vincent Sunn Chen, Frederic Sala, Dayiheng Liu, Junyang Lin, Zhou Yu, Yu Su, Siva Reddy, Xin Eric Wang, Peng Qi, Tianbao Xie, Tao Yu","sourceUrl":"https://github.com/xlang-ai/OSWorld-V2/releases","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"cveRediscovery","benchmarkName":"Anthropic CVE Rediscovery","category":"agentic","score":81.3,"unit":"percent","family":"cveRediscovery","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-data-analysis:overall","benchmarkName":"LiveBench Data Analysis","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":80.307,"unit":"percent","family":"livebench","index":65.65213926396092,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Data%20Analysis","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.8145593333333334,"latency":null},{"benchmarkId":"livebench:livebench-browser:overall","benchmarkName":"LiveBench Browser Use","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":80,"unit":"percent","family":"livebench-2-agentic","index":77.09595917600676,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Browser","measuredAt":"2026-10-14","stderr":null,"costPerTest":14.669788,"latency":null},{"benchmarkId":"vals-ai:biomysterybench:overall","benchmarkName":"BioMysteryBench","category":null,"domain":"Science","eligible":false,"ranked":true,"score":79.259,"unit":"percent","family":"vals-ai:biomysterybench:overall","index":null,"aboveFit":false,"run":"Terminus 2","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/biomysterybench","measuredAt":"2026-11-09","stderr":0.98,"costPerTest":2.947297,"latency":739.252},{"benchmarkId":"officeQa","benchmarkName":"OfficeQA","category":"multimodal","score":78.9,"unit":"percent","family":"officeQa","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"parseBenchLlamaIndex","benchmarkName":"ParseBench: document parsing for AI agents (LlamaIndex leaderboard)","category":"multimodal","score":78.8,"unit":"percent","family":"parseBenchLlamaIndex","index":83.66586843841714,"aboveFit":false,"run":null,"source":"Boyang Zhang, Sebastián G. Acosta, Preston Carlson, Sacha Bron, Pierre-Loïc Doulcet, Daniel B. Ospina and Simon Suo (LlamaIndex)","sourceUrl":"https://arxiv.org/abs/2604.08538","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"toolathlonVerified","benchmarkName":"Toolathlon-Verified","category":"agentic","score":77.8,"unit":"percent","family":"toolathlonVerified","index":71.6099643123515,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"healthBenchProfessionalRaw","benchmarkName":"HealthBench Professional raw score","category":"knowledge","score":77.1,"unit":"percent","family":"healthBenchProfessionalRaw","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hermesBench","benchmarkName":"Hermes Bench: 150 agent tasks graded on the files and state the agent leaves behind","category":"agentic","score":76.73,"unit":"percent","family":"hermesBench","index":null,"aboveFit":false,"run":null,"source":"Nous Research","sourceUrl":"https://portal.nousresearch.com/bench","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:emb:overall","benchmarkName":"EMB","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":75.938,"unit":"percent","family":"vals-ai:emb:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/emb","measuredAt":"2026-10-20","stderr":2.383,"costPerTest":9.330415,"latency":2106.434},{"benchmarkId":"deepSwe","benchmarkName":"DeepSWE","category":"agentic","score":74.2,"unit":"percent","family":"deepSwe","index":75.0180492652176,"aboveFit":false,"run":null,"source":"Datacurve AI","sourceUrl":"https://deepswe.datacurve.ai/blog","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"protocolsTroubleshooting","benchmarkName":"Molecular Biology Protocols Troubleshooting","category":"knowledge","score":73.7,"unit":"percent","family":"protocolsTroubleshooting","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"proteinDomainIdentification","benchmarkName":"Protein domain identification","category":"knowledge","score":73.6,"unit":"percent","family":"proteinDomainIdentification","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"benchCadVision2Code","benchmarkName":"BenchCAD Vision2Code voxel IoU without tools","category":"multimodal","score":73,"unit":"percent","family":"benchCadVision2Code","index":null,"aboveFit":false,"run":null,"source":"Zhang et al. and Anthropic","sourceUrl":"https://arxiv.org/abs/2605.10865","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"wandr","benchmarkName":"WANDR (Wide ANd Deep Research)","category":"agentic","score":72.3,"unit":"percent","family":"wandr","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"toolathlonVerifiedPass3All","benchmarkName":"Toolathlon Verified Pass cubed","category":"agentic","score":72.2,"unit":"percent","family":"toolathlonVerifiedPass3All","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-simpleqa-verified:overall","benchmarkName":"SimpleQA Verified","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":72.2,"unit":"percent","family":"epoch-ai:epoch-simpleqa-verified:overall","index":84.06802600206248,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.4174516461485274,"costPerTest":null,"latency":null},{"benchmarkId":"spatialBenchVerified","benchmarkName":"LatchBio SpatialBench Verified","category":"knowledge","score":72,"unit":"percent","family":"spatialBenchVerified","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-agentic-coding:overall","benchmarkName":"LiveBench Agentic Coding","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":71.717,"unit":"percent","family":"livebench:livebench-agentic-coding:overall","index":83.1232110015922,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Agentic%20Coding","measuredAt":"2026-06-25","stderr":null,"costPerTest":4.477686111111112,"latency":null},{"benchmarkId":"epoch-ai:epoch-ebr-bench:overall","benchmarkName":"EBR-bench","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":71.42857142857142,"unit":"percent","family":"epoch-ai:epoch-ebr-bench:overall","index":85.20730611637052,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":7.063046178186346,"costPerTest":null,"latency":null},{"benchmarkId":"biomedicalImageAnalysis","benchmarkName":"Anthropic biomedical-image-analysis evaluation","category":"multimodal","score":71.4,"unit":"percent","family":"biomedicalImageAnalysis","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaHealthBenchProfessionalRaw","benchmarkName":"Artificial Analysis HealthBench Professional raw score","category":"knowledge","score":71.4,"unit":"percent","family":"aaHealthBenchProfessionalRaw","index":null,"aboveFit":false,"run":null,"source":"OpenAI (benchmark); Artificial Analysis (evaluation)","sourceUrl":"https://artificialanalysis.ai/evaluations/healthbench-professional","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-mystery-game-puzzles:overall","benchmarkName":"Mystery Game Puzzles","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":71,"unit":"percent","family":"epoch-ai:epoch-mystery-game-puzzles:overall","index":93.6644558681374,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":4.560480215720688,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:public-benefits-bench:overall","benchmarkName":"Public Benefits Bench v1.1","category":null,"domain":"Public benefits","eligible":false,"ranked":true,"score":70.636,"unit":"percent","family":"vals-ai:public-benefits-bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/public-benefits-bench","measuredAt":"2026-10-12","stderr":1.185,"costPerTest":5.285453,"latency":4771.81},{"benchmarkId":"scarfBench","benchmarkName":"ScarfBench: Cross-Framework Application Migration in Enterprise Java","category":"coding","score":70.5,"unit":"percent","family":"scarfBench","index":74.73882455545224,"aboveFit":false,"run":null,"source":"Advait Pavuluri, Bridget McGinn, Ashita Saxena, George Safta, Srikanth Tamilselvam, Raju Pavuluri, Michele Merler, Baishakhi Ray, Rahul Krishna (IBM Research)","sourceUrl":"https://arxiv.org/abs/2605.06754","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaAutomationBench","benchmarkName":"Artificial Analysis AutomationBench","category":"agentic","score":69.5,"unit":"percent","family":"aaAutomationBench","index":79.0424774028398,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/automationbench-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"protocolsUnderstanding","benchmarkName":"Benchling Molecular Biology Protocols Understanding","category":"knowledge","score":69,"unit":"percent","family":"protocolsUnderstanding","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"physicianBench","benchmarkName":"PhysicianBench","category":"knowledge","score":68.4,"unit":"percent","family":"physicianBench","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gdpvalAaNormalized","benchmarkName":"GDPval-AA normalized","category":"agentic","score":68.3,"unit":"percent","family":"gdpvalAaNormalized","index":82.86668774074491,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"healthBench","benchmarkName":"HealthBench raw score","category":"knowledge","score":68.1,"unit":"percent","family":"healthBench","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hleWithTools","benchmarkName":"Humanity's Last Exam with tools","category":"agentic","score":67.7,"unit":"percent","family":"hleWithTools","index":75.12342825153976,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"officeQaPro","benchmarkName":"OfficeQA Pro","category":"multimodal","score":67.7,"unit":"percent","family":"officeQaPro","index":77.08459931613089,"aboveFit":false,"run":null,"source":"OfficeQA Pro authors","sourceUrl":"https://arxiv.org/abs/2603.08655","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"cweBenchV2","benchmarkName":"CWE-bench v2","category":"agentic","score":67.1,"unit":"percent","family":"cweBenchV2","index":79.94465324419019,"aboveFit":false,"run":null,"source":"Collinear AI","sourceUrl":"https://cwe-bench.com/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"cweBenchV1","benchmarkName":"CWE-bench v1","category":"agentic","score":67,"unit":"percent","family":"cweBenchV1","index":78.63035574799756,"aboveFit":false,"run":null,"source":"Collinear AI","sourceUrl":"https://cwe-bench.com/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-chess-puzzles:overall","benchmarkName":"Chess Puzzles","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":67,"unit":"percent","family":"epoch-ai:epoch-chess-puzzles:overall","index":84.83869380019539,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":4.725815626252609,"costPerTest":null,"latency":null},{"benchmarkId":"aaSciCode","benchmarkName":"Artificial Analysis SciCode","category":"coding","score":66.9,"unit":"percent","family":"scicode-family","index":83.05673613765691,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/scicode","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"graphwalksBfs256k1m","benchmarkName":"GraphWalks BFS 256K–1M","category":"reasoning","score":66.8,"unit":"percent","family":"graphwalksBfs256k1m","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:code-migration:overall","benchmarkName":"Code Migration","category":"coding","domain":null,"eligible":true,"ranked":true,"score":66.646,"unit":"percent","family":"vals-ai:code-migration:overall","index":80.59416472945891,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/code-migration","measuredAt":"2026-10-29","stderr":4.329,"costPerTest":69.480849,"latency":11925.532},{"benchmarkId":"terminal-bench:terminal-bench-4-1-0:overall","benchmarkName":"Terminal-Bench 4.1.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":65.76,"unit":"percent","family":"terminal-bench:terminal-bench-4-1-0:overall","index":83.62534227535137,"aboveFit":false,"run":"max effort · Claude Code","source":"Terminal-Bench","sourceUrl":"https://www.tbench.ai/leaderboard","measuredAt":"2026-09-29","stderr":1.586734693877551,"costPerTest":14.298939393939392,"latency":4790.4},{"benchmarkId":"livebench:livebench-instruction-following:overall","benchmarkName":"LiveBench Instruction Following","category":"instruction","domain":null,"eligible":true,"ranked":true,"score":65.73775,"unit":"percent","family":"livebench:livebench-instruction-following:overall","index":59.60287879646549,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=IF","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.937248,"latency":null},{"benchmarkId":"healthBenchProfessional","benchmarkName":"HealthBench Professional","category":"knowledge","score":65.6,"unit":"percent","family":"healthBenchProfessional","index":78.0360294405329,"aboveFit":false,"run":null,"source":"Rebecca Soskin Hicks, Mikhail Trofimov, Dominick Lim, Rahul K. Arora, Foivos Tsimpourlas, Preston Bowman, Michael Sharman, Chi Tong, Kavin Karthik, Arnav Dugar, Akshay Jagadeesh, Khaled Saab, Johannes Heidecke, Ashley Alexander, Nate Gross, Karan Singhal","sourceUrl":"https://arxiv.org/abs/2604.27470","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"omniscienceAccuracy","benchmarkName":"Artificial Analysis Omniscience Accuracy","category":"knowledge","score":65.5,"unit":"percent","family":"omniscienceAccuracy","index":93.23671742252421,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"benchCad21WithTools","benchmarkName":"BenchCAD 2.1 Core, with tools","category":"multimodal","score":65.3,"unit":"percent","family":"benchCad21WithTools","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:terminal-bench-4:overall","benchmarkName":"Terminal-Bench 4.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":65.152,"unit":"percent","family":"terminal-bench-4","index":85.45269998005702,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-4","measuredAt":"2026-11-09","stderr":0,"costPerTest":12.042479,"latency":3810.852},{"benchmarkId":"terminal-bench:terminal-bench-4-0-0:overall","benchmarkName":"Terminal-Bench 4.0.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":64.85,"unit":"percent","family":"terminal-bench-4","index":85.30929871336852,"aboveFit":false,"run":"max effort · Claude Code","source":"Terminal-Bench","sourceUrl":"https://www.tbench.ai/leaderboard","measuredAt":"2026-09-29","stderr":1.586734693877551,"costPerTest":14.298939393939392,"latency":4790.4},{"benchmarkId":"chartography","benchmarkName":"Chartography without tools","category":"multimodal","score":64.4,"unit":"percent","family":"chartography","index":null,"aboveFit":false,"run":null,"source":"Surge AI and Anthropic","sourceUrl":"https://surgehq.ai/blog/chartography","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hleNoTools","benchmarkName":"Humanity's Last Exam without tools","category":"knowledge","score":64.4,"unit":"percent","family":"hle","index":88.99869709323224,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaHealthBench","benchmarkName":"Artificial Analysis HealthBench raw score","category":"knowledge","score":64.3,"unit":"percent","family":"aaHealthBench","index":null,"aboveFit":false,"run":null,"source":"OpenAI (benchmark); Artificial Analysis (evaluation)","sourceUrl":"https://artificialanalysis.ai/evaluations/healthbench","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"frontierCode11Extended","benchmarkName":"FrontierCode 1.1 Extended","category":"coding","score":63.6,"unit":"percent","family":"frontierCode11Extended","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"medicinalChemistry","benchmarkName":"Anthropic medicinal-chemistry evaluation","category":"knowledge","score":63.5,"unit":"percent","family":"medicinalChemistry","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hermesIndex","benchmarkName":"Hermes Index: agentic capability and cost per task inside Hermes Agent","category":"agentic","score":63.31,"unit":"percent","family":"hermesIndex","index":85.16820081869221,"aboveFit":false,"run":null,"source":"Nous Research","sourceUrl":"https://portal.nousresearch.com/bench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"frontierSweV2","benchmarkName":"FrontierSWE v2","category":"coding","score":62.3,"unit":"percent","family":"frontierSweV2","index":83.25720343374732,"aboveFit":false,"run":null,"source":"Proximal","sourceUrl":"https://www.frontierswe.com/blog/v2","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":61.4,"unit":"percent","family":"hle","index":85.98881885814285,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaBriefcaseRubric","benchmarkName":"AA-Briefcase v1.1, rubric pass rate","category":"agentic","score":61.2,"unit":"percent","family":"aaBriefcaseRubric","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"singleCellBench","benchmarkName":"LatchBio SingleCellBench","category":"knowledge","score":61.2,"unit":"percent","family":"singleCellBench","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"healthBenchLengthAdjusted","benchmarkName":"HealthBench length-adjusted score","category":"knowledge","score":60.6,"unit":"percent","family":"healthBenchLengthAdjusted","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"proteinDesign","benchmarkName":"Anthropic Protein Design evaluation","category":"knowledge","score":60.2,"unit":"percent","family":"proteinDesign","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaHealthBenchProfessional","benchmarkName":"Artificial Analysis HealthBench Professional length-adjusted score","category":"knowledge","score":59.9,"unit":"percent","family":"aaHealthBenchProfessional","index":null,"aboveFit":false,"run":null,"source":"OpenAI (benchmark); Artificial Analysis (evaluation)","sourceUrl":"https://artificialanalysis.ai/evaluations/healthbench-professional","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaTerminalBench4","benchmarkName":"Artificial Analysis Terminal-Bench v4.0","category":"agentic","score":59.6,"unit":"percent","family":"aaTerminalBench4","index":79.53215389821452,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/terminalbench-v4-0","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"terminalBenchScience","benchmarkName":"Terminal-Bench-Science 0.1","category":"agentic","score":58.7,"unit":"percent","family":"terminalBenchScience","index":78.06097856183052,"aboveFit":false,"run":null,"source":"Terminal-Bench-Science Team","sourceUrl":"https://www.terminal-bench-science.ai/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:fabv2:overall","benchmarkName":"Finance Agent (v2)","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":58.587,"unit":"percent","family":"vals-ai:fabv2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/fabv2","measuredAt":"2026-10-29","stderr":0.17,"costPerTest":9.02999,"latency":1987.81},{"benchmarkId":"aaCweBench","benchmarkName":"Artificial Analysis CWE-Bench-AA","category":"agentic","score":58.3,"unit":"percent","family":"aaCweBench","index":70.43833818983995,"aboveFit":false,"run":null,"source":"Artificial Analysis; dataset by Collinear AI","sourceUrl":"https://artificialanalysis.ai/evaluations/cwe-bench-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"cursorBench40","benchmarkName":"cursorBench40","category":"coding","score":57.8,"unit":"percent","family":"cursorBench40","index":81.29640846289749,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaHealthBenchLengthAdjusted","benchmarkName":"Artificial Analysis HealthBench length-adjusted score","category":"knowledge","score":57,"unit":"percent","family":"aaHealthBenchLengthAdjusted","index":null,"aboveFit":false,"run":null,"source":"OpenAI (benchmark); Artificial Analysis (evaluation)","sourceUrl":"https://artificialanalysis.ai/evaluations/healthbench","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaAnalystAgent","benchmarkName":"Artificial Analysis AnalystAgent","category":"agentic","score":56.3,"unit":"percent","family":"aaAnalystAgent","index":79.13069265236766,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/aa-analyst-agent","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"proteinDesignLibraryRanking","benchmarkName":"Anthropic protein-design library-ranking task","category":"knowledge","score":56,"unit":"percent","family":"proteinDesignLibraryRanking","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:cyber:overall","benchmarkName":"CyberBench v1.1","category":null,"domain":"Other","eligible":false,"ranked":true,"score":55.357,"unit":"percent","family":"vals-ai:cyber:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/cyber","measuredAt":"2026-11-05","stderr":5.133,"costPerTest":2.846058,"latency":1091.126},{"benchmarkId":"frontierCode","benchmarkName":"FrontierCode 1.1 Main","category":"coding","score":54.4,"unit":"percent","family":"frontierCode","index":77.32378164812403,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:web_search:overall","benchmarkName":"Vals Web Search Index","category":null,"domain":"Other","eligible":false,"ranked":true,"score":53.747402,"unit":"percent","family":"vals-ai:web_search:overall","index":null,"aboveFit":false,"run":"Native","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/web_search","measuredAt":"2026-11-04","stderr":2.028077,"costPerTest":3.478783,"latency":596.794055},{"benchmarkId":"stemProfessorBench","benchmarkName":"Anthropic STEM Professor Bench","category":"reasoning","score":53.7,"unit":"percent","family":"stemProfessorBench","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mka","benchmarkName":"Multimodal Knowledge Agent Benchmark","category":"agentic","score":53,"unit":"percent","family":"mka","index":80.51495295798406,"aboveFit":false,"run":null,"source":"Google DeepMind, in partnership with Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mka","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"extractBenchPro","benchmarkName":"ExtractBench Pro: long-tail document extraction challenges (LlamaIndex)","category":"multimodal","score":51,"unit":"percent","family":"extractBenchPro","index":null,"aboveFit":false,"run":null,"source":"LlamaIndex","sourceUrl":"https://www.extractbench.ai/pro","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:legal_research:overall","benchmarkName":"Legal Research Bench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":50.481,"unit":"percent","family":"vals-ai:legal_research:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_research","measuredAt":"2026-11-09","stderr":3.475,"costPerTest":23.613691,"latency":6749.531},{"benchmarkId":"bioMysteryBenchHumanDifficult","benchmarkName":"BioMysteryBench Human Difficult","category":"knowledge","score":50,"unit":"percent","family":"bioMysteryBenchHumanDifficult","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:medcode:overall","benchmarkName":"MedCode","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":49.797,"unit":"percent","family":"vals-ai:medcode:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medcode","measuredAt":"2026-10-29","stderr":2.273,"costPerTest":0.634301,"latency":246.876},{"benchmarkId":"vals-ai:mysterymechanism:overall","benchmarkName":"MysteryMechanism","category":null,"domain":"Science","eligible":false,"ranked":true,"score":49.55,"unit":"percent","family":"vals-ai:mysterymechanism:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mysterymechanism","measuredAt":"2026-11-11","stderr":3.363,"costPerTest":4.061695,"latency":1178.454},{"benchmarkId":"proteinMutationAttribution","benchmarkName":"Protein mutation attribution","category":"knowledge","score":49.5,"unit":"percent","family":"proteinMutationAttribution","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"postTrainBench11","benchmarkName":"PostTrainBench v1.1","category":"coding","score":49.3,"unit":"percent","family":"postTrainBench11","index":81.64389672294374,"aboveFit":false,"run":null,"source":"PostTrainBench team","sourceUrl":"https://posttrainbench.com/?version=v1.1","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"benchCad21","benchmarkName":"BenchCAD 2.1 Core, without tools","category":"multimodal","score":49,"unit":"percent","family":"benchCad21","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"osWorld2","benchmarkName":"OSWorld 2.0","category":"agentic","score":48.7,"unit":"percent","family":"osWorld2","index":73.76209269967717,"aboveFit":false,"run":null,"source":"Mengqi Yuan, Zilong Zhou, Xinzhuang Xiong, Weiming Wu, Jiayang Sun, Jiamin Song, Kaiqian Cui, Bowen Wang, Haoyuan Wu, Yitong Li, Dunjie Lu, Haikong Lu, Qi Zhen, Xinyuan Wang, Jiaqi Deng, Yuhao Yang, Cheng Chen, Boyuan Zheng, Alex Su, Xiao Yu, Hao Zou, Saaket Agashe, Xing Han Lu, Manpreet Kaur, Zhengyang Qi, Vincent Sunn Chen, Frederic Sala, Dayiheng Liu, Junyang Lin, Zhou Yu, Yu Su, Siva Reddy, Xin Eric Wang, Peng Qi, Tianbao Xie, Tao Yu","sourceUrl":"https://arxiv.org/abs/2606.29537","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:terminal-bench-science:overall","benchmarkName":"Terminal-Bench-Science","category":null,"domain":"Science","eligible":false,"ranked":true,"score":47.143,"unit":"percent","family":"vals-ai:terminal-bench-science:overall","index":null,"aboveFit":false,"run":"Mini-SWE-agent","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-science","measuredAt":"2026-11-09","stderr":6.009,"costPerTest":19.121966,"latency":8856.116},{"benchmarkId":"frontierCode2","benchmarkName":"FrontierCode 2.0","category":"coding","score":45.5,"unit":"percent","family":"frontierCode2","index":85.75952517033565,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:tax_agent_bench:overall","benchmarkName":"Tax Agent Bench","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":44.537,"unit":"percent","family":"vals-ai:tax_agent_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench","measuredAt":"2026-11-09","stderr":3.585,"costPerTest":14.39883,"latency":4012.548},{"benchmarkId":"bug-hunt-bench","benchmarkName":"Bug Hunt Bench","category":"coding","score":41.7,"unit":"percent","family":"bug-hunt-bench","index":77.58912390310242,"aboveFit":false,"run":null,"source":"Pawel Huryn","sourceUrl":"https://bughunt.productcompass.pm/method","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"automationBench","benchmarkName":"AutomationBench","category":"agentic","score":40,"unit":"percent","family":"automationBench","index":58.174184644643795,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"roboDojoSim","benchmarkName":"RoboDojo Sim","category":"agentic","score":39.4,"unit":"percent","family":"roboDojoSim","index":null,"aboveFit":false,"run":null,"source":"Anthropic (reported run)","sourceUrl":"https://www.anthropic.com/claude-fable-and-mythos-5-2","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaItbench","benchmarkName":"Artificial Analysis ITBench-AA","category":"agentic","score":38.2,"unit":"percent","family":"aaItbench","index":55.65822961558741,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/itbench-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"morphologyToMolecule","benchmarkName":"Axiom Bio morphology-to-molecule matching","category":"knowledge","score":34,"unit":"percent","family":"morphologyToMolecule","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:srebench:overall","benchmarkName":"SRE Bench","category":null,"domain":"Other","eligible":false,"ranked":true,"score":33.588,"unit":"percent","family":"vals-ai:srebench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/srebench","measuredAt":"2026-11-05","stderr":2.923,"costPerTest":26.477879,"latency":5795.029},{"benchmarkId":"exploitGym","benchmarkName":"ExploitGym","category":"agentic","score":33.3,"unit":"percent","family":"exploitGym","index":79.19276557114365,"aboveFit":false,"run":null,"source":"Zhun Wang, Nico Schiller, Hongwei Li, Srijiith Sesha Narayana, Milad Nasr, Nicholas Carlini, Xiangyu Qi, Eric Wallace, Elie Bursztein, Luca Invernizzi, Kurt Thomas, Yan Shoshitaishvili, Wenbo Guo, Jingxuan He, Thorsten Holz, Dawn Song","sourceUrl":"https://arxiv.org/abs/2605.11086","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":31.7,"unit":"percent","family":"critpt","index":95,"aboveFit":true,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:vcb-1-100:overall","benchmarkName":"Vibe Code Bench 1-100","category":"coding","domain":null,"eligible":true,"ranked":true,"score":30.361,"unit":"percent","family":"vals-ai:vcb-1-100:overall","index":77.90925448791388,"aboveFit":false,"run":"OpenHands","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vcb-1-100","measuredAt":"2026-10-29","stderr":4.829,"costPerTest":44.115239,"latency":8909.323},{"benchmarkId":"aaCyberIndex","benchmarkName":"Artificial Analysis Cyber Index","category":"agentic","score":28.7,"unit":"percent","family":"aaCyberIndex","index":56.70849240745534,"aboveFit":false,"run":null,"source":"Artificial Analysis with the Cyber Index Alliance (Collinear AI, IBM, NVIDIA, Pasta Lab, Vercel)","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-cyber-index","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaMlcr","benchmarkName":"Medical Long Context Reasoning (MLCR-AA)","category":"reasoning","score":27.8,"unit":"percent","family":"aaMlcr","index":72.33295709720197,"aboveFit":false,"run":null,"source":"Wisedocs and Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mlcr-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"toolathlonVerifiedAvgTurns","benchmarkName":"Toolathlon Verified average assistant turns","category":"agentic","score":26.9,"unit":"percent","family":"toolathlonVerifiedAvgTurns","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaDeepsecBench","benchmarkName":"Artificial Analysis DeepsecBench-AA","category":"agentic","score":26.9,"unit":"percent","family":"aaDeepsecBench","index":70.97676478086578,"aboveFit":false,"run":null,"source":"Artificial Analysis; dataset by Vercel","sourceUrl":"https://artificialanalysis.ai/evaluations/deepsecbench-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaGdpPdf","benchmarkName":"Artificial Analysis GDP.pdf","category":"agentic","score":26.2,"unit":"percent","family":"aaGdpPdf","index":73.47438024404812,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdp-pdf","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"rliAutomationRate","benchmarkName":"Remote Labor Index (RLI)","category":"agentic","score":21.3,"unit":"percent","family":"rliAutomationRate","index":null,"aboveFit":false,"run":null,"source":"Center for AI Safety; Scale AI","sourceUrl":"https://scale.com/leaderboard/rli","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lmarena:agent-task-outcome:overall","benchmarkName":"Agent Arena task outcome","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":20.673956736290666,"unit":"index","family":"agent-arena","index":83.52803483151438,"aboveFit":false,"run":"max effort","source":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset","measuredAt":"2026-11-11","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lmarena:agent-steerability:overall","benchmarkName":"Agent Arena steerability","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":18.58149995279254,"unit":"index","family":"agent-arena","index":81.8077073182508,"aboveFit":false,"run":"max effort","source":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset","measuredAt":"2026-11-11","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:programbench:overall","benchmarkName":"ProgramBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":18.5,"unit":"percent","family":"vals-ai:programbench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/programbench","measuredAt":"2026-11-09","stderr":2.753,"costPerTest":42.65866,"latency":8660.211},{"benchmarkId":"vals-ai:cua_bench:overall","benchmarkName":"CUA-bench","category":null,"domain":"Other","eligible":false,"ranked":true,"score":14,"unit":"percent","family":"vals-ai:cua_bench:overall","index":null,"aboveFit":false,"run":"claude-code","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/cua_bench","measuredAt":"2026-10-13","stderr":0,"costPerTest":188.040183,"latency":14.71},{"benchmarkId":"lmarena:agent-bash-recovery:overall","benchmarkName":"Agent Arena command recovery","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":13.55335853279116,"unit":"index","family":"agent-arena","index":77.67378680260337,"aboveFit":false,"run":"high effort","source":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset","measuredAt":"2026-10-08","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mazeBench","benchmarkName":"MazeBench: visual spatial reasoning in a 3D open world","category":"agentic","score":13,"unit":"percent","family":"mazeBench","index":85.63845875136961,"aboveFit":false,"run":null,"source":"Jonathan Pappas and David Pappas","sourceUrl":"https://mazebench.com/blog?post=maze-bench-results","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:hlab:overall","benchmarkName":"HLAB-Vals-1.1","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":9.167,"unit":"percent","family":"vals-ai:hlab:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/hlab","measuredAt":"2026-11-09","stderr":2.366,"costPerTest":21.812248,"latency":3507.428},{"benchmarkId":"legalAgentBenchHeldoutAllPass","benchmarkName":"Legal Agent Benchmark all-pass rate — Harvey held-out set","category":"agentic","score":8.3,"unit":"percent","family":"legalAgentBenchHeldoutAllPass","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-spireclimb-slay-the-spire-2-ascension-10:overall","benchmarkName":"SpireClimb (Slay the Spire 2, Ascension 10)","category":null,"domain":"Other","eligible":false,"ranked":true,"score":6.666666666666667,"unit":"percent","family":"epoch-ai:epoch-spireclimb-slay-the-spire-2-ascension-10:overall","index":null,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.8997255746722663,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-spireclimb:overall","benchmarkName":"SpireClimb","category":null,"domain":"Other","eligible":false,"ranked":true,"score":5.833333333333333,"unit":"percent","family":"epoch-ai:epoch-spireclimb:overall","index":null,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.0285004230980284,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-spireclimb-slay-the-spire-1-ascension-20-heart:overall","benchmarkName":"SpireClimb (Slay the Spire 1, Ascension 20 Heart)","category":null,"domain":"Other","eligible":false,"ranked":true,"score":5,"unit":"percent","family":"epoch-ai:epoch-spireclimb-slay-the-spire-1-ascension-20-heart:overall","index":null,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.8374015326751194,"costPerTest":null,"latency":null},{"benchmarkId":"aaHarveyLabV11","benchmarkName":"Harvey LAB-AA v1.1","category":"agentic","score":4.2,"unit":"percent","family":"aaHarveyLabV11","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-erdos:overall","benchmarkName":"FrontierMath-Erdos","category":"math","domain":null,"eligible":true,"ranked":true,"score":2.94,"unit":"percent","family":"epoch-ai:epoch-frontiermath-erdos:overall","index":null,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.06,"costPerTest":null,"latency":null},{"benchmarkId":"stemBench120","benchmarkName":"STEMBench120: six-domain scientific research progress","category":"reasoning","score":2.621,"unit":"percent","family":"stemBench120","index":null,"aboveFit":false,"run":null,"source":"Ulam","sourceUrl":"https://www.ulam.ai/stembench","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaCyberGymE2e","benchmarkName":"Artificial Analysis CyberGym-E2E-AA","category":"agentic","score":0.8,"unit":"percent","family":"aaCyberGymE2e","index":5,"aboveFit":false,"run":null,"source":"Artificial Analysis; dataset by Berkeley RDI","sourceUrl":"https://artificialanalysis.ai/evaluations/cybergym-e2e-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":4,"outputPerMillion":20,"batch":{"inputPerMillion":2,"outputPerMillion":10},"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":76,"timeToFirstToken":3.416,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":{"elo":1513.0021399366474,"eloUpper":1519.0422401810033,"eloLower":1506.9620396922915,"votes":15249,"rank":3,"rankUpper":null,"rankLower":null,"arenaName":"Claude Opus 5.5-high","organization":"anthropic","updatedAt":"2026-11-09"},"quality":{"coveragePercent":95,"benchmarkCount":73,"categoryCount":7,"confidence":"high","state":"available"},"hosts":[{"host":"amazon-bedrock","name":"Amazon Bedrock","kind":"cloud","url":"https://docs.aws.amazon.com/bedrock/latest/userguide/models-supported.html","inputPerMillion":4,"outputPerMillion":20,"cacheReadPerMillion":0.2,"contextTokens":1000000,"maxOutputTokens":128000,"deprecated":false},{"host":"anthropic","name":"Anthropic","kind":"maker","url":"https://docs.anthropic.com/en/docs/about-claude/models","inputPerMillion":4,"outputPerMillion":20,"cacheReadPerMillion":0.2,"contextTokens":1000000,"maxOutputTokens":128000,"deprecated":false},{"host":"digitalocean","name":"DigitalOcean","kind":"cloud","url":"https://docs.digitalocean.com/products/gradient-ai-platform/details/models/","inputPerMillion":4,"outputPerMillion":20,"cacheReadPerMillion":0.2,"contextTokens":1000000,"maxOutputTokens":128000,"deprecated":false},{"host":"google-vertex","name":"Google Vertex AI","kind":"cloud","url":"https://cloud.google.com/vertex-ai/generative-ai/docs/models","inputPerMillion":4,"outputPerMillion":20,"cacheReadPerMillion":0.2,"contextTokens":1000000,"maxOutputTokens":128000,"deprecated":false},{"host":"azure","name":"Microsoft Azure","kind":"cloud","url":"https://learn.microsoft.com/en-us/azure/ai-services/openai/concepts/models","inputPerMillion":4,"outputPerMillion":20,"cacheReadPerMillion":0.2,"contextTokens":1000000,"maxOutputTokens":128000,"deprecated":false},{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":4,"outputPerMillion":20,"cacheReadPerMillion":0.2,"contextTokens":1000000,"maxOutputTokens":128000,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:arc-prize-foundation","name":"ARC Prize Foundation","url":"https://arcprize.org","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:livebench","name":"LiveBench","url":"https://livebench.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:terminal-bench","name":"Terminal-Bench","url":"https://www.tbench.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:lmarena","name":"LMArena","url":"https://huggingface.co","license":"CC BY 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gemini-4-argon","providerId":"google","slug":"gemini-4-argon","name":"Gemini 4 Argon","provider":"Google","description":"Gemini 4 Argon is a reasoning model from Google in the Gemini 4 family. 48 benchmarks count toward its index, in 5 categories.","logoUrl":null,"releaseDate":"2026-09-30","contextWindowTokens":1048576,"modalities":["audio","file","image","text","video"],"openWeights":false,"details":{"apiId":"google/gemini-4-argon","maxOutputTokens":943718,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.09999999999999999,"weightsUrl":null},"reasoning":true,"overallScore":78.52223305306748,"overallScoreRange":{"low":74.18829458986842,"high":82.85617151626654},"badges":[],"capabilityScores":{"agentic":77.34751305022718,"coding":76.69782007568932,"reasoning":79.76306094389093,"multimodal":null,"knowledge":76.5236680666477,"multilingual":null,"instruction":null,"math":75.93164684355528},"capabilityCounts":{"coding":13,"math":4,"reasoning":10,"knowledge":4,"agentic":17},"benchmarks":[{"benchmarkId":"vals-ai:ioi:overall","benchmarkName":"IOI","category":"coding","domain":null,"eligible":true,"ranked":true,"score":100,"unit":"percent","family":"vals-ai:ioi:overall","index":83.65887425316964,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/ioi","measuredAt":"2026-10-29","stderr":0,"costPerTest":5.307626,"latency":1459.27},{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":100,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":67.09849544987334,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0,"costPerTest":null,"latency":null},{"benchmarkId":"graphwalksBfs128k","benchmarkName":"Graphwalks BFS 0K-128K","category":"reasoning","score":99.7,"unit":"percent","family":"graphwalksBfs128k","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"arc-prize:v1-public-eval:overall","benchmarkName":"ARC-AGI-1 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":99.5,"unit":"percent","family":"arc-prize:v1-public-eval:overall","index":null,"aboveFit":false,"run":"high effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.12766658000000017,"latency":null},{"benchmarkId":"vals-ai:proof_bench:overall","benchmarkName":"ProofBench v1.1","category":"math","domain":null,"eligible":true,"ranked":true,"score":99,"unit":"percent","family":"vals-ai:proof_bench:overall","index":81.33584598884255,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/proof_bench","measuredAt":"2026-11-09","stderr":1,"costPerTest":0.782978,"latency":380.246},{"benchmarkId":"arc-prize:v1-semi-private:overall","benchmarkName":"ARC-AGI-1 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":98.5,"unit":"percent","family":"arc-prize:v1-semi-private:overall","index":72.27136500316884,"aboveFit":false,"run":"high effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.16167408000000005,"latency":null},{"benchmarkId":"arc-prize:v2-public-eval:overall","benchmarkName":"ARC-AGI-2 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":97.08333333333333,"unit":"percent","family":"arc-prize:v2-public-eval:overall","index":null,"aboveFit":false,"run":"high effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.3556000333333334,"latency":null},{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":96.40151515151516,"unit":"percent","family":"gpqa-diamond","index":65.62593625568488,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.1918129589602695,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-reasoning:overall","benchmarkName":"LiveBench Reasoning","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":95.1,"unit":"percent","family":"livebench","index":84.81549436895693,"aboveFit":false,"run":"high effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Reasoning","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.07806071287128713,"latency":null},{"benchmarkId":"livebench:livebench-browser:overall","benchmarkName":"LiveBench Browser Use","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":92,"unit":"percent","family":"livebench-2-agentic","index":85.87193231505225,"aboveFit":false,"run":"high effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Browser","measuredAt":"2026-10-14","stderr":null,"costPerTest":3.433868,"latency":null},{"benchmarkId":"vals-ai:vibe-code:overall","benchmarkName":"Vibe Code Bench v1.1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":91.906,"unit":"percent","family":"vals-ai:vibe-code:overall","index":75.41065134251707,"aboveFit":false,"run":"OpenHands","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","measuredAt":"2026-11-09","stderr":1.899,"costPerTest":8.10797,"latency":2247.675},{"benchmarkId":"lvBench","benchmarkName":"LVBench","category":"multimodal","score":91.7,"unit":"percent","family":"lvBench","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"arc-prize:v2-semi-private:overall","benchmarkName":"ARC-AGI-2 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":91.25,"unit":"percent","family":"arc-prize:v2-semi-private:overall","index":81.08455686631679,"aboveFit":false,"run":"high effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.3248761333333333,"latency":null},{"benchmarkId":"livebench:livebench-coding-2:overall","benchmarkName":"LiveBench Coding 2.0","category":"coding","domain":null,"eligible":true,"ranked":true,"score":89.7,"unit":"percent","family":"livebench:livebench-coding-2:overall","index":73.14387819634575,"aboveFit":false,"run":"high effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Coding","measuredAt":"2026-10-14","stderr":null,"costPerTest":5.076327586206897,"latency":null},{"benchmarkId":"vals-ai:sage:overall","benchmarkName":"SAGE","category":null,"domain":"Education","eligible":false,"ranked":true,"score":89.46,"unit":"percent","family":"vals-ai:sage:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/sage","measuredAt":"2026-11-05","stderr":1.642,"costPerTest":0.074854,"latency":29.17},{"benchmarkId":"labBench2","benchmarkName":"LABBench2: An Improved Benchmark for AI Systems Performing Biology Research","category":"knowledge","score":88.8,"unit":"percent","family":"labBench2","index":null,"aboveFit":false,"run":null,"source":"Jon M. Laurent, Albert Bou, Michael Pieler, Conor Igoe, Alex Andonian, Siddharth Narayanan, James Braza, Alexandros Sanchez Vassopoulos, Jacob L. Steenwyk, Blake Lash, Andrew D. White, Samuel G. Rodriques","sourceUrl":"https://arxiv.org/abs/2604.09554","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:legal_bench:overall","benchmarkName":"LegalBench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":88.299,"unit":"percent","family":"vals-ai:legal_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","measuredAt":"2026-10-13","stderr":0.366,"costPerTest":0.016117,"latency":12.623},{"benchmarkId":"vals-ai:medscribe:overall","benchmarkName":"MedScribe","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":87.427,"unit":"percent","family":"vals-ai:medscribe:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medscribe","measuredAt":"2026-10-29","stderr":1.964,"costPerTest":0.147175,"latency":87.439},{"benchmarkId":"epoch-ai:epoch-frontiermath-tiers-1-3-v2-private:overall","benchmarkName":"FrontierMath-Tiers-1-3-v2-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":85.96491228070175,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tiers-1-3-v2-private:overall","index":76.14247499591445,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.0611471958164294,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-data-analysis:overall","benchmarkName":"LiveBench Data Analysis","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":85.2,"unit":"percent","family":"livebench","index":71.99069780710602,"aboveFit":false,"run":"high effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Data%20Analysis","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.28433922666666667,"latency":null},{"benchmarkId":"graphwalksBfs256k1m","benchmarkName":"GraphWalks BFS 256K–1M","category":"reasoning","score":84.2,"unit":"percent","family":"graphwalksBfs256k1m","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-research:overall","benchmarkName":"LiveBench Research","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":83.1,"unit":"percent","family":"livebench-2-agentic","index":79.36308557026018,"aboveFit":false,"run":"high effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Research","measuredAt":"2026-10-14","stderr":null,"costPerTest":5.9555560000000005,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":79.7,"unit":"percent","family":"lcr","index":61.97866734834783,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tier-4-v2-private:overall","benchmarkName":"FrontierMath-Tier-4-v2-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":78.04878048780488,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tier-4-v2-private:overall","index":79.14977093959078,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":6.5445892024384085,"costPerTest":null,"latency":null},{"benchmarkId":"deepSwe","benchmarkName":"DeepSWE","category":"agentic","score":77.9,"unit":"percent","family":"deepSwe","index":77.70784597599524,"aboveFit":false,"run":null,"source":"Datacurve AI","sourceUrl":"https://deepswe.datacurve.ai/blog","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:cyber:overall","benchmarkName":"CyberBench v1.1","category":null,"domain":"Other","eligible":false,"ranked":true,"score":77.857,"unit":"percent","family":"vals-ai:cyber:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/cyber","measuredAt":"2026-11-05","stderr":5.296,"costPerTest":5.113798,"latency":1150.368},{"benchmarkId":"aaAutomationBench","benchmarkName":"Artificial Analysis AutomationBench","category":"agentic","score":77.5,"unit":"percent","family":"aaAutomationBench","index":86.34372496078068,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/automationbench-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:biomysterybench:overall","benchmarkName":"BioMysteryBench","category":null,"domain":"Science","eligible":false,"ranked":true,"score":76.296,"unit":"percent","family":"vals-ai:biomysterybench:overall","index":null,"aboveFit":false,"run":"Terminus 2","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/biomysterybench","measuredAt":"2026-11-09","stderr":0.37,"costPerTest":5.707134,"latency":2396.816},{"benchmarkId":"epoch-ai:epoch-simpleqa-verified:overall","benchmarkName":"SimpleQA Verified","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":76,"unit":"percent","family":"epoch-ai:epoch-simpleqa-verified:overall","index":87.23376145969101,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.3512312258920844,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:emb:overall","benchmarkName":"EMB","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":75.244,"unit":"percent","family":"vals-ai:emb:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/emb","measuredAt":"2026-10-20","stderr":2.459,"costPerTest":11.8029,"latency":2305.821},{"benchmarkId":"scarfBench","benchmarkName":"ScarfBench: Cross-Framework Application Migration in Enterprise Java","category":"coding","score":75.2,"unit":"percent","family":"scarfBench","index":78.25804967215696,"aboveFit":false,"run":null,"source":"Advait Pavuluri, Bridget McGinn, Ashita Saxena, George Safta, Srikanth Tamilselvam, Raju Pavuluri, Michele Merler, Baishakhi Ray, Rahul Krishna (IBM Research)","sourceUrl":"https://arxiv.org/abs/2605.06754","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"osWorld21Offline","benchmarkName":"OSWorld 2.1 offline subset (82 tasks)","category":"agentic","score":74.7,"unit":"percent","family":"osWorld21Offline","index":76.88629867161958,"aboveFit":false,"run":null,"source":"Mengqi Yuan, Zilong Zhou, Xinzhuang Xiong, Weiming Wu, Jiayang Sun, Jiamin Song, Kaiqian Cui, Bowen Wang, Haoyuan Wu, Yitong Li, Dunjie Lu, Haikong Lu, Qi Zhen, Xinyuan Wang, Jiaqi Deng, Yuhao Yang, Cheng Chen, Boyuan Zheng, Alex Su, Xiao Yu, Hao Zou, Saaket Agashe, Xing Han Lu, Manpreet Kaur, Zhengyang Qi, Vincent Sunn Chen, Frederic Sala, Dayiheng Liu, Junyang Lin, Zhou Yu, Yu Su, Siva Reddy, Xin Eric Wang, Peng Qi, Tianbao Xie, Tao Yu","sourceUrl":"https://github.com/xlang-ai/OSWorld-V2/releases","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-design:overall","benchmarkName":"LiveBench Design","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":73.7,"unit":"percent","family":"livebench-2-agentic","index":72.48857327800789,"aboveFit":false,"run":"high effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Design","measuredAt":"2026-10-14","stderr":null,"costPerTest":1.16044,"latency":null},{"benchmarkId":"epoch-ai:epoch-furniture-assembly:overall","benchmarkName":"Furniture Assembly","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":72.5,"unit":"percent","family":"epoch-ai:epoch-furniture-assembly:overall","index":80.46131001144212,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":5.6279417919483965,"costPerTest":null,"latency":null},{"benchmarkId":"chartography","benchmarkName":"Chartography without tools","category":"multimodal","score":71.6,"unit":"percent","family":"chartography","index":null,"aboveFit":false,"run":null,"source":"Surge AI and Anthropic","sourceUrl":"https://surgehq.ai/blog/chartography","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-long-horizon:overall","benchmarkName":"LiveBench Long Agentic","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":69.9,"unit":"percent","family":"livebench-2-agentic","index":69.70951511731015,"aboveFit":false,"run":"high effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Long%20Agentic","measuredAt":"2026-10-14","stderr":null,"costPerTest":13.841872222222221,"latency":null},{"benchmarkId":"vals-ai:public-benefits-bench:overall","benchmarkName":"Public Benefits Bench v1.1","category":null,"domain":"Public benefits","eligible":false,"ranked":true,"score":69.756,"unit":"percent","family":"vals-ai:public-benefits-bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/public-benefits-bench","measuredAt":"2026-10-12","stderr":0,"costPerTest":2.793902,"latency":1566.785},{"benchmarkId":"osWorld2","benchmarkName":"OSWorld 2.0","category":"agentic","score":69.2,"unit":"percent","family":"osWorld2","index":82.05089133481594,"aboveFit":false,"run":null,"source":"Mengqi Yuan, Zilong Zhou, Xinzhuang Xiong, Weiming Wu, Jiayang Sun, Jiamin Song, Kaiqian Cui, Bowen Wang, Haoyuan Wu, Yitong Li, Dunjie Lu, Haikong Lu, Qi Zhen, Xinyuan Wang, Jiaqi Deng, Yuhao Yang, Cheng Chen, Boyuan Zheng, Alex Su, Xiao Yu, Hao Zou, Saaket Agashe, Xing Han Lu, Manpreet Kaur, Zhengyang Qi, Vincent Sunn Chen, Frederic Sala, Dayiheng Liu, Junyang Lin, Zhou Yu, Yu Su, Siva Reddy, Xin Eric Wang, Peng Qi, Tianbao Xie, Tao Yu","sourceUrl":"https://arxiv.org/abs/2606.29537","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:code-migration:overall","benchmarkName":"Code Migration","category":"coding","domain":null,"eligible":true,"ranked":true,"score":68.173,"unit":"percent","family":"vals-ai:code-migration:overall","index":81.40536835254096,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/code-migration","measuredAt":"2026-10-29","stderr":4.349,"costPerTest":57.822126,"latency":7891.461},{"benchmarkId":"cweBenchV1","benchmarkName":"CWE-bench v1","category":"agentic","score":68,"unit":"percent","family":"cweBenchV1","index":79.47014074802985,"aboveFit":false,"run":null,"source":"Collinear AI","sourceUrl":"https://cwe-bench.com/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:fabv2:overall","benchmarkName":"Finance Agent (v2)","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":65.401,"unit":"percent","family":"vals-ai:fabv2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/fabv2","measuredAt":"2026-10-29","stderr":0.321,"costPerTest":4.378161,"latency":980.524},{"benchmarkId":"cweBenchV2","benchmarkName":"CWE-bench v2","category":"agentic","score":65,"unit":"percent","family":"cweBenchV2","index":78.23441465005149,"aboveFit":false,"run":null,"source":"Collinear AI","sourceUrl":"https://cwe-bench.com/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-ebr-bench:overall","benchmarkName":"EBR-bench","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":64.76190476190476,"unit":"percent","family":"epoch-ai:epoch-ebr-bench:overall","index":82.14779021448203,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":4.902681019517619,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-mystery-game-puzzles:overall","benchmarkName":"Mystery Game Puzzles","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":64,"unit":"percent","family":"epoch-ai:epoch-mystery-game-puzzles:overall","index":88.24355159120118,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":4.824181513244217,"costPerTest":null,"latency":null},{"benchmarkId":"aaSciCode","benchmarkName":"Artificial Analysis SciCode","category":"coding","score":61.8,"unit":"percent","family":"scicode-family","index":76.40791129094177,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/scicode","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-chess-puzzles:overall","benchmarkName":"Chess Puzzles","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":59,"unit":"percent","family":"epoch-ai:epoch-chess-puzzles:overall","index":78.76657223899463,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":4.943110704237103,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-mirrorcode:overall","benchmarkName":"MirrorCode","category":"coding","domain":null,"eligible":true,"ranked":true,"score":58.88888888888889,"unit":"percent","family":"epoch-ai:epoch-mirrorcode:overall","index":75.42794732917439,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":11.71381252539371,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:medcode:overall","benchmarkName":"MedCode","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":58.801,"unit":"percent","family":"vals-ai:medcode:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medcode","measuredAt":"2026-10-29","stderr":2.1,"costPerTest":0.13716,"latency":79.527},{"benchmarkId":"vals-ai:terminal-bench-2-1:overall","benchmarkName":"Terminal-Bench 2.1","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":57.678,"unit":"percent","family":"terminal-bench-2-1","index":54.87440891942742,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2-1","measuredAt":"2026-10-01","stderr":2.456,"costPerTest":null,"latency":841.802},{"benchmarkId":"terminalBenchScience6xVerifier","benchmarkName":"Terminal-Bench-Science 0.1 with sixfold verifier timeout","category":"agentic","score":57.6,"unit":"percent","family":"terminalBenchScience6xVerifier","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:terminal-bench-4:overall","benchmarkName":"Terminal-Bench 4.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":57.576,"unit":"percent","family":"terminal-bench-4","index":81.85532250842776,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-4","measuredAt":"2026-11-09","stderr":2.314,"costPerTest":17.63994,"latency":3214.793},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":57.1,"unit":"percent","family":"hle","index":81.67466005451473,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gdpvalAaNormalized","benchmarkName":"GDPval-AA normalized","category":"agentic","score":56.2,"unit":"percent","family":"gdpvalAaNormalized","index":74.21546892427375,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"frontierSweV2","benchmarkName":"FrontierSWE v2","category":"coding","score":55,"unit":"percent","family":"frontierSweV2","index":79.73672267308294,"aboveFit":false,"run":null,"source":"Proximal","sourceUrl":"https://www.frontierswe.com/blog/v2","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:legal_research:overall","benchmarkName":"Legal Research Bench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":54.808,"unit":"percent","family":"vals-ai:legal_research:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_research","measuredAt":"2026-11-09","stderr":3.459,"costPerTest":6.44908,"latency":1881.045},{"benchmarkId":"blueprintBench2","benchmarkName":"Blueprint-Bench 2","category":"multimodal","score":54.4,"unit":"percent","family":"blueprintBench2","index":null,"aboveFit":false,"run":null,"source":"Google DeepMind","sourceUrl":"https://x.com/GoogleDeepMind","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"cursorBench40","benchmarkName":"cursorBench40","category":"coding","score":52.9,"unit":"percent","family":"cursorBench40","index":77.35306614858726,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:time_horizon_index:overall","benchmarkName":"Time Horizon Index: KSP","category":null,"domain":"Other","eligible":false,"ranked":true,"score":52.5,"unit":"percent","family":"vals-ai:time_horizon_index:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/time_horizon_index","measuredAt":"2026-10-07","stderr":8.856,"costPerTest":708.627391,"latency":0},{"benchmarkId":"automationBench","benchmarkName":"AutomationBench","category":"agentic","score":51.3,"unit":"percent","family":"automationBench","index":79.66773355563492,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mka","benchmarkName":"Multimodal Knowledge Agent Benchmark","category":"agentic","score":51.3,"unit":"percent","family":"mka","index":79.05053038491528,"aboveFit":false,"run":null,"source":"Google DeepMind, in partnership with Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mka","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:tax_agent_bench:overall","benchmarkName":"Tax Agent Bench","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":49.223,"unit":"percent","family":"vals-ai:tax_agent_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench","measuredAt":"2026-11-09","stderr":3.608,"costPerTest":3.473645,"latency":985.634},{"benchmarkId":"aaAnalystAgent","benchmarkName":"Artificial Analysis AnalystAgent","category":"agentic","score":48.8,"unit":"percent","family":"aaAnalystAgent","index":70.65496437702963,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/aa-analyst-agent","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"omniscienceAccuracy","benchmarkName":"Artificial Analysis Omniscience Accuracy","category":"knowledge","score":48.3,"unit":"percent","family":"omniscienceAccuracy","index":71.56031449670019,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"arc-prize:v3-semi-private:overall","benchmarkName":"ARC-AGI-3 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":46.265668402958575,"unit":"percent","family":"arc-prize:v3-semi-private:overall","index":84.96214618161177,"aboveFit":false,"run":"medium effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:mysterymechanism:overall","benchmarkName":"MysteryMechanism","category":null,"domain":"Science","eligible":false,"ranked":true,"score":45.495,"unit":"percent","family":"vals-ai:mysterymechanism:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mysterymechanism","measuredAt":"2026-11-11","stderr":3.35,"costPerTest":4.594426,"latency":1494.947},{"benchmarkId":"postTrainBench11","benchmarkName":"PostTrainBench v1.1","category":"coding","score":45.3,"unit":"percent","family":"postTrainBench11","index":78.39509549308292,"aboveFit":false,"run":null,"source":"PostTrainBench team","sourceUrl":"https://posttrainbench.com/?version=v1.1","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:terminal-bench-science:overall","benchmarkName":"Terminal-Bench-Science","category":null,"domain":"Science","eligible":false,"ranked":true,"score":44.286,"unit":"percent","family":"vals-ai:terminal-bench-science:overall","index":null,"aboveFit":false,"run":"Mini-SWE-agent","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-science","measuredAt":"2026-11-09","stderr":5.98,"costPerTest":44.904641,"latency":6369.095},{"benchmarkId":"vals-ai:srebench:overall","benchmarkName":"SRE Bench","category":null,"domain":"Other","eligible":false,"ranked":true,"score":44.275,"unit":"percent","family":"vals-ai:srebench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/srebench","measuredAt":"2026-11-05","stderr":3.075,"costPerTest":30.451343,"latency":5695.712},{"benchmarkId":"bug-hunt-bench","benchmarkName":"Bug Hunt Bench","category":"coding","score":40.7,"unit":"percent","family":"bug-hunt-bench","index":76.94451335195427,"aboveFit":false,"run":null,"source":"Pawel Huryn","sourceUrl":"https://bughunt.productcompass.pm/method","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"agentsLastExam","benchmarkName":"Agents' Last Exam","category":"agentic","score":39.5,"unit":"percent","family":"agentsLastExam","index":73.84995907659419,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"exploitGym","benchmarkName":"ExploitGym","category":"agentic","score":39.5,"unit":"percent","family":"exploitGym","index":82.9550315102708,"aboveFit":false,"run":null,"source":"Zhun Wang, Nico Schiller, Hongwei Li, Srijiith Sesha Narayana, Milad Nasr, Nicholas Carlini, Xiangyu Qi, Eric Wallace, Elie Bursztein, Luca Invernizzi, Kurt Thomas, Yan Shoshitaishvili, Wenbo Guo, Jingxuan He, Thorsten Holz, Dawn Song","sourceUrl":"https://arxiv.org/abs/2605.11086","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"roboDojoSim","benchmarkName":"RoboDojo Sim","category":"agentic","score":34.2,"unit":"percent","family":"roboDojoSim","index":null,"aboveFit":false,"run":null,"source":"Anthropic (reported run)","sourceUrl":"https://www.anthropic.com/claude-fable-and-mythos-5-2","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"extractBenchPro","benchmarkName":"ExtractBench Pro: long-tail document extraction challenges (LlamaIndex)","category":"multimodal","score":30,"unit":"percent","family":"extractBenchPro","index":null,"aboveFit":false,"run":null,"source":"LlamaIndex","sourceUrl":"https://www.extractbench.ai/pro","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":27.1,"unit":"percent","family":"critpt","index":89.31155389531276,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:hlab:overall","benchmarkName":"HLAB-Vals-1.1","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":27.083,"unit":"percent","family":"vals-ai:hlab:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/hlab","measuredAt":"2026-11-09","stderr":3.738,"costPerTest":10.54836,"latency":1755.397},{"benchmarkId":"frontierCode2","benchmarkName":"FrontierCode 2.0","category":"coding","score":26.3,"unit":"percent","family":"frontierCode2","index":71.52537667023024,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"morphologyToMolecule","benchmarkName":"Axiom Bio morphology-to-molecule matching","category":"knowledge","score":25.2,"unit":"percent","family":"morphologyToMolecule","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:vcb-1-100:overall","benchmarkName":"Vibe Code Bench 1-100","category":"coding","domain":null,"eligible":true,"ranked":true,"score":21.2,"unit":"percent","family":"vals-ai:vcb-1-100:overall","index":69.40420621017687,"aboveFit":false,"run":"OpenHands","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vcb-1-100","measuredAt":"2026-10-29","stderr":3.811,"costPerTest":18.659304,"latency":6179.48},{"benchmarkId":"lmarena:agent-task-outcome:overall","benchmarkName":"Agent Arena task outcome","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":13.247770670622783,"unit":"index","family":"agent-arena","index":77.42254567248608,"aboveFit":false,"run":"high effort","source":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset","measuredAt":"2026-11-11","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lmarena:agent-steerability:overall","benchmarkName":"Agent Arena steerability","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":13.155496738738398,"unit":"index","family":"agent-arena","index":77.3466820345703,"aboveFit":false,"run":"high effort","source":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset","measuredAt":"2026-11-11","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mazeBench","benchmarkName":"MazeBench: visual spatial reasoning in a 3D open world","category":"agentic","score":12,"unit":"percent","family":"mazeBench","index":83.818474797291,"aboveFit":false,"run":null,"source":"Jonathan Pappas and David Pappas","sourceUrl":"https://mazebench.com/blog?post=maze-bench-results","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:cua_bench:overall","benchmarkName":"CUA-bench","category":null,"domain":"Other","eligible":false,"ranked":true,"score":4.833333,"unit":"percent","family":"vals-ai:cua_bench:overall","index":null,"aboveFit":false,"run":"antigravity","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/cua_bench","measuredAt":"2026-10-13","stderr":0,"costPerTest":193.778921,"latency":70.51},{"benchmarkId":"vals-ai:programbench:overall","benchmarkName":"ProgramBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":2.5,"unit":"percent","family":"vals-ai:programbench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/programbench","measuredAt":"2026-11-09","stderr":1.107,"costPerTest":17.656424,"latency":4252.235},{"benchmarkId":"stemBench120","benchmarkName":"STEMBench120: six-domain scientific research progress","category":"reasoning","score":2.444,"unit":"percent","family":"stemBench120","index":null,"aboveFit":false,"run":null,"source":"Ulam","sourceUrl":"https://www.ulam.ai/stembench","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-erdos:overall","benchmarkName":"FrontierMath-Erdos","category":"math","domain":null,"eligible":true,"ranked":true,"score":1.4705882352941175,"unit":"percent","family":"epoch-ai:epoch-frontiermath-erdos:overall","index":null,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.4705882352941175,"costPerTest":null,"latency":null},{"benchmarkId":"lmarena:agent-bash-recovery:overall","benchmarkName":"Agent Arena command recovery","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":-21.40574116354572,"unit":"index","family":"agent-arena","index":48.93192626773971,"aboveFit":false,"run":"high effort","source":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset","measuredAt":"2026-10-08","stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":2,"outputPerMillion":10,"batch":{"inputPerMillion":1,"outputPerMillion":5},"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":28,"timeToFirstToken":1.584,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":{"elo":1530.160034009476,"eloUpper":1535.597933491806,"eloLower":1524.7221345271455,"votes":19787,"rank":2,"rankUpper":null,"rankLower":null,"arenaName":"Gemini 4 Argon-high","organization":"google","updatedAt":"2026-11-09"},"quality":{"coveragePercent":80,"benchmarkCount":48,"categoryCount":5,"confidence":"medium","state":"available"},"hosts":[{"host":"google-ai-studio","name":"Google AI Studio","kind":"maker","url":"https://ai.google.dev/gemini-api/docs/models","inputPerMillion":2,"outputPerMillion":10,"cacheReadPerMillion":0.1,"contextTokens":1048576,"maxOutputTokens":32768,"deprecated":false},{"host":"google-vertex","name":"Google Vertex AI","kind":"maker","url":"https://cloud.google.com/vertex-ai/generative-ai/docs/models","inputPerMillion":2,"outputPerMillion":10,"cacheReadPerMillion":0.1,"contextTokens":1048576,"maxOutputTokens":65536,"deprecated":false},{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":2,"outputPerMillion":10,"cacheReadPerMillion":0.1,"contextTokens":1048576,"maxOutputTokens":943718,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:arc-prize-foundation","name":"ARC Prize Foundation","url":"https://arcprize.org","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:livebench","name":"LiveBench","url":"https://livebench.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:lmarena","name":"LMArena","url":"https://huggingface.co","license":"CC BY 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gpt-6-astra","providerId":"openai","slug":"gpt-6-astra","name":"GPT-6 Astra","provider":"OpenAI","description":"GPT-6 Astra is a reasoning model from OpenAI in the GPT-6 family. 76 benchmarks count toward its index, in 8 categories.","logoUrl":null,"releaseDate":"2026-09-03","contextWindowTokens":1050000,"modalities":["file","image","text"],"openWeights":false,"details":{"apiId":"openai/gpt-6-astra","maxOutputTokens":128000,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":1,"weightsUrl":null},"reasoning":true,"overallScore":78.03632542729953,"overallScoreRange":{"low":76.18149921895603,"high":79.89115163564303},"badges":[],"capabilityScores":{"agentic":74.60551753552032,"coding":76.94958608425847,"reasoning":83.77726727823674,"multimodal":71.24723652957175,"knowledge":78.51194699541212,"multilingual":73.62352231206805,"instruction":79.51625325135363,"math":78.033182633621},"capabilityCounts":{"coding":14,"math":5,"reasoning":11,"knowledge":7,"multimodal":4,"agentic":33,"multilingual":1,"instruction":1},"benchmarks":[{"benchmarkId":"mrcrv2_256_512","benchmarkName":"OpenAI MRCR v2 8-needle 256K-512K","category":"reasoning","score":100,"unit":"percent","family":"mrcrv2_256_512","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:ioi:overall","benchmarkName":"IOI","category":"coding","domain":null,"eligible":true,"ranked":true,"score":100,"unit":"percent","family":"vals-ai:ioi:overall","index":83.65887425316964,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/ioi","measuredAt":"2026-10-29","stderr":0,"costPerTest":6.495051,"latency":1899.783},{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":100,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":67.09849544987334,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:proof_bench:overall","benchmarkName":"ProofBench v1.1","category":"math","domain":null,"eligible":true,"ranked":true,"score":99,"unit":"percent","family":"vals-ai:proof_bench:overall","index":81.33584598884255,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/proof_bench","measuredAt":"2026-11-09","stderr":1,"costPerTest":1.71238,"latency":307.867},{"benchmarkId":"arc-prize:v1-public-eval:overall","benchmarkName":"ARC-AGI-1 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":97.75,"unit":"percent","family":"arc-prize:v1-public-eval:overall","index":null,"aboveFit":false,"run":"max effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.3296810250000001,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tier-4-v2-private:overall","benchmarkName":"FrontierMath-Tier-4-v2-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":97.6,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tier-4-v2-private:overall","index":86.830468283397,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.4,"costPerTest":null,"latency":null},{"benchmarkId":"arc-prize:v1-semi-private:overall","benchmarkName":"ARC-AGI-1 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":97.5,"unit":"percent","family":"arc-prize:v1-semi-private:overall","index":71.81162890019444,"aboveFit":false,"run":"max effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.4328827000000001,"latency":null},{"benchmarkId":"arc-prize:v2-public-eval:overall","benchmarkName":"ARC-AGI-2 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":97.5,"unit":"percent","family":"arc-prize:v2-public-eval:overall","index":null,"aboveFit":false,"run":"max effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.96997525,"latency":null},{"benchmarkId":"livebench:livebench-math:overall","benchmarkName":"LiveBench Mathematics","category":"math","domain":null,"eligible":true,"ranked":true,"score":96.80624999999999,"unit":"percent","family":"livebench:livebench-math:overall","index":74.69556260847348,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Mathematics","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.2484477572559367,"latency":null},{"benchmarkId":"mrcrv2_512_1m","benchmarkName":"OpenAI MRCR v2 8-needle 512K-1M","category":"reasoning","score":96.3,"unit":"percent","family":"mrcrv2_512_1m","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":96.1,"unit":"percent","family":"gpqa-diamond","index":65.34939479764087,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gpqa","benchmarkName":"Graduate-Level Google-Proof Q&A","category":"knowledge","score":96,"unit":"percent","family":"gpqa-diamond","index":65.2576775301489,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gpqaDiamond","benchmarkName":"GPQA Diamond","category":"knowledge","score":96,"unit":"percent","family":"gpqa-diamond","index":65.2576775301489,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"benchCadVision2CodeWithTools","benchmarkName":"BenchCAD Vision2Code voxel IoU with tools","category":"multimodal","score":95.89999999999999,"unit":"percent","family":"benchCadVision2CodeWithTools","index":null,"aboveFit":false,"run":null,"source":"Zhang et al. and Anthropic","sourceUrl":"https://arxiv.org/abs/2605.10865","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":95.77020202020202,"unit":"percent","family":"gpqa-diamond","index":65.04691310232641,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.3696812097020312,"costPerTest":null,"latency":null},{"benchmarkId":"arc-prize:v2-semi-private:overall","benchmarkName":"ARC-AGI-2 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":95,"unit":"percent","family":"arc-prize:v2-semi-private:overall","index":82.80631199497736,"aboveFit":false,"run":"max effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":1.1194680833333335,"latency":null},{"benchmarkId":"livebench:livebench-coding-2:overall","benchmarkName":"LiveBench Coding 2.0","category":"coding","domain":null,"eligible":true,"ranked":true,"score":93.7,"unit":"percent","family":"livebench:livebench-coding-2:overall","index":76.0411785004019,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Coding","measuredAt":"2026-10-14","stderr":null,"costPerTest":3.3153689655172416,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tiers-1-3-v2-private:overall","benchmarkName":"FrontierMath-Tiers-1-3-v2-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":93.6842105263158,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tiers-1-3-v2-private:overall","index":80.2055408375186,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.4434038295343787,"costPerTest":null,"latency":null},{"benchmarkId":"screenSpotPro","benchmarkName":"ScreenSpot Pro","category":"multimodal","score":92.7,"unit":"percent","family":"screenSpotPro","index":71.58087199177595,"aboveFit":false,"run":null,"source":"Kaixin Li, Ziyang Meng, Hongzhan Lin, Ziyang Luo, Yuchen Tian, Jing Ma, Zhiyong Huang, Tat-Seng Chua","sourceUrl":"https://arxiv.org/abs/2504.07981","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-reasoning:overall","benchmarkName":"LiveBench Reasoning","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":92.65375,"unit":"percent","family":"livebench","index":81.64653895588341,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Reasoning","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.38984257425742574,"latency":null},{"benchmarkId":"extractBenchLlamaIndex","benchmarkName":"ExtractBench: schema-guided enterprise document extraction (LlamaIndex leaderboard)","category":"multimodal","score":91.91,"unit":"percent","family":"extractBenchLlamaIndex","index":74.0408557603913,"aboveFit":false,"run":null,"source":"Boyang Zhang, Adrian Lyjak, Eli Stewart, Zhaoqi Li and Simon Suo (LlamaIndex)","sourceUrl":"https://arxiv.org/abs/2607.29677","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"browseComp","benchmarkName":"BrowseComp","category":"agentic","score":91.5,"unit":"percent","family":"browseComp","index":72.1958062995354,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:time_horizon_index:overall","benchmarkName":"Time Horizon Index: KSP","category":null,"domain":"Other","eligible":false,"ranked":true,"score":90.5,"unit":"percent","family":"vals-ai:time_horizon_index:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/time_horizon_index","measuredAt":"2026-10-07","stderr":0,"costPerTest":4206.58511,"latency":0},{"benchmarkId":"vals-ai:vibe-code:overall","benchmarkName":"Vibe Code Bench v1.1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":89.594,"unit":"percent","family":"vals-ai:vibe-code:overall","index":74.57644541827277,"aboveFit":false,"run":"OpenHands","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","measuredAt":"2026-11-09","stderr":2.174,"costPerTest":38.512323,"latency":2344.127},{"benchmarkId":"livebench:livebench-language:overall","benchmarkName":"LiveBench Language","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":89.43066666666668,"unit":"percent","family":"livebench:livebench-language:overall","index":75.68779987368437,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Language","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.221862,"latency":null},{"benchmarkId":"aaTerminalBench21","benchmarkName":"Artificial Analysis Terminal-Bench v2.1","category":"agentic","score":88.4,"unit":"percent","family":"aaTerminalBench21","index":71.05522338919815,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/terminalbench-v2-1","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:medscribe:overall","benchmarkName":"MedScribe","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":87.908,"unit":"percent","family":"vals-ai:medscribe:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medscribe","measuredAt":"2026-10-29","stderr":1.938,"costPerTest":0.581991,"latency":153.728},{"benchmarkId":"vals-ai:terminal-bench-2-1:overall","benchmarkName":"Terminal-Bench 2.1","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":87.266,"unit":"percent","family":"terminal-bench-2-1","index":72.17910944345388,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2-1","measuredAt":"2026-10-01","stderr":0.375,"costPerTest":1.339493,"latency":274.44},{"benchmarkId":"aaMmmuPro","benchmarkName":"Artificial Analysis MMMU-Pro","category":"multimodal","score":86.9,"unit":"percent","family":"mmmu-pro","index":68.24554086669113,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmmu-pro","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-research:overall","benchmarkName":"LiveBench Research","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":85.2,"unit":"percent","family":"livebench-2-agentic","index":80.89888086959314,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Research","measuredAt":"2026-10-14","stderr":null,"costPerTest":17.659868181818183,"latency":null},{"benchmarkId":"epoch-ai:epoch-mystery-game-puzzles:overall","benchmarkName":"Mystery Game Puzzles","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":84,"unit":"percent","family":"epoch-ai:epoch-mystery-game-puzzles:overall","index":95,"aboveFit":true,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":3.6845294917747067,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-browser:overall","benchmarkName":"LiveBench Browser Use","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":84,"unit":"percent","family":"livebench-2-agentic","index":80.02128355568858,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Browser","measuredAt":"2026-10-14","stderr":null,"costPerTest":5.9096720000000005,"latency":null},{"benchmarkId":"livebench:livebench-data-analysis:overall","benchmarkName":"LiveBench Data Analysis","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":82.97333333333334,"unit":"percent","family":"livebench","index":69.1061981090197,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Data%20Analysis","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.698902,"latency":null},{"benchmarkId":"livebench:livebench-design:overall","benchmarkName":"LiveBench Design","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":82,"unit":"percent","family":"livebench-2-agentic","index":78.55862136584767,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Design","measuredAt":"2026-10-14","stderr":null,"costPerTest":2.87562,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":80.7,"unit":"percent","family":"lcr","index":62.664747873933486,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lifeVal","benchmarkName":"LifeVal: everyday life tasks","category":"agentic","score":80.6,"unit":"percent","family":"lifeVal","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-coding:overall","benchmarkName":"LiveBench Coding","category":"coding","domain":null,"eligible":true,"ranked":true,"score":80.35849999999999,"unit":"percent","family":"livebench:livebench-coding:overall","index":67.84107514758742,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Coding","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.29609743589743587,"latency":null},{"benchmarkId":"epoch-ai:epoch-furniture-assembly:overall","benchmarkName":"Furniture Assembly","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":80,"unit":"percent","family":"epoch-ai:epoch-furniture-assembly:overall","index":83.95763132237491,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":4.928872620664329,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:biomysterybench:overall","benchmarkName":"BioMysteryBench","category":null,"domain":"Science","eligible":false,"ranked":true,"score":79.259,"unit":"percent","family":"vals-ai:biomysterybench:overall","index":null,"aboveFit":false,"run":"Terminus 2","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/biomysterybench","measuredAt":"2026-11-09","stderr":0.98,"costPerTest":1.719326,"latency":286.147},{"benchmarkId":"euroeval:euroeval-italian:overall","benchmarkName":"EuroEval Italian","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":77.80625,"unit":"percent","family":"euroeval","index":77.78541400042637,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/italian","measuredAt":null,"stderr":0.2928416807764906,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":77.5825,"unit":"percent","family":"euroeval","index":77.48183673383461,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":0.4572352375965789,"costPerTest":null,"latency":null},{"benchmarkId":"biomedicalImageAnalysis","benchmarkName":"Anthropic biomedical-image-analysis evaluation","category":"multimodal","score":77.5,"unit":"percent","family":"biomedicalImageAnalysis","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-french:overall","benchmarkName":"EuroEval French","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":76.26124999999999,"unit":"percent","family":"euroeval","index":75.68920449479843,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/french","measuredAt":null,"stderr":0.3797552665862582,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-ebr-bench:overall","benchmarkName":"EBR-bench","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":76.19047619047619,"unit":"percent","family":"epoch-ai:epoch-ebr-bench:overall","index":87.39267461771944,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":8.384198505551907,"costPerTest":null,"latency":null},{"benchmarkId":"osWorld21","benchmarkName":"OSWorld 2.1 (all 108 tasks)","category":"agentic","score":75.8,"unit":"percent","family":"osWorld21","index":74.21119180940526,"aboveFit":false,"run":null,"source":"Mengqi Yuan, Zilong Zhou, Xinzhuang Xiong, Weiming Wu, Jiayang Sun, Jiamin Song, Kaiqian Cui, Bowen Wang, Haoyuan Wu, Yitong Li, Dunjie Lu, Haikong Lu, Qi Zhen, Xinyuan Wang, Jiaqi Deng, Yuhao Yang, Cheng Chen, Boyuan Zheng, Alex Su, Xiao Yu, Hao Zou, Saaket Agashe, Xing Han Lu, Manpreet Kaur, Zhengyang Qi, Vincent Sunn Chen, Frederic Sala, Dayiheng Liu, Junyang Lin, Zhou Yu, Yu Su, Siva Reddy, Xin Eric Wang, Peng Qi, Tianbao Xie, Tao Yu","sourceUrl":"https://github.com/xlang-ai/OSWorld-V2/releases","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-simpleqa-verified:overall","benchmarkName":"SimpleQA Verified","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":75.6,"unit":"percent","family":"epoch-ai:epoch-simpleqa-verified:overall","index":86.90052614836169,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.3588548437881387,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-instruction-following:overall","benchmarkName":"LiveBench Instruction Following","category":"instruction","domain":null,"eligible":true,"ranked":true,"score":75.57900000000001,"unit":"percent","family":"livebench:livebench-instruction-following:overall","index":79.51625325135363,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=IF","measuredAt":"2026-06-25","stderr":null,"costPerTest":1.235208,"latency":null},{"benchmarkId":"euroeval:euroeval-portuguese:overall","benchmarkName":"EuroEval Portuguese","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":74.7475,"unit":"percent","family":"euroeval","index":73.63539404875038,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/portuguese","measuredAt":null,"stderr":0.3883880952089031,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-dutch:overall","benchmarkName":"EuroEval Dutch","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":74.73,"unit":"percent","family":"euroeval","index":73.61165057538567,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/dutch","measuredAt":null,"stderr":0.3426746648439682,"costPerTest":null,"latency":null},{"benchmarkId":"deepSwe","benchmarkName":"DeepSWE","category":"agentic","score":74.1,"unit":"percent","family":"deepSwe","index":74.94535205681821,"aboveFit":false,"run":null,"source":"Datacurve AI","sourceUrl":"https://deepswe.datacurve.ai/blog","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-chess-puzzles:overall","benchmarkName":"Chess Puzzles","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":74,"unit":"percent","family":"epoch-ai:epoch-chess-puzzles:overall","index":90.15180016624606,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":4.4084400227680804,"costPerTest":null,"latency":null},{"benchmarkId":"hermesBench","benchmarkName":"Hermes Bench: 150 agent tasks graded on the files and state the agent leaves behind","category":"agentic","score":73.67,"unit":"percent","family":"hermesBench","index":null,"aboveFit":false,"run":null,"source":"Nous Research","sourceUrl":"https://portal.nousresearch.com/bench","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"osWorld2","benchmarkName":"OSWorld 2.0","category":"agentic","score":72.6,"unit":"percent","family":"osWorld2","index":83.42561891332676,"aboveFit":false,"run":null,"source":"Mengqi Yuan, Zilong Zhou, Xinzhuang Xiong, Weiming Wu, Jiayang Sun, Jiamin Song, Kaiqian Cui, Bowen Wang, Haoyuan Wu, Yitong Li, Dunjie Lu, Haikong Lu, Qi Zhen, Xinyuan Wang, Jiaqi Deng, Yuhao Yang, Cheng Chen, Boyuan Zheng, Alex Su, Xiao Yu, Hao Zou, Saaket Agashe, Xing Han Lu, Manpreet Kaur, Zhengyang Qi, Vincent Sunn Chen, Frederic Sala, Dayiheng Liu, Junyang Lin, Zhou Yu, Yu Su, Siva Reddy, Xin Eric Wang, Peng Qi, Tianbao Xie, Tao Yu","sourceUrl":"https://arxiv.org/abs/2606.29537","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"graphwalksBfs256k1m","benchmarkName":"GraphWalks BFS 256K–1M","category":"reasoning","score":71.8,"unit":"percent","family":"graphwalksBfs256k1m","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:emb:overall","benchmarkName":"EMB","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":71.7,"unit":"percent","family":"vals-ai:emb:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/emb","measuredAt":"2026-10-20","stderr":2.475,"costPerTest":5.827063,"latency":804.28},{"benchmarkId":"euroeval:euroeval-spanish:overall","benchmarkName":"EuroEval Spanish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":69.9975,"unit":"percent","family":"euroeval","index":67.19073699261278,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/spanish","measuredAt":null,"stderr":0.3985697084576298,"costPerTest":null,"latency":null},{"benchmarkId":"parseBenchLlamaIndex","benchmarkName":"ParseBench: document parsing for AI agents (LlamaIndex leaderboard)","category":"multimodal","score":69.81,"unit":"percent","family":"parseBenchLlamaIndex","index":71.12167749942861,"aboveFit":false,"run":null,"source":"Boyang Zhang, Sebastián G. Acosta, Preston Carlson, Sacha Bron, Pierre-Loïc Doulcet, Daniel B. Ospina and Simon Suo (LlamaIndex)","sourceUrl":"https://arxiv.org/abs/2604.08538","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaHealthBenchProfessionalRaw","benchmarkName":"Artificial Analysis HealthBench Professional raw score","category":"knowledge","score":69.7,"unit":"percent","family":"aaHealthBenchProfessionalRaw","index":null,"aboveFit":false,"run":null,"source":"OpenAI (benchmark); Artificial Analysis (evaluation)","sourceUrl":"https://artificialanalysis.ai/evaluations/healthbench-professional","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaAutomationBench","benchmarkName":"Artificial Analysis AutomationBench","category":"agentic","score":68.5,"unit":"percent","family":"aaAutomationBench","index":78.12982145809718,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/automationbench-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"healthBenchProfessionalRaw","benchmarkName":"HealthBench Professional raw score","category":"knowledge","score":68.2,"unit":"percent","family":"healthBenchProfessionalRaw","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":68.03875,"unit":"percent","family":"euroeval","index":64.5331639381476,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.4866033420559296,"costPerTest":null,"latency":null},{"benchmarkId":"apprenticeBench","benchmarkName":"ApprenticeBench: end-to-end computer use, continual learning, and long-horizon agency on a real accounts-payable job","category":"agentic","score":68,"unit":"percent","family":"apprenticeBench","index":92.6810872998575,"aboveFit":false,"run":null,"source":"NeoCognition","sourceUrl":"https://neocognition.io/blog/apprentice-bench/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"cweBenchV1","benchmarkName":"CWE-bench v1","category":"agentic","score":68,"unit":"percent","family":"cweBenchV1","index":79.47014074802985,"aboveFit":false,"run":null,"source":"Collinear AI","sourceUrl":"https://cwe-bench.com/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:code-migration:overall","benchmarkName":"Code Migration","category":"coding","domain":null,"eligible":true,"ranked":true,"score":67.74,"unit":"percent","family":"vals-ai:code-migration:overall","index":81.17534139196825,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/code-migration","measuredAt":"2026-10-29","stderr":4.216,"costPerTest":44.364548,"latency":3510.003},{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":67.38857142857144,"unit":"percent","family":"euroeval","index":63.65102121854623,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.3766067618368322,"costPerTest":null,"latency":null},{"benchmarkId":"bioBasicsBench","benchmarkName":"BioBasicsBench: everyday wet-lab science","category":"reasoning","score":66.98,"unit":"percent","family":"bioBasicsBench","index":null,"aboveFit":false,"run":null,"source":"OpenAI, Ligo","sourceUrl":"https://openai.com/index/measuring-how-ai-agents-handle-wet-lab-science/","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaHealthBenchProfessional","benchmarkName":"Artificial Analysis HealthBench Professional length-adjusted score","category":"knowledge","score":66,"unit":"percent","family":"aaHealthBenchProfessional","index":null,"aboveFit":false,"run":null,"source":"OpenAI (benchmark); Artificial Analysis (evaluation)","sourceUrl":"https://artificialanalysis.ai/evaluations/healthbench-professional","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"frontierSweV2","benchmarkName":"FrontierSWE v2","category":"coding","score":65.5,"unit":"percent","family":"frontierSweV2","index":84.80042787677831,"aboveFit":false,"run":null,"source":"Proximal","sourceUrl":"https://www.frontierswe.com/blog/v2","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"scarfBench","benchmarkName":"ScarfBench: Cross-Framework Application Migration in Enterprise Java","category":"coding","score":65.4,"unit":"percent","family":"scarfBench","index":70.92009091817691,"aboveFit":false,"run":null,"source":"Advait Pavuluri, Bridget McGinn, Ashita Saxena, George Safta, Srikanth Tamilselvam, Raju Pavuluri, Michele Merler, Baishakhi Ray, Rahul Krishna (IBM Research)","sourceUrl":"https://arxiv.org/abs/2605.06754","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"healthBenchProfessional","benchmarkName":"HealthBench Professional","category":"knowledge","score":64.7,"unit":"percent","family":"healthBenchProfessional","index":77.12602117676713,"aboveFit":false,"run":null,"source":"Rebecca Soskin Hicks, Mikhail Trofimov, Dominick Lim, Rahul K. Arora, Foivos Tsimpourlas, Preston Bowman, Michael Sharman, Chi Tong, Kavin Karthik, Arnav Dugar, Akshay Jagadeesh, Khaled Saab, Johannes Heidecke, Ashley Alexander, Nate Gross, Karan Singhal","sourceUrl":"https://arxiv.org/abs/2604.27470","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"terminalBenchScience","benchmarkName":"Terminal-Bench-Science 0.1","category":"agentic","score":64.6,"unit":"percent","family":"terminalBenchScience","index":81.38339558444306,"aboveFit":false,"run":null,"source":"Terminal-Bench-Science Team","sourceUrl":"https://www.terminal-bench-science.ai/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"frontierCode11Extended","benchmarkName":"FrontierCode 1.1 Extended","category":"coding","score":64.5,"unit":"percent","family":"frontierCode11Extended","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"cweBenchV2","benchmarkName":"CWE-bench v2","category":"agentic","score":64.2,"unit":"percent","family":"cweBenchV2","index":77.58289518561772,"aboveFit":false,"run":null,"source":"Collinear AI","sourceUrl":"https://cwe-bench.com/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaCweBench","benchmarkName":"Artificial Analysis CWE-Bench-AA","category":"agentic","score":63.3,"unit":"percent","family":"aaCweBench","index":74.67974800325587,"aboveFit":false,"run":null,"source":"Artificial Analysis; dataset by Collinear AI","sourceUrl":"https://artificialanalysis.ai/evaluations/cwe-bench-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:terminal-bench-science:overall","benchmarkName":"Terminal-Bench-Science","category":null,"domain":"Science","eligible":false,"ranked":true,"score":62.857,"unit":"percent","family":"vals-ai:terminal-bench-science:overall","index":null,"aboveFit":false,"run":"Mini-SWE-agent","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-science","measuredAt":"2026-11-09","stderr":5.817,"costPerTest":20.802802,"latency":6534.868},{"benchmarkId":"arc-prize:v3-semi-private:overall","benchmarkName":"ARC-AGI-3 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":62.71280210060628,"unit":"percent","family":"arc-prize:v3-semi-private:overall","index":91.94992052522568,"aboveFit":false,"run":"max effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"omniscienceAccuracy","benchmarkName":"Artificial Analysis Omniscience Accuracy","category":"knowledge","score":61.6,"unit":"percent","family":"omniscienceAccuracy","index":88.32171908469203,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"terminal-bench:terminal-bench-4-1-0:overall","benchmarkName":"Terminal-Bench 4.1.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":60.3,"unit":"percent","family":"terminal-bench:terminal-bench-4-1-0:overall","index":81.08147195092756,"aboveFit":false,"run":"max effort · Codex","source":"Terminal-Bench","sourceUrl":"https://www.tbench.ai/leaderboard","measuredAt":"2026-09-29","stderr":1.4387755102040816,"costPerTest":9.900545454545455,"latency":2796.3},{"benchmarkId":"epoch-ai:epoch-mirrorcode:overall","benchmarkName":"MirrorCode","category":"coding","domain":null,"eligible":true,"ranked":true,"score":60,"unit":"percent","family":"epoch-ai:epoch-mirrorcode:overall","index":75.91591528896458,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":11.818736805705578,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:terminal-bench-4:overall","benchmarkName":"Terminal-Bench 4.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":59.596,"unit":"percent","family":"terminal-bench-4","index":82.81449654389385,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-4","measuredAt":"2026-11-09","stderr":4.403,"costPerTest":9.584259,"latency":2142.082},{"benchmarkId":"agentsLastExam","benchmarkName":"Agents' Last Exam","category":"agentic","score":59.3,"unit":"percent","family":"agentsLastExam","index":90.28713223878145,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaTerminalBench4","benchmarkName":"Artificial Analysis Terminal-Bench v4.0","category":"agentic","score":59.1,"unit":"percent","family":"aaTerminalBench4","index":79.29061216717952,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/terminalbench-v4-0","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"healthBenchLengthAdjusted","benchmarkName":"HealthBench length-adjusted score","category":"knowledge","score":58.3,"unit":"percent","family":"healthBenchLengthAdjusted","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaHealthBenchLengthAdjusted","benchmarkName":"Artificial Analysis HealthBench length-adjusted score","category":"knowledge","score":58.2,"unit":"percent","family":"aaHealthBenchLengthAdjusted","index":null,"aboveFit":false,"run":null,"source":"OpenAI (benchmark); Artificial Analysis (evaluation)","sourceUrl":"https://artificialanalysis.ai/evaluations/healthbench","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"terminal-bench:terminal-bench-4-0-0:overall","benchmarkName":"Terminal-Bench 4.0.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":58.18,"unit":"percent","family":"terminal-bench-4","index":82.14212504180475,"aboveFit":false,"run":"max effort · Codex","source":"Terminal-Bench","sourceUrl":"https://www.tbench.ai/leaderboard","measuredAt":"2026-09-29","stderr":1.4234693877551021,"costPerTest":9.900545454545455,"latency":2796.3},{"benchmarkId":"extractBenchPro","benchmarkName":"ExtractBench Pro: long-tail document extraction challenges (LlamaIndex)","category":"multimodal","score":58,"unit":"percent","family":"extractBenchPro","index":null,"aboveFit":false,"run":null,"source":"LlamaIndex","sourceUrl":"https://www.extractbench.ai/pro","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-agentic-coding:overall","benchmarkName":"LiveBench Agentic Coding","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":57.323,"unit":"percent","family":"livebench:livebench-agentic-coding:overall","index":68.27892395560225,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Agentic%20Coding","measuredAt":"2026-06-25","stderr":null,"costPerTest":2.4324874999999997,"latency":null},{"benchmarkId":"hleWithTools","benchmarkName":"Humanity's Last Exam with tools","category":"agentic","score":57.2,"unit":"percent","family":"hleWithTools","index":66.17650623223959,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"healthBench","benchmarkName":"HealthBench raw score","category":"knowledge","score":56.9,"unit":"percent","family":"healthBench","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaHealthBench","benchmarkName":"Artificial Analysis HealthBench raw score","category":"knowledge","score":56.9,"unit":"percent","family":"aaHealthBench","index":null,"aboveFit":false,"run":null,"source":"OpenAI (benchmark); Artificial Analysis (evaluation)","sourceUrl":"https://artificialanalysis.ai/evaluations/healthbench","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:srebench:overall","benchmarkName":"SRE Bench","category":null,"domain":"Other","eligible":false,"ranked":true,"score":56.87,"unit":"percent","family":"vals-ai:srebench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/srebench","measuredAt":"2026-11-05","stderr":3.066,"costPerTest":10.238062,"latency":1070.676},{"benchmarkId":"aaSciCode","benchmarkName":"Artificial Analysis SciCode","category":"coding","score":56.5,"unit":"percent","family":"scicode-family","index":69.49834821494369,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/scicode","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hermesIndex","benchmarkName":"Hermes Index: agentic capability and cost per task inside Hermes Agent","category":"agentic","score":56.25,"unit":"percent","family":"hermesIndex","index":80.04452413443454,"aboveFit":false,"run":null,"source":"Nous Research","sourceUrl":"https://portal.nousresearch.com/bench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":54.7,"unit":"percent","family":"hle","index":79.26675746644324,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gdpvalAaNormalized","benchmarkName":"GDPval-AA normalized","category":"agentic","score":53.7,"unit":"percent","family":"gdpvalAaNormalized","index":72.42802702004417,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:fabv2:overall","benchmarkName":"Finance Agent (v2)","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":53.54,"unit":"percent","family":"vals-ai:fabv2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/fabv2","measuredAt":"2026-10-29","stderr":2.078,"costPerTest":6.81746,"latency":692.672},{"benchmarkId":"frontierCode","benchmarkName":"FrontierCode 1.1 Main","category":"coding","score":53.3,"unit":"percent","family":"frontierCode","index":76.42262738216259,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:mysterymechanism:overall","benchmarkName":"MysteryMechanism","category":null,"domain":"Science","eligible":false,"ranked":true,"score":53.153,"unit":"percent","family":"vals-ai:mysterymechanism:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mysterymechanism","measuredAt":"2026-11-11","stderr":3.357,"costPerTest":1.559704,"latency":488.597},{"benchmarkId":"aaBriefcaseRubric","benchmarkName":"AA-Briefcase v1.1, rubric pass rate","category":"agentic","score":52,"unit":"percent","family":"aaBriefcaseRubric","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaAnalystAgent","benchmarkName":"Artificial Analysis AnalystAgent","category":"agentic","score":51.2,"unit":"percent","family":"aaAnalystAgent","index":73.36719742513782,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/aa-analyst-agent","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mka","benchmarkName":"Multimodal Knowledge Agent Benchmark","category":"agentic","score":50.2,"unit":"percent","family":"mka","index":78.10296283763549,"aboveFit":false,"run":null,"source":"Google DeepMind, in partnership with Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mka","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaItbench","benchmarkName":"Artificial Analysis ITBench-AA","category":"agentic","score":48.6,"unit":"percent","family":"aaItbench","index":69.90453111437807,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/itbench-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:medcode:overall","benchmarkName":"MedCode","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":48.486,"unit":"percent","family":"vals-ai:medcode:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medcode","measuredAt":"2026-10-29","stderr":2.131,"costPerTest":0.451358,"latency":102.865},{"benchmarkId":"physicianBench","benchmarkName":"PhysicianBench","category":"knowledge","score":48,"unit":"percent","family":"physicianBench","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"bug-hunt-bench","benchmarkName":"Bug Hunt Bench","category":"coding","score":45,"unit":"percent","family":"bug-hunt-bench","index":79.7163387218913,"aboveFit":false,"run":null,"source":"Pawel Huryn","sourceUrl":"https://bughunt.productcompass.pm/method","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"postTrainBench11","benchmarkName":"PostTrainBench v1.1","category":"coding","score":44.3,"unit":"percent","family":"postTrainBench11","index":77.58289518561772,"aboveFit":false,"run":null,"source":"PostTrainBench team","sourceUrl":"https://posttrainbench.com/?version=v1.1","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"frontierCode2","benchmarkName":"FrontierCode 2.0","category":"coding","score":42.8,"unit":"percent","family":"frontierCode2","index":83.75784803750832,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"exploitGym","benchmarkName":"ExploitGym","category":"agentic","score":42.4,"unit":"percent","family":"exploitGym","index":84.71480106244317,"aboveFit":false,"run":null,"source":"Zhun Wang, Nico Schiller, Hongwei Li, Srijiith Sesha Narayana, Milad Nasr, Nicholas Carlini, Xiangyu Qi, Eric Wallace, Elie Bursztein, Luca Invernizzi, Kurt Thomas, Yan Shoshitaishvili, Wenbo Guo, Jingxuan He, Thorsten Holz, Dawn Song","sourceUrl":"https://arxiv.org/abs/2605.11086","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaMlcr","benchmarkName":"Medical Long Context Reasoning (MLCR-AA)","category":"reasoning","score":41.7,"unit":"percent","family":"aaMlcr","index":85.43885612748115,"aboveFit":false,"run":null,"source":"Wisedocs and Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mlcr-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"automationBench","benchmarkName":"AutomationBench","category":"agentic","score":41.4,"unit":"percent","family":"automationBench","index":60.837102208837386,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaTau3Banking","benchmarkName":"Artificial Analysis Tau3-Banking","category":"agentic","score":41.4,"unit":"percent","family":"aaTau3Banking","index":65.73814074685932,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/tau3-banking","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:cyber:overall","benchmarkName":"CyberBench v1.1","category":null,"domain":"Other","eligible":false,"ranked":true,"score":41.072,"unit":"percent","family":"vals-ai:cyber:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/cyber","measuredAt":"2026-11-05","stderr":2.559,"costPerTest":1.840737,"latency":250.415},{"benchmarkId":"vals-ai:legal_research:overall","benchmarkName":"Legal Research Bench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":39.423,"unit":"percent","family":"vals-ai:legal_research:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_research","measuredAt":"2026-11-09","stderr":3.397,"costPerTest":10.577143,"latency":1491.339},{"benchmarkId":"geneBenchPro","benchmarkName":"GeneBench-Pro","category":"reasoning","score":37.8,"unit":"percent","family":"geneBenchPro","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaDeepsecBench","benchmarkName":"Artificial Analysis DeepsecBench-AA","category":"agentic","score":36.9,"unit":"percent","family":"aaDeepsecBench","index":78.66928046851096,"aboveFit":false,"run":null,"source":"Artificial Analysis; dataset by Vercel","sourceUrl":"https://artificialanalysis.ai/evaluations/deepsecbench-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"healthBenchHard","benchmarkName":"HealthBench Hard","category":"knowledge","score":36.6,"unit":"percent","family":"healthBenchHard","index":77.02312768778751,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-spireclimb-slay-the-spire-2-ascension-10:overall","benchmarkName":"SpireClimb (Slay the Spire 2, Ascension 10)","category":null,"domain":"Other","eligible":false,"ranked":true,"score":34.66666666666667,"unit":"percent","family":"epoch-ai:epoch-spireclimb-slay-the-spire-2-ascension-10:overall","index":null,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":5.532323798062311,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-spireclimb:overall","benchmarkName":"SpireClimb","category":null,"domain":"Other","eligible":false,"ranked":true,"score":34,"unit":"percent","family":"epoch-ai:epoch-spireclimb:overall","index":null,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":4.131325151634216,"costPerTest":null,"latency":null},{"benchmarkId":"aaCyberIndex","benchmarkName":"Artificial Analysis Cyber Index","category":"agentic","score":33.4,"unit":"percent","family":"aaCyberIndex","index":61.53038963668777,"aboveFit":false,"run":null,"source":"Artificial Analysis with the Cyber Index Alliance (Collinear AI, IBM, NVIDIA, Pasta Lab, Vercel)","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-cyber-index","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-spireclimb-slay-the-spire-1-ascension-20-heart:overall","benchmarkName":"SpireClimb (Slay the Spire 1, Ascension 20 Heart)","category":null,"domain":"Other","eligible":false,"ranked":true,"score":33.33333333333333,"unit":"percent","family":"epoch-ai:epoch-spireclimb-slay-the-spire-1-ascension-20-heart:overall","index":null,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":6.137164119322156,"costPerTest":null,"latency":null},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":31.7,"unit":"percent","family":"critpt","index":95,"aboveFit":true,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaGdpPdf","benchmarkName":"Artificial Analysis GDP.pdf","category":"agentic","score":31,"unit":"percent","family":"aaGdpPdf","index":79.012017156606,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdp-pdf","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"roboDojoSim","benchmarkName":"RoboDojo Sim","category":"agentic","score":29.8,"unit":"percent","family":"roboDojoSim","index":null,"aboveFit":false,"run":null,"source":"Anthropic (reported run)","sourceUrl":"https://www.anthropic.com/claude-fable-and-mythos-5-2","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:vcb-1-100:overall","benchmarkName":"Vibe Code Bench 1-100","category":"coding","domain":null,"eligible":true,"ranked":true,"score":27.644,"unit":"percent","family":"vals-ai:vcb-1-100:overall","index":75.38679884217527,"aboveFit":false,"run":"OpenHands","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vcb-1-100","measuredAt":"2026-10-29","stderr":4.082,"costPerTest":84.720155,"latency":12339.625},{"benchmarkId":"mazeBench","benchmarkName":"MazeBench: visual spatial reasoning in a 3D open world","category":"agentic","score":23,"unit":"percent","family":"mazeBench","index":95,"aboveFit":true,"run":null,"source":"Jonathan Pappas and David Pappas","sourceUrl":"https://mazebench.com/blog?post=maze-bench-results","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"morphologyToMolecule","benchmarkName":"Axiom Bio morphology-to-molecule matching","category":"knowledge","score":22.8,"unit":"percent","family":"morphologyToMolecule","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"rliAutomationRate","benchmarkName":"Remote Labor Index (RLI)","category":"agentic","score":20.8,"unit":"percent","family":"rliAutomationRate","index":null,"aboveFit":false,"run":null,"source":"Center for AI Safety; Scale AI","sourceUrl":"https://scale.com/leaderboard/rli","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:tax_agent_bench:overall","benchmarkName":"Tax Agent Bench","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":20.717,"unit":"percent","family":"vals-ai:tax_agent_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench","measuredAt":"2026-11-09","stderr":2.924,"costPerTest":5.7963,"latency":900.586},{"benchmarkId":"vals-ai:cua_bench:overall","benchmarkName":"CUA-bench","category":null,"domain":"Other","eligible":false,"ranked":true,"score":19.166667,"unit":"percent","family":"vals-ai:cua_bench:overall","index":null,"aboveFit":false,"run":"codex","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/cua_bench","measuredAt":"2026-10-13","stderr":0,"costPerTest":1821.373046,"latency":32.09},{"benchmarkId":"lmarena:agent-steerability:overall","benchmarkName":"Agent Arena steerability","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":14.817267999381695,"unit":"index","family":"agent-arena","index":78.71291852890596,"aboveFit":false,"run":"max effort","source":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset","measuredAt":"2026-11-11","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lmarena:agent-task-outcome:overall","benchmarkName":"Agent Arena task outcome","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":14.046800888110234,"unit":"index","family":"agent-arena","index":78.07947377628884,"aboveFit":false,"run":"max effort","source":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset","measuredAt":"2026-11-11","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaHarveyLabV11","benchmarkName":"Harvey LAB-AA v1.1","category":"agentic","score":8.6,"unit":"percent","family":"aaHarveyLabV11","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:hlab:overall","benchmarkName":"HLAB-Vals-1.1","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":7.083,"unit":"percent","family":"vals-ai:hlab:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/hlab","measuredAt":"2026-11-09","stderr":1.899,"costPerTest":25.460295,"latency":1953.362},{"benchmarkId":"lmarena:agent-bash-recovery:overall","benchmarkName":"Agent Arena command recovery","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":5.5274748105542,"unit":"index","family":"agent-arena","index":71.07525215447745,"aboveFit":false,"run":"max effort","source":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset","measuredAt":"2026-10-08","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:programbench:overall","benchmarkName":"ProgramBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":5.5,"unit":"percent","family":"vals-ai:programbench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/programbench","measuredAt":"2026-11-09","stderr":1.616,"costPerTest":11.574605,"latency":1621.514},{"benchmarkId":"epoch-ai:epoch-frontiermath-erdos:overall","benchmarkName":"FrontierMath-Erdos","category":"math","domain":null,"eligible":true,"ranked":true,"score":2.941176470588235,"unit":"percent","family":"epoch-ai:epoch-frontiermath-erdos:overall","index":null,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.06414713705158,"costPerTest":null,"latency":null},{"benchmarkId":"stemBench120","benchmarkName":"STEMBench120: six-domain scientific research progress","category":"reasoning","score":2.809,"unit":"percent","family":"stemBench120","index":null,"aboveFit":false,"run":null,"source":"Ulam","sourceUrl":"https://www.ulam.ai/stembench","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaCyberGymE2e","benchmarkName":"Artificial Analysis CyberGym-E2E-AA","category":"agentic","score":0,"unit":"percent","family":"aaCyberGymE2e","index":5,"aboveFit":false,"run":null,"source":"Artificial Analysis; dataset by Berkeley RDI","sourceUrl":"https://artificialanalysis.ai/evaluations/cybergym-e2e-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":10,"outputPerMillion":50,"batch":{"inputPerMillion":5,"outputPerMillion":25},"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":34,"timeToFirstToken":3.7735,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":{"elo":1439.895953820857,"eloUpper":1445.4649614101904,"eloLower":1434.326946231523,"votes":18315,"rank":77,"rankUpper":null,"rankLower":null,"arenaName":"GPT-6 Astra-max","organization":"openai","updatedAt":"2026-11-09"},"quality":{"coveragePercent":100,"benchmarkCount":76,"categoryCount":8,"confidence":"high","state":"available"},"hosts":[{"host":"amazon-bedrock","name":"Amazon Bedrock","kind":"cloud","url":"https://docs.aws.amazon.com/bedrock/latest/userguide/models-supported.html","inputPerMillion":10,"outputPerMillion":50,"cacheReadPerMillion":1,"contextTokens":1050000,"maxOutputTokens":128000,"deprecated":false},{"host":"digitalocean","name":"DigitalOcean","kind":"cloud","url":"https://docs.digitalocean.com/products/gradient-ai-platform/details/models/","inputPerMillion":10,"outputPerMillion":50,"cacheReadPerMillion":1,"contextTokens":1050000,"maxOutputTokens":128000,"deprecated":false},{"host":"azure","name":"Microsoft Azure","kind":"cloud","url":"https://learn.microsoft.com/en-us/azure/ai-services/openai/concepts/models","inputPerMillion":10,"outputPerMillion":50,"cacheReadPerMillion":1,"contextTokens":1050000,"maxOutputTokens":128000,"deprecated":false},{"host":"openai","name":"OpenAI","kind":"maker","url":"https://platform.openai.com/docs/models","inputPerMillion":10,"outputPerMillion":50,"cacheReadPerMillion":1,"contextTokens":1050000,"maxOutputTokens":128000,"deprecated":false},{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":10,"outputPerMillion":50,"cacheReadPerMillion":1,"contextTokens":1050000,"maxOutputTokens":128000,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:arc-prize-foundation","name":"ARC Prize Foundation","url":"https://arcprize.org","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:livebench","name":"LiveBench","url":"https://livebench.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:terminal-bench","name":"Terminal-Bench","url":"https://www.tbench.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:lmarena","name":"LMArena","url":"https://huggingface.co","license":"CC BY 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gpt-6-1-sol","providerId":"openai","slug":"gpt-6-1-sol","name":"GPT-6.1 Sol","provider":"OpenAI","description":"GPT-6.1 Sol is a reasoning model from OpenAI in the GPT-6 family. 55 benchmarks count toward its index, in 8 categories.","logoUrl":null,"releaseDate":"2026-09-29","contextWindowTokens":1050000,"modalities":["file","image","text"],"openWeights":false,"details":{"apiId":"openai/gpt-6.1-sol","maxOutputTokens":128000,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.09999999999999999,"weightsUrl":null},"reasoning":true,"overallScore":77.9792079718941,"overallScoreRange":{"low":75.79884561668797,"high":80.15957032710023},"badges":[],"capabilityScores":{"agentic":75.96568548654875,"coding":75.84368467145431,"reasoning":81.81958644977483,"multimodal":72.1732980431031,"knowledge":77.97994635797437,"multilingual":73.17088879602001,"instruction":76.63332880023188,"math":78.2288586206382},"capabilityCounts":{"math":5,"coding":10,"reasoning":11,"knowledge":7,"multimodal":3,"agentic":17,"multilingual":1,"instruction":1},"benchmarks":[{"benchmarkId":"epoch-ai:epoch-frontiermath-tier-4-v2-private:overall","benchmarkName":"FrontierMath-Tier-4-v2-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":100,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tier-4-v2-private:overall","index":87.77330837649896,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":100,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":67.09849544987334,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:proof_bench:overall","benchmarkName":"ProofBench v1.1","category":"math","domain":null,"eligible":true,"ranked":true,"score":99,"unit":"percent","family":"vals-ai:proof_bench:overall","index":81.33584598884255,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/proof_bench","measuredAt":"2026-11-09","stderr":1,"costPerTest":0.233057,"latency":284.697},{"benchmarkId":"arc-prize:v1-public-eval:overall","benchmarkName":"ARC-AGI-1 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":98.5,"unit":"percent","family":"arc-prize:v1-public-eval:overall","index":null,"aboveFit":false,"run":"max effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.076886565,"latency":null},{"benchmarkId":"arc-prize:v2-public-eval:overall","benchmarkName":"ARC-AGI-2 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":97.08333333333333,"unit":"percent","family":"arc-prize:v2-public-eval:overall","index":null,"aboveFit":false,"run":"max effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.22565961666666653,"latency":null},{"benchmarkId":"vals-ai:ioi:overall","benchmarkName":"IOI","category":"coding","domain":null,"eligible":true,"ranked":true,"score":96.889,"unit":"percent","family":"vals-ai:ioi:overall","index":82.49093516428414,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/ioi","measuredAt":"2026-10-29","stderr":3.111,"costPerTest":1.261598,"latency":1890.714},{"benchmarkId":"livebench:livebench-math:overall","benchmarkName":"LiveBench Mathematics","category":"math","domain":null,"eligible":true,"ranked":true,"score":96.83175,"unit":"percent","family":"livebench:livebench-math:overall","index":74.73110245045757,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Mathematics","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.04943562005277045,"latency":null},{"benchmarkId":"arc-prize:v1-semi-private:overall","benchmarkName":"ARC-AGI-1 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":96.5,"unit":"percent","family":"arc-prize:v1-semi-private:overall","index":71.35189279722005,"aboveFit":false,"run":"max effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.09543634000000001,"latency":null},{"benchmarkId":"livebench:livebench-coding-2:overall","benchmarkName":"LiveBench Coding 2.0","category":"coding","domain":null,"eligible":true,"ranked":true,"score":96,"unit":"percent","family":"livebench:livebench-coding-2:overall","index":77.70712617523418,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Coding","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.4822,"latency":null},{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":95.39141414141415,"unit":"percent","family":"gpqa-diamond","index":64.69949921031134,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.3780748450364693,"costPerTest":null,"latency":null},{"benchmarkId":"extractBenchLlamaIndex","benchmarkName":"ExtractBench: schema-guided enterprise document extraction (LlamaIndex leaderboard)","category":"multimodal","score":95.31,"unit":"percent","family":"extractBenchLlamaIndex","index":78.57510969045286,"aboveFit":false,"run":null,"source":"Boyang Zhang, Adrian Lyjak, Eli Stewart, Zhaoqi Li and Simon Suo (LlamaIndex)","sourceUrl":"https://arxiv.org/abs/2607.29677","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"arc-prize:v2-semi-private:overall","benchmarkName":"ARC-AGI-2 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":94.16666666666667,"unit":"percent","family":"arc-prize:v2-semi-private:overall","index":82.42369974416391,"aboveFit":false,"run":"max effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.25374978333333337,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tiers-1-3-v2-private:overall","benchmarkName":"FrontierMath-Tiers-1-3-v2-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":93.6842105263158,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tiers-1-3-v2-private:overall","index":80.2055408375186,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.4434038295343787,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-reasoning:overall","benchmarkName":"LiveBench Reasoning","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":92.6345,"unit":"percent","family":"livebench","index":81.6216018514576,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Reasoning","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.08031336633663368,"latency":null},{"benchmarkId":"vals-ai:sage:overall","benchmarkName":"SAGE","category":null,"domain":"Education","eligible":false,"ranked":true,"score":90.493,"unit":"percent","family":"vals-ai:sage:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/sage","measuredAt":"2026-11-05","stderr":1.528,"costPerTest":0.082669,"latency":50.427},{"benchmarkId":"livebench:livebench-language:overall","benchmarkName":"LiveBench Language","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":90.12599999999999,"unit":"percent","family":"livebench:livebench-language:overall","index":76.47408950832553,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Language","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.04059533333333333,"latency":null},{"benchmarkId":"vals-ai:vibe-code:overall","benchmarkName":"Vibe Code Bench v1.1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":88.931,"unit":"percent","family":"vals-ai:vibe-code:overall","index":74.33722460176153,"aboveFit":false,"run":"OpenHands","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","measuredAt":"2026-11-09","stderr":1.893,"costPerTest":6.234794,"latency":4489.755},{"benchmarkId":"livebench:livebench-browser:overall","benchmarkName":"LiveBench Browser Use","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":88,"unit":"percent","family":"livebench-2-agentic","index":82.94660793537042,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Browser","measuredAt":"2026-10-14","stderr":null,"costPerTest":2.437332,"latency":null},{"benchmarkId":"vals-ai:medscribe:overall","benchmarkName":"MedScribe","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":86.453,"unit":"percent","family":"vals-ai:medscribe:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medscribe","measuredAt":"2026-10-29","stderr":1.898,"costPerTest":0.100902,"latency":169.619},{"benchmarkId":"aaMmmuPro","benchmarkName":"Artificial Analysis MMMU-Pro","category":"multimodal","score":86,"unit":"percent","family":"mmmu-pro","index":67.07426982808389,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmmu-pro","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-research:overall","benchmarkName":"LiveBench Research","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":83.5,"unit":"percent","family":"livebench-2-agentic","index":79.65561800822836,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Research","measuredAt":"2026-10-14","stderr":null,"costPerTest":2.415190909090909,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":83,"unit":"percent","family":"lcr","index":64.2427330827805,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-data-analysis:overall","benchmarkName":"LiveBench Data Analysis","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":82.65933333333334,"unit":"percent","family":"livebench","index":68.69943183422967,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Data%20Analysis","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.145418,"latency":null},{"benchmarkId":"livebench:livebench-design:overall","benchmarkName":"LiveBench Design","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":82.1,"unit":"percent","family":"livebench-2-agentic","index":78.63175447533972,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Design","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.370544,"latency":null},{"benchmarkId":"livebench:livebench-coding:overall","benchmarkName":"LiveBench Coding","category":"coding","domain":null,"eligible":true,"ranked":true,"score":80.35849999999999,"unit":"percent","family":"livebench:livebench-coding:overall","index":67.84107514758742,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Coding","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.05827606837606837,"latency":null},{"benchmarkId":"epoch-ai:epoch-mystery-game-puzzles:overall","benchmarkName":"Mystery Game Puzzles","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":80,"unit":"percent","family":"epoch-ai:epoch-mystery-game-puzzles:overall","index":95,"aboveFit":true,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":4.020151261036848,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-furniture-assembly:overall","benchmarkName":"Furniture Assembly","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":80,"unit":"percent","family":"epoch-ai:epoch-furniture-assembly:overall","index":83.95763132237491,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":4.928872620664329,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:biomysterybench:overall","benchmarkName":"BioMysteryBench","category":null,"domain":"Science","eligible":false,"ranked":true,"score":79.63,"unit":"percent","family":"vals-ai:biomysterybench:overall","index":null,"aboveFit":false,"run":"Terminus 2","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/biomysterybench","measuredAt":"2026-11-09","stderr":0.741,"costPerTest":0.372147,"latency":530.7},{"benchmarkId":"euroeval:euroeval-italian:overall","benchmarkName":"EuroEval Italian","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":78.54375,"unit":"percent","family":"euroeval","index":78.78603180651089,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/italian","measuredAt":null,"stderr":0.2965267694492354,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":77.47500000000001,"unit":"percent","family":"euroeval","index":77.33598396887994,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":0.4384756977302163,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-french:overall","benchmarkName":"EuroEval French","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":76.1075,"unit":"percent","family":"euroeval","index":75.48060112166557,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/french","measuredAt":null,"stderr":0.3779281512404177,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-portuguese:overall","benchmarkName":"EuroEval Portuguese","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":74.7625,"unit":"percent","family":"euroeval","index":73.65574559734871,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/portuguese","measuredAt":null,"stderr":0.3719963037450776,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-instruction-following:overall","benchmarkName":"LiveBench Instruction Following","category":"instruction","domain":null,"eligible":true,"ranked":true,"score":74.15425,"unit":"percent","family":"livebench:livebench-instruction-following:overall","index":76.63332880023188,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=IF","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.239508,"latency":null},{"benchmarkId":"euroeval:euroeval-dutch:overall","benchmarkName":"EuroEval Dutch","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":74.04777777777778,"unit":"percent","family":"euroeval","index":72.68603199469129,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/dutch","measuredAt":null,"stderr":0.351088255233585,"costPerTest":null,"latency":null},{"benchmarkId":"hermesBench","benchmarkName":"Hermes Bench: 150 agent tasks graded on the files and state the agent leaves behind","category":"agentic","score":73.93,"unit":"percent","family":"hermesBench","index":null,"aboveFit":false,"run":null,"source":"Nous Research","sourceUrl":"https://portal.nousresearch.com/bench","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-simpleqa-verified:overall","benchmarkName":"SimpleQA Verified","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":73.9,"unit":"percent","family":"epoch-ai:epoch-simpleqa-verified:overall","index":85.4842760752121,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.3895037677965185,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-long-horizon:overall","benchmarkName":"LiveBench Long Agentic","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":73,"unit":"percent","family":"livebench-2-agentic","index":71.97664151156356,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Long%20Agentic","measuredAt":"2026-10-14","stderr":null,"costPerTest":6.4876611111111115,"latency":null},{"benchmarkId":"deepSwe","benchmarkName":"DeepSWE","category":"agentic","score":71.9,"unit":"percent","family":"deepSwe","index":73.34601347203152,"aboveFit":false,"run":null,"source":"Datacurve AI","sourceUrl":"https://deepswe.datacurve.ai/blog","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:emb:overall","benchmarkName":"EMB","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":70.773,"unit":"percent","family":"vals-ai:emb:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/emb","measuredAt":"2026-10-20","stderr":2.712,"costPerTest":1.143199,"latency":1107.811},{"benchmarkId":"parseBenchLlamaIndex","benchmarkName":"ParseBench: document parsing for AI agents (LlamaIndex leaderboard)","category":"multimodal","score":69.63,"unit":"percent","family":"parseBenchLlamaIndex","index":70.87051461077255,"aboveFit":false,"run":null,"source":"Boyang Zhang, Sebastián G. Acosta, Preston Carlson, Sacha Bron, Pierre-Loïc Doulcet, Daniel B. Ospina and Simon Suo (LlamaIndex)","sourceUrl":"https://arxiv.org/abs/2604.08538","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"scarfBench","benchmarkName":"ScarfBench: Cross-Framework Application Migration in Enterprise Java","category":"coding","score":68.4,"unit":"percent","family":"scarfBench","index":73.16640482245651,"aboveFit":false,"run":null,"source":"Advait Pavuluri, Bridget McGinn, Ashita Saxena, George Safta, Srikanth Tamilselvam, Raju Pavuluri, Michele Merler, Baishakhi Ray, Rahul Krishna (IBM Research)","sourceUrl":"https://arxiv.org/abs/2605.06754","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-spanish:overall","benchmarkName":"EuroEval Spanish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":68.21374999999999,"unit":"percent","family":"euroeval","index":64.77059867179477,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/spanish","measuredAt":null,"stderr":0.4751233392499257,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-chess-puzzles:overall","benchmarkName":"Chess Puzzles","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":68,"unit":"percent","family":"epoch-ai:epoch-chess-puzzles:overall","index":85.5977089953455,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":4.688261722621507,"costPerTest":null,"latency":null},{"benchmarkId":"healthBenchProfessionalRaw","benchmarkName":"HealthBench Professional raw score","category":"knowledge","score":67.2,"unit":"percent","family":"healthBenchProfessionalRaw","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":67.1425,"unit":"percent","family":"euroeval","index":63.31715890939743,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.5017267059864363,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":66.89285714285714,"unit":"percent","family":"euroeval","index":62.978450993439516,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.30465437090953695,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:code-migration:overall","benchmarkName":"Code Migration","category":"coding","domain":null,"eligible":true,"ranked":true,"score":65.122,"unit":"percent","family":"vals-ai:code-migration:overall","index":79.78455482665797,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/code-migration","measuredAt":"2026-10-29","stderr":4.362,"costPerTest":6.505708,"latency":5781.916},{"benchmarkId":"aaAutomationBench","benchmarkName":"Artificial Analysis AutomationBench","category":"agentic","score":64.9,"unit":"percent","family":"aaAutomationBench","index":74.84426005702379,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/automationbench-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"healthBenchProfessional","benchmarkName":"HealthBench Professional","category":"knowledge","score":64.2,"unit":"percent","family":"healthBenchProfessional","index":76.62046103023059,"aboveFit":false,"run":null,"source":"Rebecca Soskin Hicks, Mikhail Trofimov, Dominick Lim, Rahul K. Arora, Foivos Tsimpourlas, Preston Bowman, Michael Sharman, Chi Tong, Kavin Karthik, Arnav Dugar, Akshay Jagadeesh, Khaled Saab, Johannes Heidecke, Ashley Alexander, Nate Gross, Karan Singhal","sourceUrl":"https://arxiv.org/abs/2604.27470","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"bioBasicsBench","benchmarkName":"BioBasicsBench: everyday wet-lab science","category":"reasoning","score":62.61,"unit":"percent","family":"bioBasicsBench","index":null,"aboveFit":false,"run":null,"source":"OpenAI, Ligo","sourceUrl":"https://openai.com/index/measuring-how-ai-agents-handle-wet-lab-science/","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"omniscienceAccuracy","benchmarkName":"Artificial Analysis Omniscience Accuracy","category":"knowledge","score":61.8,"unit":"percent","family":"omniscienceAccuracy","index":88.57377028150393,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-mirrorcode:overall","benchmarkName":"MirrorCode","category":"coding","domain":null,"eligible":true,"ranked":true,"score":61.111111111111114,"unit":"percent","family":"epoch-ai:epoch-mirrorcode:overall","index":76.40388324875478,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":11.34670857184858,"costPerTest":null,"latency":null},{"benchmarkId":"terminal-bench:terminal-bench-4-1-0:overall","benchmarkName":"Terminal-Bench 4.1.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":60.91,"unit":"percent","family":"terminal-bench:terminal-bench-4-1-0:overall","index":81.36567724358298,"aboveFit":false,"run":"max effort · Codex","source":"Terminal-Bench","sourceUrl":"https://www.tbench.ai/leaderboard","measuredAt":"2026-09-29","stderr":1.5153061224489797,"costPerTest":1.921939393939394,"latency":3176.7},{"benchmarkId":"vals-ai:public-benefits-bench:overall","benchmarkName":"Public Benefits Bench v1.1","category":null,"domain":"Public benefits","eligible":false,"ranked":true,"score":59.337,"unit":"percent","family":"vals-ai:public-benefits-bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/public-benefits-bench","measuredAt":"2026-10-12","stderr":1.278,"costPerTest":0.690795,"latency":3024.455},{"benchmarkId":"healthBenchLengthAdjusted","benchmarkName":"HealthBench length-adjusted score","category":"knowledge","score":58.5,"unit":"percent","family":"healthBenchLengthAdjusted","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"terminal-bench:terminal-bench-4-0-0:overall","benchmarkName":"Terminal-Bench 4.0.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":58.18,"unit":"percent","family":"terminal-bench-4","index":82.14212504180475,"aboveFit":false,"run":"max effort · Codex","source":"Terminal-Bench","sourceUrl":"https://www.tbench.ai/leaderboard","measuredAt":"2026-09-29","stderr":1.586734693877551,"costPerTest":1.921939393939394,"latency":3176.7},{"benchmarkId":"hermesIndex","benchmarkName":"Hermes Index: agentic capability and cost per task inside Hermes Agent","category":"agentic","score":57.09,"unit":"percent","family":"hermesIndex","index":80.6541400572074,"aboveFit":false,"run":null,"source":"Nous Research","sourceUrl":"https://portal.nousresearch.com/bench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"terminalBenchScience","benchmarkName":"Terminal-Bench-Science 0.1","category":"agentic","score":57,"unit":"percent","family":"terminalBenchScience","index":77.10367196209471,"aboveFit":false,"run":null,"source":"Terminal-Bench-Science Team","sourceUrl":"https://www.terminal-bench-science.ai/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"healthBench","benchmarkName":"HealthBench raw score","category":"knowledge","score":56.7,"unit":"percent","family":"healthBench","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaTerminalBench4","benchmarkName":"Artificial Analysis Terminal-Bench v4.0","category":"agentic","score":56.1,"unit":"percent","family":"aaTerminalBench4","index":77.84136178096946,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/terminalbench-v4-0","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:terminal-bench-4:overall","benchmarkName":"Terminal-Bench 4.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":55.051,"unit":"percent","family":"terminal-bench-4","index":80.65635496409516,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-4","measuredAt":"2026-11-09","stderr":1.821,"costPerTest":1.72496,"latency":2711.667},{"benchmarkId":"gdpvalAaNormalized","benchmarkName":"GDPval-AA normalized","category":"agentic","score":54.8,"unit":"percent","family":"gdpvalAaNormalized","index":73.21450145790519,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-agentic-coding:overall","benchmarkName":"LiveBench Agentic Coding","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":54.54533333333333,"unit":"percent","family":"livebench:livebench-agentic-coding:overall","index":65.41436363060029,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Agentic%20Coding","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.40510416666666665,"latency":null},{"benchmarkId":"epoch-ai:epoch-ebr-bench:overall","benchmarkName":"EBR-bench","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":54.28571428571429,"unit":"percent","family":"epoch-ai:epoch-ebr-bench:overall","index":77.33997951151443,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":5.753831415997416,"costPerTest":null,"latency":null},{"benchmarkId":"aaSciCode","benchmarkName":"Artificial Analysis SciCode","category":"coding","score":54.2,"unit":"percent","family":"scicode-family","index":66.4998585781898,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/scicode","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":52.9,"unit":"percent","family":"hle","index":77.4608305253896,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:terminal-bench-science:overall","benchmarkName":"Terminal-Bench-Science","category":null,"domain":"Science","eligible":false,"ranked":true,"score":52.857,"unit":"percent","family":"vals-ai:terminal-bench-science:overall","index":null,"aboveFit":false,"run":"Mini-SWE-agent","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-science","measuredAt":"2026-11-09","stderr":6.009,"costPerTest":3.436948,"latency":5003.987},{"benchmarkId":"arc-prize:v3-semi-private:overall","benchmarkName":"ARC-AGI-3 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":52.72904006820879,"unit":"percent","family":"arc-prize:v3-semi-private:overall","index":87.70819199883135,"aboveFit":false,"run":"max effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:fabv2:overall","benchmarkName":"Finance Agent (v2)","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":52.033,"unit":"percent","family":"vals-ai:fabv2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/fabv2","measuredAt":"2026-10-29","stderr":0.308,"costPerTest":1.619399,"latency":925.46},{"benchmarkId":"vals-ai:srebench:overall","benchmarkName":"SRE Bench","category":null,"domain":"Other","eligible":false,"ranked":true,"score":50.763,"unit":"percent","family":"vals-ai:srebench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/srebench","measuredAt":"2026-11-05","stderr":3.095,"costPerTest":2.685175,"latency":1952.046},{"benchmarkId":"aaAnalystAgent","benchmarkName":"Artificial Analysis AnalystAgent","category":"agentic","score":50,"unit":"percent","family":"aaAnalystAgent","index":72.01108090108372,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/aa-analyst-agent","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:medcode:overall","benchmarkName":"MedCode","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":48.839,"unit":"percent","family":"vals-ai:medcode:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medcode","measuredAt":"2026-10-29","stderr":2.12,"costPerTest":0.084726,"latency":115.342},{"benchmarkId":"mka","benchmarkName":"Multimodal Knowledge Agent Benchmark","category":"agentic","score":47.7,"unit":"percent","family":"mka","index":75.9494002301814,"aboveFit":false,"run":null,"source":"Google DeepMind, in partnership with Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mka","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:mysterymechanism:overall","benchmarkName":"MysteryMechanism","category":null,"domain":"Science","eligible":false,"ranked":true,"score":46.396,"unit":"percent","family":"vals-ai:mysterymechanism:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mysterymechanism","measuredAt":"2026-11-11","stderr":3.355,"costPerTest":0.284881,"latency":437.025},{"benchmarkId":"bug-hunt-bench","benchmarkName":"Bug Hunt Bench","category":"coding","score":44.3,"unit":"percent","family":"bug-hunt-bench","index":79.2651113360876,"aboveFit":false,"run":null,"source":"Pawel Huryn","sourceUrl":"https://bughunt.productcompass.pm/method","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:cyber:overall","benchmarkName":"CyberBench v1.1","category":null,"domain":"Other","eligible":false,"ranked":true,"score":39.286,"unit":"percent","family":"vals-ai:cyber:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/cyber","measuredAt":"2026-11-05","stderr":2.742,"costPerTest":0.285372,"latency":221.992},{"benchmarkId":"frontierCode2","benchmarkName":"FrontierCode 2.0","category":"coding","score":39,"unit":"percent","family":"frontierCode2","index":80.94067281352915,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:legal_research:overall","benchmarkName":"Legal Research Bench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":38.462,"unit":"percent","family":"vals-ai:legal_research:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_research","measuredAt":"2026-11-09","stderr":3.381,"costPerTest":3.384561,"latency":2060.48},{"benchmarkId":"aaMlcr","benchmarkName":"Medical Long Context Reasoning (MLCR-AA)","category":"reasoning","score":38.3,"unit":"percent","family":"aaMlcr","index":82.23309665244882,"aboveFit":false,"run":null,"source":"Wisedocs and Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mlcr-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"healthBenchHard","benchmarkName":"HealthBench Hard","category":"knowledge","score":36.2,"unit":"percent","family":"healthBenchHard","index":76.5466978748475,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"automationBench","benchmarkName":"AutomationBench","category":"agentic","score":36.1,"unit":"percent","family":"automationBench","index":50.756057144390226,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"exploitGym","benchmarkName":"ExploitGym","category":"agentic","score":35.1,"unit":"percent","family":"exploitGym","index":80.28503632766444,"aboveFit":false,"run":null,"source":"Zhun Wang, Nico Schiller, Hongwei Li, Srijiith Sesha Narayana, Milad Nasr, Nicholas Carlini, Xiangyu Qi, Eric Wallace, Elie Bursztein, Luca Invernizzi, Kurt Thomas, Yan Shoshitaishvili, Wenbo Guo, Jingxuan He, Thorsten Holz, Dawn Song","sourceUrl":"https://arxiv.org/abs/2605.11086","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":31.7,"unit":"percent","family":"critpt","index":95,"aboveFit":true,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaGdpPdf","benchmarkName":"Artificial Analysis GDP.pdf","category":"agentic","score":31,"unit":"percent","family":"aaGdpPdf","index":79.012017156606,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdp-pdf","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-spireclimb-slay-the-spire-2-ascension-10:overall","benchmarkName":"SpireClimb (Slay the Spire 2, Ascension 10)","category":null,"domain":"Other","eligible":false,"ranked":true,"score":29.333333333333332,"unit":"percent","family":"epoch-ai:epoch-spireclimb-slay-the-spire-2-ascension-10:overall","index":null,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":5.292637528870838,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-spireclimb:overall","benchmarkName":"SpireClimb","category":null,"domain":"Other","eligible":false,"ranked":true,"score":23.833333333333336,"unit":"percent","family":"epoch-ai:epoch-spireclimb:overall","index":null,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":3.653377199465138,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:tax_agent_bench:overall","benchmarkName":"Tax Agent Bench","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":21.754,"unit":"percent","family":"vals-ai:tax_agent_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench","measuredAt":"2026-11-09","stderr":2.977,"costPerTest":1.323601,"latency":1213.878},{"benchmarkId":"epoch-ai:epoch-spireclimb-slay-the-spire-1-ascension-20-heart:overall","benchmarkName":"SpireClimb (Slay the Spire 1, Ascension 20 Heart)","category":null,"domain":"Other","eligible":false,"ranked":true,"score":18.333333333333332,"unit":"percent","family":"epoch-ai:epoch-spireclimb-slay-the-spire-1-ascension-20-heart:overall","index":null,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":5.0375239785310155,"costPerTest":null,"latency":null},{"benchmarkId":"mazeBench","benchmarkName":"MazeBench: visual spatial reasoning in a 3D open world","category":"agentic","score":17,"unit":"percent","family":"mazeBench","index":92.918394567684,"aboveFit":false,"run":null,"source":"Jonathan Pappas and David Pappas","sourceUrl":"https://mazebench.com/blog?post=maze-bench-results","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lmarena:agent-steerability:overall","benchmarkName":"Agent Arena steerability","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":13.392534307306963,"unit":"index","family":"agent-arena","index":77.541564076628,"aboveFit":false,"run":"max effort","source":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset","measuredAt":"2026-11-11","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"extractBenchPro","benchmarkName":"ExtractBench Pro: long-tail document extraction challenges (LlamaIndex)","category":"multimodal","score":12,"unit":"percent","family":"extractBenchPro","index":null,"aboveFit":false,"run":null,"source":"LlamaIndex","sourceUrl":"https://www.extractbench.ai/pro","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lmarena:agent-task-outcome:overall","benchmarkName":"Agent Arena task outcome","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":11.709384466159744,"unit":"index","family":"agent-arena","index":76.15775103756967,"aboveFit":false,"run":"max effort","source":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset","measuredAt":"2026-11-11","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaHarveyLabV11","benchmarkName":"Harvey LAB-AA v1.1","category":"agentic","score":6.9,"unit":"percent","family":"aaHarveyLabV11","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:hlab:overall","benchmarkName":"HLAB-Vals-1.1","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":6.25,"unit":"percent","family":"vals-ai:hlab:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/hlab","measuredAt":"2026-11-09","stderr":1.83,"costPerTest":3.536302,"latency":2425.625},{"benchmarkId":"lmarena:agent-bash-recovery:overall","benchmarkName":"Agent Arena command recovery","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":4.528409239854958,"unit":"index","family":"agent-arena","index":70.2538636305267,"aboveFit":false,"run":"max effort","source":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset","measuredAt":"2026-10-08","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-erdos:overall","benchmarkName":"FrontierMath-Erdos","category":"math","domain":null,"eligible":true,"ranked":true,"score":4.411764705882353,"unit":"percent","family":"epoch-ai:epoch-frontiermath-erdos:overall","index":null,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.5088285985372343,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:programbench:overall","benchmarkName":"ProgramBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":3,"unit":"percent","family":"vals-ai:programbench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/programbench","measuredAt":"2026-11-09","stderr":1.209,"costPerTest":2.868128,"latency":4223.187},{"benchmarkId":"stemBench120","benchmarkName":"STEMBench120: six-domain scientific research progress","category":"reasoning","score":2.759,"unit":"percent","family":"stemBench120","index":null,"aboveFit":false,"run":null,"source":"Ulam","sourceUrl":"https://www.ulam.ai/stembench","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":2,"outputPerMillion":10,"batch":{"inputPerMillion":1,"outputPerMillion":5},"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":34,"timeToFirstToken":4.558,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":{"elo":1445.9406945198307,"eloUpper":1452.241850048958,"eloLower":1439.6395389907034,"votes":12952,"rank":62,"rankUpper":null,"rankLower":null,"arenaName":"GPT-6.1 Sol-max","organization":"openai","updatedAt":"2026-11-09"},"quality":{"coveragePercent":100,"benchmarkCount":55,"categoryCount":8,"confidence":"high","state":"available"},"hosts":[{"host":"amazon-bedrock","name":"Amazon Bedrock","kind":"cloud","url":"https://docs.aws.amazon.com/bedrock/latest/userguide/models-supported.html","inputPerMillion":2,"outputPerMillion":10,"cacheReadPerMillion":0.1,"contextTokens":1050000,"maxOutputTokens":128000,"deprecated":false},{"host":"digitalocean","name":"DigitalOcean","kind":"cloud","url":"https://docs.digitalocean.com/products/gradient-ai-platform/details/models/","inputPerMillion":2,"outputPerMillion":10,"cacheReadPerMillion":0.1,"contextTokens":1050000,"maxOutputTokens":128000,"deprecated":false},{"host":"azure","name":"Microsoft Azure","kind":"cloud","url":"https://learn.microsoft.com/en-us/azure/ai-services/openai/concepts/models","inputPerMillion":2,"outputPerMillion":10,"cacheReadPerMillion":0.1,"contextTokens":1050000,"maxOutputTokens":128000,"deprecated":false},{"host":"openai","name":"OpenAI","kind":"maker","url":"https://platform.openai.com/docs/models","inputPerMillion":2,"outputPerMillion":10,"cacheReadPerMillion":0.1,"contextTokens":1050000,"maxOutputTokens":128000,"deprecated":false},{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":2,"outputPerMillion":10,"cacheReadPerMillion":0.1,"contextTokens":1050000,"maxOutputTokens":128000,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:livebench","name":"LiveBench","url":"https://livebench.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:arc-prize-foundation","name":"ARC Prize Foundation","url":"https://arcprize.org","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:terminal-bench","name":"Terminal-Bench","url":"https://www.tbench.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:lmarena","name":"LMArena","url":"https://huggingface.co","license":"CC BY 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"muse-wonder-full","providerId":"meta","slug":"muse-wonder-full","name":"Muse Wonder Full","provider":"Meta","description":"Muse Wonder Full is a reasoning model from Meta in the Muse Wonder family. 24 benchmarks count toward its index, in 5 categories.","logoUrl":null,"releaseDate":"2026-10-12","contextWindowTokens":1000000,"modalities":["text"],"openWeights":false,"details":null,"reasoning":true,"overallScore":77.28621216839132,"overallScoreRange":{"low":71.98552473999098,"high":82.58689959679165},"badges":[],"capabilityScores":{"agentic":78.17677392895808,"coding":77.29874691329667,"reasoning":75.59195737280915,"multimodal":null,"knowledge":71.74097124445592,"multilingual":null,"instruction":null,"math":79.60962248388469},"capabilityCounts":{"math":1,"coding":7,"reasoning":3,"agentic":10,"knowledge":3},"benchmarks":[{"benchmarkId":"vals-ai:proof_bench:overall","benchmarkName":"ProofBench v1.1","category":"math","domain":null,"eligible":true,"ranked":true,"score":94,"unit":"percent","family":"vals-ai:proof_bench:overall","index":79.60962248388469,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/proof_bench","measuredAt":"2026-11-09","stderr":2.387,"costPerTest":0.856208,"latency":695.558},{"benchmarkId":"vals-ai:medscribe:overall","benchmarkName":"MedScribe","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":89.335,"unit":"percent","family":"vals-ai:medscribe:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medscribe","measuredAt":"2026-10-29","stderr":2.004,"costPerTest":0.19181,"latency":115.718},{"benchmarkId":"vals-ai:legal_bench:overall","benchmarkName":"LegalBench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":87.418,"unit":"percent","family":"vals-ai:legal_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","measuredAt":"2026-10-13","stderr":0.413,"costPerTest":0.025453,"latency":38.267},{"benchmarkId":"vals-ai:vibe-code:overall","benchmarkName":"Vibe Code Bench v1.1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":85.996,"unit":"percent","family":"vals-ai:vibe-code:overall","index":73.27823048945314,"aboveFit":false,"run":"OpenHands","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","measuredAt":"2026-11-09","stderr":1.933,"costPerTest":2.293448,"latency":648.336},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":83.7,"unit":"percent","family":"lcr","index":64.72298945069046,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:ioi:overall","benchmarkName":"IOI","category":"coding","domain":null,"eligible":true,"ranked":true,"score":79.833,"unit":"percent","family":"vals-ai:ioi:overall","index":76.08773069625744,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/ioi","measuredAt":"2026-10-29","stderr":2.992,"costPerTest":7.078463,"latency":5446.154},{"benchmarkId":"deepSwe","benchmarkName":"DeepSWE","category":"agentic","score":79,"unit":"percent","family":"deepSwe","index":78.50751526838859,"aboveFit":false,"run":null,"source":"Datacurve AI","sourceUrl":"https://deepswe.datacurve.ai/blog","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"osWorld21","benchmarkName":"OSWorld 2.1 (all 108 tasks)","category":"agentic","score":78.4,"unit":"percent","family":"osWorld21","index":76.25550345881204,"aboveFit":false,"run":null,"source":"Mengqi Yuan, Zilong Zhou, Xinzhuang Xiong, Weiming Wu, Jiayang Sun, Jiamin Song, Kaiqian Cui, Bowen Wang, Haoyuan Wu, Yitong Li, Dunjie Lu, Haikong Lu, Qi Zhen, Xinyuan Wang, Jiaqi Deng, Yuhao Yang, Cheng Chen, Boyuan Zheng, Alex Su, Xiao Yu, Hao Zou, Saaket Agashe, Xing Han Lu, Manpreet Kaur, Zhengyang Qi, Vincent Sunn Chen, Frederic Sala, Dayiheng Liu, Junyang Lin, Zhou Yu, Yu Su, Siva Reddy, Xin Eric Wang, Peng Qi, Tianbao Xie, Tao Yu","sourceUrl":"https://github.com/xlang-ai/OSWorld-V2/releases","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"jobBench","benchmarkName":"JobBench","category":"agentic","score":75.3,"unit":"percent","family":"jobBench","index":78.14196190281706,"aboveFit":false,"run":null,"source":"Yuetai Li, Yichen Feng, Zhangchen Xu, Zixian Ma, Kaiyuan Zheng, Fengqing Jiang, Xinghua Sun, Rulin Shao, Zichen Chen, Yue Huang, Xinyang Han, Brian Lee, Kayla Xu, Shenglai Zeng, Hang Hua, Xiangliang Zhang, Basel Alomair, Ranjay Krishna, Luke Zettlemoyer, Pang Wei Koh, Bhaskar Ramasubramanian, Luyao Niu, Xiang Yue, Radha Poovendran","sourceUrl":"https://arxiv.org/abs/2605.26329","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:public-benefits-bench:overall","benchmarkName":"Public Benefits Bench v1.1","category":null,"domain":"Public benefits","eligible":false,"ranked":true,"score":73.748,"unit":"percent","family":"vals-ai:public-benefits-bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/public-benefits-bench","measuredAt":"2026-10-12","stderr":1.145,"costPerTest":4.635817,"latency":4364.892},{"benchmarkId":"vals-ai:cyber:overall","benchmarkName":"CyberBench v1.1","category":null,"domain":"Other","eligible":false,"ranked":true,"score":73.631,"unit":"percent","family":"vals-ai:cyber:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/cyber","measuredAt":"2026-11-05","stderr":5.564,"costPerTest":3.627348,"latency":2126.512},{"benchmarkId":"vals-ai:biomysterybench:overall","benchmarkName":"BioMysteryBench","category":null,"domain":"Science","eligible":false,"ranked":true,"score":73.333,"unit":"percent","family":"vals-ai:biomysterybench:overall","index":null,"aboveFit":false,"run":"Terminus 2","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/biomysterybench","measuredAt":"2026-11-09","stderr":0.642,"costPerTest":3.473154,"latency":1574.165},{"benchmarkId":"vals-ai:emb:overall","benchmarkName":"EMB","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":71.917,"unit":"percent","family":"vals-ai:emb:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/emb","measuredAt":"2026-10-20","stderr":2.658,"costPerTest":4.58223,"latency":1684.504},{"benchmarkId":"gdpvalAaNormalized","benchmarkName":"GDPval-AA normalized","category":"agentic","score":67.9,"unit":"percent","family":"gdpvalAaNormalized","index":82.58069703606819,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaSciCode","benchmarkName":"Artificial Analysis SciCode","category":"coding","score":62.6,"unit":"percent","family":"scicode-family","index":77.45086420807355,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/scicode","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"healthBenchProfessional","benchmarkName":"HealthBench Professional","category":"knowledge","score":62.3,"unit":"percent","family":"healthBenchProfessional","index":74.69933247339172,"aboveFit":false,"run":null,"source":"Rebecca Soskin Hicks, Mikhail Trofimov, Dominick Lim, Rahul K. Arora, Foivos Tsimpourlas, Preston Bowman, Michael Sharman, Chi Tong, Kavin Karthik, Arnav Dugar, Akshay Jagadeesh, Khaled Saab, Johannes Heidecke, Ashley Alexander, Nate Gross, Karan Singhal","sourceUrl":"https://arxiv.org/abs/2604.27470","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:code-migration:overall","benchmarkName":"Code Migration","category":"coding","domain":null,"eligible":true,"ranked":true,"score":61.033,"unit":"percent","family":"vals-ai:code-migration:overall","index":77.61231408351291,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/code-migration","measuredAt":"2026-10-29","stderr":4.419,"costPerTest":38.410556,"latency":13956.531},{"benchmarkId":"vals-ai:fabv2:overall","benchmarkName":"Finance Agent (v2)","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":59.061,"unit":"percent","family":"vals-ai:fabv2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/fabv2","measuredAt":"2026-10-29","stderr":0.215,"costPerTest":2.389758,"latency":895.804},{"benchmarkId":"programBench","benchmarkName":"ProgramBench: Can Language Models Rebuild Programs From Scratch?","category":"coding","score":59,"unit":"percent","family":"programBench","index":77.61231408351291,"aboveFit":false,"run":null,"source":"John Yang, Kilian Lieret, Jeffrey Ma, Parth Thakkar, Dmitrii Pedchenko, Sten Sootla, Emily McMilin, Pengcheng Yin, Rui Hou, Gabriel Synnaeve, Diyi Yang, Ofir Press","sourceUrl":"https://programbench.com/static/paper.pdf","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:terminal-bench-4:overall","benchmarkName":"Terminal-Bench 4.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":58.081,"unit":"percent","family":"terminal-bench-4","index":82.09511601729429,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-4","measuredAt":"2026-11-09","stderr":3.312,"costPerTest":10.528007,"latency":4330.665},{"benchmarkId":"sweMarathon","benchmarkName":"SWE-Marathon","category":"coding","score":57.5,"unit":"percent","family":"sweMarathon","index":null,"aboveFit":false,"run":null,"source":"Abundant AI and BenchFlow","sourceUrl":"https://www.swe-marathon.org/","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaTerminalBench4","benchmarkName":"Artificial Analysis Terminal-Bench v4.0","category":"agentic","score":57.1,"unit":"percent","family":"aaTerminalBench4","index":78.32444524303948,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/terminalbench-v4-0","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":53.6,"unit":"percent","family":"hle","index":78.16313544691046,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:legal_research:overall","benchmarkName":"Legal Research Bench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":53.365,"unit":"percent","family":"vals-ai:legal_research:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_research","measuredAt":"2026-11-09","stderr":3.467,"costPerTest":3.595401,"latency":1654.378},{"benchmarkId":"vals-ai:medcode:overall","benchmarkName":"MedCode","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":49.932,"unit":"percent","family":"vals-ai:medcode:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medcode","measuredAt":"2026-10-29","stderr":2.177,"costPerTest":0.247179,"latency":182.7},{"benchmarkId":"aaAutomationBench","benchmarkName":"Artificial Analysis AutomationBench","category":"agentic","score":45.4,"unit":"percent","family":"aaAutomationBench","index":57.04746913454287,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/automationbench-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:tax_agent_bench:overall","benchmarkName":"Tax Agent Bench","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":42.997,"unit":"percent","family":"vals-ai:tax_agent_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench","measuredAt":"2026-11-09","stderr":3.572,"costPerTest":2.147235,"latency":1084.296},{"benchmarkId":"bug-hunt-bench","benchmarkName":"Bug Hunt Bench","category":"coding","score":42,"unit":"percent","family":"bug-hunt-bench","index":77.78250706844686,"aboveFit":false,"run":null,"source":"Pawel Huryn","sourceUrl":"https://bughunt.productcompass.pm/method","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"omniscienceAccuracy","benchmarkName":"Artificial Analysis Omniscience Accuracy","category":"knowledge","score":41,"unit":"percent","family":"omniscienceAccuracy","index":62.360445813065574,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:terminal-bench-science:overall","benchmarkName":"Terminal-Bench-Science","category":null,"domain":"Science","eligible":false,"ranked":true,"score":37.143,"unit":"percent","family":"vals-ai:terminal-bench-science:overall","index":null,"aboveFit":false,"run":"Mini-SWE-agent","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-science","measuredAt":"2026-11-09","stderr":5.817,"costPerTest":22.814086,"latency":12919.209},{"benchmarkId":"vals-ai:mysterymechanism:overall","benchmarkName":"MysteryMechanism","category":null,"domain":"Science","eligible":false,"ranked":true,"score":35.135,"unit":"percent","family":"vals-ai:mysterymechanism:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mysterymechanism","measuredAt":"2026-11-11","stderr":3.211,"costPerTest":4.163398,"latency":2494.329},{"benchmarkId":"vals-ai:vcb-1-100:overall","benchmarkName":"Vibe Code Bench 1-100","category":"coding","domain":null,"eligible":true,"ranked":true,"score":33.978,"unit":"percent","family":"vals-ai:vcb-1-100:overall","index":81.26726776381987,"aboveFit":false,"run":"OpenHands","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vcb-1-100","measuredAt":"2026-10-29","stderr":5.318,"costPerTest":19.5403,"latency":7008.163},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":32.9,"unit":"percent","family":"critpt","index":95,"aboveFit":true,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaGdpPdf","benchmarkName":"Artificial Analysis GDP.pdf","category":"agentic","score":32.6,"unit":"percent","family":"aaGdpPdf","index":80.85789612745864,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdp-pdf","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:hlab:overall","benchmarkName":"HLAB-Vals-1.1","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":23.333,"unit":"percent","family":"vals-ai:hlab:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/hlab","measuredAt":"2026-11-09","stderr":3.547,"costPerTest":5.413598,"latency":2649.375},{"benchmarkId":"aaMlcr","benchmarkName":"Medical Long Context Reasoning (MLCR-AA)","category":"reasoning","score":22.2,"unit":"percent","family":"aaMlcr","index":67.05288266773698,"aboveFit":false,"run":null,"source":"Wisedocs and Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mlcr-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mazeBench","benchmarkName":"MazeBench: visual spatial reasoning in a 3D open world","category":"agentic","score":17,"unit":"percent","family":"mazeBench","index":92.918394567684,"aboveFit":false,"run":null,"source":"Jonathan Pappas and David Pappas","sourceUrl":"https://mazebench.com/blog?post=maze-bench-results","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaHarveyLabV11","benchmarkName":"Harvey LAB-AA v1.1","category":"agentic","score":13.1,"unit":"percent","family":"aaHarveyLabV11","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:srebench:overall","benchmarkName":"SRE Bench","category":null,"domain":"Other","eligible":false,"ranked":true,"score":11.832,"unit":"percent","family":"vals-ai:srebench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/srebench","measuredAt":"2026-11-05","stderr":1.999,"costPerTest":33.810651,"latency":20035.497},{"benchmarkId":"lmarena:agent-task-outcome:overall","benchmarkName":"Agent Arena task outcome","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":11.138125782574548,"unit":"index","family":"agent-arena","index":75.68808684268035,"aboveFit":false,"run":"max effort","source":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset","measuredAt":"2026-11-11","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lmarena:agent-steerability:overall","benchmarkName":"Agent Arena steerability","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":9.558509018837446,"unit":"index","family":"agent-arena","index":74.38939422427114,"aboveFit":false,"run":"max effort","source":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset","measuredAt":"2026-11-11","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:programbench:overall","benchmarkName":"ProgramBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":0.5,"unit":"percent","family":"vals-ai:programbench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/programbench","measuredAt":"2026-11-09","stderr":0.5,"costPerTest":28.645593,"latency":21822.228}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":80,"benchmarkCount":24,"categoryCount":5,"confidence":"medium","state":"available"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:lmarena","name":"LMArena","url":"https://huggingface.co","license":"CC BY 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"claude-sonnet-5-5","providerId":"anthropic","slug":"claude-sonnet-5-5","name":"Claude Sonnet 5.5","provider":"Anthropic","description":"Claude Sonnet 5.5 is a reasoning model from Anthropic. 59 benchmarks count toward its index, in 7 categories.","logoUrl":null,"releaseDate":"2026-09-28","contextWindowTokens":1000000,"modalities":["text","image","file"],"openWeights":false,"details":{"apiId":"anthropic/claude-sonnet-5.5","maxOutputTokens":128000,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.09999999999999999,"weightsUrl":null},"reasoning":true,"overallScore":77.2414936983622,"overallScoreRange":{"low":74.63633900780228,"high":79.84664838892213},"badges":[],"capabilityScores":{"agentic":77.53262936436089,"coding":80.82542412329218,"reasoning":77.80820538315757,"multimodal":74.36678843197157,"knowledge":71.34412197228609,"multilingual":null,"instruction":41.49291295155261,"math":76.05259606688078},"capabilityCounts":{"math":5,"coding":14,"reasoning":10,"knowledge":6,"multimodal":3,"agentic":20,"instruction":1},"benchmarks":[{"benchmarkId":"vals-ai:proof_bench:overall","benchmarkName":"ProofBench v1.1","category":"math","domain":null,"eligible":true,"ranked":true,"score":100,"unit":"percent","family":"vals-ai:proof_bench:overall","index":81.68109068983412,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/proof_bench","measuredAt":"2026-11-09","stderr":0,"costPerTest":0.524472,"latency":347.583},{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":100,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":67.09849544987334,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-mirrorcode:overall","benchmarkName":"MirrorCode","category":"coding","domain":null,"eligible":true,"ranked":true,"score":97.77777777777777,"unit":"percent","family":"epoch-ai:epoch-mirrorcode:overall","index":92.50682592183128,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.5141892085983264,"costPerTest":null,"latency":null},{"benchmarkId":"arc-prize:v1-public-eval:overall","benchmarkName":"ARC-AGI-1 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":97.75,"unit":"percent","family":"arc-prize:v1-public-eval:overall","index":null,"aboveFit":false,"run":"max effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.299132995,"latency":null},{"benchmarkId":"arc-prize:v1-semi-private:overall","benchmarkName":"ARC-AGI-1 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":96.5,"unit":"percent","family":"arc-prize:v1-semi-private:overall","index":71.35189279722005,"aboveFit":false,"run":"max effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.5019055800000001,"latency":null},{"benchmarkId":"benchCadVision2CodeWithTools","benchmarkName":"BenchCAD Vision2Code voxel IoU with tools","category":"multimodal","score":96.3,"unit":"percent","family":"benchCadVision2CodeWithTools","index":null,"aboveFit":false,"run":null,"source":"Zhang et al. and Anthropic","sourceUrl":"https://arxiv.org/abs/2605.10865","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-math:overall","benchmarkName":"LiveBench Mathematics","category":"math","domain":null,"eligible":true,"ranked":true,"score":96.13175,"unit":"percent","family":"livebench:livebench-math:overall","index":73.75549894501205,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Mathematics","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.16808258575197887,"latency":null},{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":95.58080808080808,"unit":"percent","family":"gpqa-diamond","index":64.87320615631889,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.3710320714521202,"costPerTest":null,"latency":null},{"benchmarkId":"arxivMathAugust2026WithTools","benchmarkName":"ArXivMath August 2026 with tools","category":"math","score":95.2,"unit":"percent","family":"arxivMathAugust2026WithTools","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-coding-2:overall","benchmarkName":"LiveBench Coding 2.0","category":"coding","domain":null,"eligible":true,"ranked":true,"score":94.7,"unit":"percent","family":"livebench:livebench-coding-2:overall","index":76.76550357641594,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Coding","measuredAt":"2026-10-14","stderr":null,"costPerTest":4.021820689655173,"latency":null},{"benchmarkId":"extractBenchLlamaIndex","benchmarkName":"ExtractBench: schema-guided enterprise document extraction (LlamaIndex leaderboard)","category":"multimodal","score":94.68,"unit":"percent","family":"extractBenchLlamaIndex","index":77.7349391092944,"aboveFit":false,"run":null,"source":"Boyang Zhang, Adrian Lyjak, Eli Stewart, Zhaoqi Li and Simon Suo (LlamaIndex)","sourceUrl":"https://arxiv.org/abs/2607.29677","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"legalAgentBenchHeldoutCriterionPass","benchmarkName":"Legal Agent Benchmark mean criterion-pass rate — Harvey held-out set","category":"agentic","score":93.1,"unit":"percent","family":"legalAgentBenchHeldoutCriterionPass","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaHarveyLab","benchmarkName":"Artificial Analysis Harvey LAB-AA v1.0","category":"agentic","score":93.1,"unit":"percent","family":"aaHarveyLab","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:vibe-code:overall","benchmarkName":"Vibe Code Bench v1.1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":92.392,"unit":"percent","family":"vals-ai:vibe-code:overall","index":75.58600777814974,"aboveFit":false,"run":"OpenHands","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","measuredAt":"2026-11-09","stderr":1.26,"costPerTest":19.729112,"latency":3672.106},{"benchmarkId":"gmmlu","benchmarkName":"Global MMLU","category":"multilingual","score":92.1,"unit":"percent","family":"gmmlu","index":null,"aboveFit":false,"run":null,"source":"Singh et al.","sourceUrl":"https://arxiv.org/abs/2412.03304","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-reasoning:overall","benchmarkName":"LiveBench Reasoning","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":91.6345,"unit":"percent","family":"livebench","index":80.32616785531104,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Reasoning","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.20906336633663367,"latency":null},{"benchmarkId":"milu","benchmarkName":"Multi-task Indic Language Understanding Benchmark","category":"multilingual","score":91.6,"unit":"percent","family":"milu","index":null,"aboveFit":false,"run":null,"source":"Verma et al.","sourceUrl":"https://arxiv.org/abs/2411.02538","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-coding:overall","benchmarkName":"LiveBench Coding","category":"coding","domain":null,"eligible":true,"ranked":true,"score":91.366,"unit":"percent","family":"livebench:livebench-coding:overall","index":83.32188662629953,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Coding","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.15368290598290596,"latency":null},{"benchmarkId":"vals-ai:medscribe:overall","benchmarkName":"MedScribe","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":91.101,"unit":"percent","family":"vals-ai:medscribe:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medscribe","measuredAt":"2026-10-29","stderr":1.96,"costPerTest":0.508604,"latency":306.811},{"benchmarkId":"chartographyWithTools","benchmarkName":"Chartography with image and code tools","category":"multimodal","score":90.2,"unit":"percent","family":"chartographyWithTools","index":null,"aboveFit":false,"run":null,"source":"Surge AI and Anthropic","sourceUrl":"https://surgehq.ai/blog/chartography","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"arc-prize:v2-public-eval:overall","benchmarkName":"ARC-AGI-2 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":90.13888888888889,"unit":"percent","family":"arc-prize:v2-public-eval:overall","index":null,"aboveFit":false,"run":"xhigh effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.4597384166666664,"latency":null},{"benchmarkId":"bioMysteryBenchHumanSolvable","benchmarkName":"BioMysteryBench Human Solvable","category":"knowledge","score":89.2,"unit":"percent","family":"bioMysteryBenchHumanSolvable","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tiers-1-3-v2-private:overall","benchmarkName":"FrontierMath-Tiers-1-3-v2-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":88.7719298245614,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tiers-1-3-v2-private:overall","index":77.6199534837705,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.8734030360524248,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:sage:overall","benchmarkName":"SAGE","category":null,"domain":"Education","eligible":false,"ranked":true,"score":88.028,"unit":"percent","family":"vals-ai:sage:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/sage","measuredAt":"2026-11-05","stderr":1.621,"costPerTest":0.160255,"latency":81.122},{"benchmarkId":"draco","benchmarkName":"Data Research and Analysis with Complex Operations","category":"agentic","score":87,"unit":"percent","family":"draco","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"arxivMathAugust2026","benchmarkName":"ArXivMath August 2026 without tools","category":"math","score":86.8,"unit":"percent","family":"arxivMathAugust2026","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"toolathlonVerifiedPass3","benchmarkName":"Toolathlon Verified Pass@3","category":"agentic","score":85.2,"unit":"percent","family":"toolathlonVerifiedPass3","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-design:overall","benchmarkName":"LiveBench Design","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":85,"unit":"percent","family":"livebench-2-agentic","index":80.75261465060905,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Design","measuredAt":"2026-10-14","stderr":null,"costPerTest":5.016608,"latency":null},{"benchmarkId":"livebench:livebench-browser:overall","benchmarkName":"LiveBench Browser Use","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":84,"unit":"percent","family":"livebench-2-agentic","index":80.02128355568858,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Browser","measuredAt":"2026-10-14","stderr":null,"costPerTest":7.658836,"latency":null},{"benchmarkId":"osWorld21Offline","benchmarkName":"OSWorld 2.1 offline subset (82 tasks)","category":"agentic","score":83.9,"unit":"percent","family":"osWorld21Offline","index":82.70087738001654,"aboveFit":false,"run":null,"source":"Mengqi Yuan, Zilong Zhou, Xinzhuang Xiong, Weiming Wu, Jiayang Sun, Jiamin Song, Kaiqian Cui, Bowen Wang, Haoyuan Wu, Yitong Li, Dunjie Lu, Haikong Lu, Qi Zhen, Xinyuan Wang, Jiaqi Deng, Yuhao Yang, Cheng Chen, Boyuan Zheng, Alex Su, Xiao Yu, Hao Zou, Saaket Agashe, Xing Han Lu, Manpreet Kaur, Zhengyang Qi, Vincent Sunn Chen, Frederic Sala, Dayiheng Liu, Junyang Lin, Zhou Yu, Yu Su, Siva Reddy, Xin Eric Wang, Peng Qi, Tianbao Xie, Tao Yu","sourceUrl":"https://github.com/xlang-ai/OSWorld-V2/releases","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"arc-prize:v2-semi-private:overall","benchmarkName":"ARC-AGI-2 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":83.88888888888889,"unit":"percent","family":"arc-prize:v2-semi-private:overall","index":77.70481531746452,"aboveFit":false,"run":"xhigh effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.5461994833333332,"latency":null},{"benchmarkId":"livebench:livebench-research:overall","benchmarkName":"LiveBench Research","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":83.6,"unit":"percent","family":"livebench-2-agentic","index":79.7287511177204,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Research","measuredAt":"2026-10-14","stderr":null,"costPerTest":8.185618181818182,"latency":null},{"benchmarkId":"vals-ai:terminal-bench-2-1:overall","benchmarkName":"Terminal-Bench 2.1","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":83.146,"unit":"percent","family":"terminal-bench-2-1","index":69.76950534182521,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2-1","measuredAt":"2026-10-01","stderr":1.716,"costPerTest":0.625494,"latency":465.828},{"benchmarkId":"vals-ai:ioi:overall","benchmarkName":"IOI","category":"coding","domain":null,"eligible":true,"ranked":true,"score":83.056,"unit":"percent","family":"vals-ai:ioi:overall","index":77.29771709403241,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/ioi","measuredAt":"2026-10-29","stderr":3.737,"costPerTest":5.805193,"latency":2751.084},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":82.7,"unit":"percent","family":"lcr","index":64.0369089251048,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"proteinBinderDesign","benchmarkName":"Anthropic de novo protein-binder design evaluation","category":"knowledge","score":82.3,"unit":"percent","family":"proteinBinderDesign","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"swePro","benchmarkName":"SWE-bench Pro","category":"coding","score":81.3,"unit":"percent","family":"swePro","index":78.07527514449653,"aboveFit":false,"run":null,"source":"Xiang Deng, Jeff Da, Edwin Pan, Yannis Yiming He, Charles Ide, Kanak Garg, Niklas Lauffer, Andrew Park, Nitin Pasari, Chetan Rane, Karmini Sampath, Maya Krishnan, Srivatsa Kundurthy, Sean Hendryx, Zifan Wang, Chen Bo Calvin Zhang, Noah Jacobson, Bing Liu, Brad Kenstler","sourceUrl":"https://arxiv.org/abs/2509.16941","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:biomysterybench:overall","benchmarkName":"BioMysteryBench","category":null,"domain":"Science","eligible":false,"ranked":true,"score":81.111,"unit":"percent","family":"vals-ai:biomysterybench:overall","index":null,"aboveFit":false,"run":"Terminus 2","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/biomysterybench","measuredAt":"2026-11-09","stderr":0.642,"costPerTest":2.945242,"latency":1086.061},{"benchmarkId":"epoch-ai:epoch-frontiermath-tier-4-v2-private:overall","benchmarkName":"FrontierMath-Tier-4-v2-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":80.48780487804879,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tier-4-v2-private:overall","index":80.10794176591392,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":6.265967111543964,"costPerTest":null,"latency":null},{"benchmarkId":"osWorld21","benchmarkName":"OSWorld 2.1 (all 108 tasks)","category":"agentic","score":80.1,"unit":"percent","family":"osWorld21","index":77.59216876803953,"aboveFit":false,"run":null,"source":"Mengqi Yuan, Zilong Zhou, Xinzhuang Xiong, Weiming Wu, Jiayang Sun, Jiamin Song, Kaiqian Cui, Bowen Wang, Haoyuan Wu, Yitong Li, Dunjie Lu, Haikong Lu, Qi Zhen, Xinyuan Wang, Jiaqi Deng, Yuhao Yang, Cheng Chen, Boyuan Zheng, Alex Su, Xiao Yu, Hao Zou, Saaket Agashe, Xing Han Lu, Manpreet Kaur, Zhengyang Qi, Vincent Sunn Chen, Frederic Sala, Dayiheng Liu, Junyang Lin, Zhou Yu, Yu Su, Siva Reddy, Xin Eric Wang, Peng Qi, Tianbao Xie, Tao Yu","sourceUrl":"https://github.com/xlang-ai/OSWorld-V2/releases","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"programBench","benchmarkName":"ProgramBench: Can Language Models Rebuild Programs From Scratch?","category":"coding","score":79.7,"unit":"percent","family":"programBench","index":91.0381659582614,"aboveFit":false,"run":null,"source":"John Yang, Kilian Lieret, Jeffrey Ma, Parth Thakkar, Dmitrii Pedchenko, Sten Sootla, Emily McMilin, Pengcheng Yin, Rui Hou, Gabriel Synnaeve, Diyi Yang, Ofir Press","sourceUrl":"https://programbench.com/static/paper.pdf","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-long-horizon:overall","benchmarkName":"LiveBench Long Agentic","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":78.8,"unit":"percent","family":"livebench-2-agentic","index":76.21836186210221,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Long%20Agentic","measuredAt":"2026-10-14","stderr":null,"costPerTest":35.0115,"latency":null},{"benchmarkId":"livebench:livebench-language:overall","benchmarkName":"LiveBench Language","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":77.984,"unit":"percent","family":"livebench:livebench-language:overall","index":62.743798889226774,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Language","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.31554800000000005,"latency":null},{"benchmarkId":"toolathlonVerified","benchmarkName":"Toolathlon-Verified","category":"agentic","score":77.8,"unit":"percent","family":"toolathlonVerified","index":71.6099643123515,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"healthBenchProfessionalRaw","benchmarkName":"HealthBench Professional raw score","category":"knowledge","score":77.1,"unit":"percent","family":"healthBenchProfessionalRaw","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"officeQa","benchmarkName":"OfficeQA","category":"multimodal","score":76.9,"unit":"percent","family":"officeQa","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:emb:overall","benchmarkName":"EMB","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":75.713,"unit":"percent","family":"vals-ai:emb:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/emb","measuredAt":"2026-10-20","stderr":2.437,"costPerTest":6.526758,"latency":2204.991},{"benchmarkId":"epoch-ai:epoch-furniture-assembly:overall","benchmarkName":"Furniture Assembly","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":75,"unit":"percent","family":"epoch-ai:epoch-furniture-assembly:overall","index":81.62675044841973,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":5.510647116121488,"costPerTest":null,"latency":null},{"benchmarkId":"benchCadVision2Code","benchmarkName":"BenchCAD Vision2Code voxel IoU without tools","category":"multimodal","score":74.7,"unit":"percent","family":"benchCadVision2Code","index":null,"aboveFit":false,"run":null,"source":"Zhang et al. and Anthropic","sourceUrl":"https://arxiv.org/abs/2605.10865","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hermesBench","benchmarkName":"Hermes Bench: 150 agent tasks graded on the files and state the agent leaves behind","category":"agentic","score":73.46,"unit":"percent","family":"hermesBench","index":null,"aboveFit":false,"run":null,"source":"Nous Research","sourceUrl":"https://portal.nousresearch.com/bench","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"spatialBenchVerified","benchmarkName":"LatchBio SpatialBench Verified","category":"knowledge","score":72.5,"unit":"percent","family":"spatialBenchVerified","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"biomedicalImageAnalysis","benchmarkName":"Anthropic biomedical-image-analysis evaluation","category":"multimodal","score":72.2,"unit":"percent","family":"biomedicalImageAnalysis","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaAutomationBench","benchmarkName":"Artificial Analysis AutomationBench","category":"agentic","score":71.8,"unit":"percent","family":"aaAutomationBench","index":81.1415860757478,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/automationbench-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"deepSwe","benchmarkName":"DeepSWE","category":"agentic","score":71,"unit":"percent","family":"deepSwe","index":72.69173859643695,"aboveFit":false,"run":null,"source":"Datacurve AI","sourceUrl":"https://deepswe.datacurve.ai/blog","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaHealthBenchProfessionalRaw","benchmarkName":"Artificial Analysis HealthBench Professional raw score","category":"knowledge","score":70.8,"unit":"percent","family":"aaHealthBenchProfessionalRaw","index":null,"aboveFit":false,"run":null,"source":"OpenAI (benchmark); Artificial Analysis (evaluation)","sourceUrl":"https://artificialanalysis.ai/evaluations/healthbench-professional","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:code-migration:overall","benchmarkName":"Code Migration","category":"coding","domain":null,"eligible":true,"ranked":true,"score":69.825,"unit":"percent","family":"vals-ai:code-migration:overall","index":82.28297698733569,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/code-migration","measuredAt":"2026-10-29","stderr":4.262,"costPerTest":46.395198,"latency":12004.483},{"benchmarkId":"healthBench","benchmarkName":"HealthBench raw score","category":"knowledge","score":69.4,"unit":"percent","family":"healthBench","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"parseBenchLlamaIndex","benchmarkName":"ParseBench: document parsing for AI agents (LlamaIndex leaderboard)","category":"multimodal","score":69.21,"unit":"percent","family":"parseBenchLlamaIndex","index":70.28446787057507,"aboveFit":false,"run":null,"source":"Boyang Zhang, Sebastián G. Acosta, Preston Carlson, Sacha Bron, Pierre-Loïc Doulcet, Daniel B. Ospina and Simon Suo (LlamaIndex)","sourceUrl":"https://arxiv.org/abs/2604.08538","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"healthBenchProfessional","benchmarkName":"HealthBench Professional","category":"knowledge","score":69.2,"unit":"percent","family":"healthBenchProfessional","index":81.67606249559603,"aboveFit":false,"run":null,"source":"Rebecca Soskin Hicks, Mikhail Trofimov, Dominick Lim, Rahul K. Arora, Foivos Tsimpourlas, Preston Bowman, Michael Sharman, Chi Tong, Kavin Karthik, Arnav Dugar, Akshay Jagadeesh, Khaled Saab, Johannes Heidecke, Ashley Alexander, Nate Gross, Karan Singhal","sourceUrl":"https://arxiv.org/abs/2604.27470","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"toolathlonVerifiedPass3All","benchmarkName":"Toolathlon Verified Pass cubed","category":"agentic","score":68.5,"unit":"percent","family":"toolathlonVerifiedPass3All","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"protocolsTroubleshooting","benchmarkName":"Molecular Biology Protocols Troubleshooting","category":"knowledge","score":67.3,"unit":"percent","family":"protocolsTroubleshooting","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:public-benefits-bench:overall","benchmarkName":"Public Benefits Bench v1.1","category":null,"domain":"Public benefits","eligible":false,"ranked":true,"score":67.185,"unit":"percent","family":"vals-ai:public-benefits-bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/public-benefits-bench","measuredAt":"2026-10-12","stderr":1.221,"costPerTest":3.763542,"latency":4527.815},{"benchmarkId":"gdpvalAaNormalized","benchmarkName":"GDPval-AA normalized","category":"agentic","score":66.9,"unit":"percent","family":"gdpvalAaNormalized","index":81.86572027437634,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"protocolsUnderstanding","benchmarkName":"Benchling Molecular Biology Protocols Understanding","category":"knowledge","score":66.6,"unit":"percent","family":"protocolsUnderstanding","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaHealthBench","benchmarkName":"Artificial Analysis HealthBench raw score","category":"knowledge","score":65.9,"unit":"percent","family":"aaHealthBench","index":null,"aboveFit":false,"run":null,"source":"OpenAI (benchmark); Artificial Analysis (evaluation)","sourceUrl":"https://artificialanalysis.ai/evaluations/healthbench","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"officeQaPro","benchmarkName":"OfficeQA Pro","category":"multimodal","score":65.6,"unit":"percent","family":"officeQaPro","index":75.08095831604524,"aboveFit":false,"run":null,"source":"OfficeQA Pro authors","sourceUrl":"https://arxiv.org/abs/2603.08655","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"healthBenchLengthAdjusted","benchmarkName":"HealthBench length-adjusted score","category":"knowledge","score":65.4,"unit":"percent","family":"healthBenchLengthAdjusted","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"medicinalChemistry","benchmarkName":"Anthropic medicinal-chemistry evaluation","category":"knowledge","score":65.3,"unit":"percent","family":"medicinalChemistry","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-mystery-game-puzzles:overall","benchmarkName":"Mystery Game Puzzles","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":65,"unit":"percent","family":"epoch-ai:epoch-mystery-game-puzzles:overall","index":89.01796648790636,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":4.793724854411022,"costPerTest":null,"latency":null},{"benchmarkId":"hleWithTools","benchmarkName":"Humanity's Last Exam with tools","category":"agentic","score":64.5,"unit":"percent","family":"hleWithTools","index":72.39674725518161,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:terminal-bench-4:overall","benchmarkName":"Terminal-Bench 4.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":64.141,"unit":"percent","family":"terminal-bench-4","index":84.97263812369255,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-4","measuredAt":"2026-11-09","stderr":1.01,"costPerTest":10.752753,"latency":4928.927},{"benchmarkId":"terminal-bench:terminal-bench-4-1-0:overall","benchmarkName":"Terminal-Bench 4.1.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":63.94,"unit":"percent","family":"terminal-bench:terminal-bench-4-1-0:overall","index":82.77738550054345,"aboveFit":false,"run":"max effort · Claude Code","source":"Terminal-Bench","sourceUrl":"https://www.tbench.ai/leaderboard","measuredAt":"2026-09-29","stderr":1.530612244897959,"costPerTest":22.223151515151518,"latency":7431.6},{"benchmarkId":"aaTerminalBench4","benchmarkName":"Artificial Analysis Terminal-Bench v4.0","category":"agentic","score":63.6,"unit":"percent","family":"aaTerminalBench4","index":81.46448774649458,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/terminalbench-v4-0","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"physicianBench","benchmarkName":"PhysicianBench","category":"knowledge","score":63.2,"unit":"percent","family":"physicianBench","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaHealthBenchProfessional","benchmarkName":"Artificial Analysis HealthBench Professional length-adjusted score","category":"knowledge","score":63,"unit":"percent","family":"aaHealthBenchProfessional","index":null,"aboveFit":false,"run":null,"source":"OpenAI (benchmark); Artificial Analysis (evaluation)","sourceUrl":"https://artificialanalysis.ai/evaluations/healthbench-professional","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaHealthBenchLengthAdjusted","benchmarkName":"Artificial Analysis HealthBench length-adjusted score","category":"knowledge","score":62.1,"unit":"percent","family":"aaHealthBenchLengthAdjusted","index":null,"aboveFit":false,"run":null,"source":"OpenAI (benchmark); Artificial Analysis (evaluation)","sourceUrl":"https://artificialanalysis.ai/evaluations/healthbench","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"frontierSweV2","benchmarkName":"FrontierSWE v2","category":"coding","score":61.9,"unit":"percent","family":"frontierSweV2","index":83.06430037836846,"aboveFit":false,"run":null,"source":"Proximal","sourceUrl":"https://www.frontierswe.com/blog/v2","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"terminal-bench:terminal-bench-4-0-0:overall","benchmarkName":"Terminal-Bench 4.0.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":61.82,"unit":"percent","family":"terminal-bench-4","index":83.87053766016939,"aboveFit":false,"run":"max effort · Claude Code","source":"Terminal-Bench","sourceUrl":"https://www.tbench.ai/leaderboard","measuredAt":"2026-09-29","stderr":1.5,"costPerTest":22.223151515151518,"latency":7431.6},{"benchmarkId":"chartography","benchmarkName":"Chartography without tools","category":"multimodal","score":61.6,"unit":"percent","family":"chartography","index":null,"aboveFit":false,"run":null,"source":"Surge AI and Anthropic","sourceUrl":"https://surgehq.ai/blog/chartography","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaSciCode","benchmarkName":"Artificial Analysis SciCode","category":"coding","score":61,"unit":"percent","family":"scicode-family","index":75.36495837380998,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/scicode","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"terminalBenchScience","benchmarkName":"Terminal-Bench-Science 0.1","category":"agentic","score":59.9,"unit":"percent","family":"terminalBenchScience","index":78.73672439693816,"aboveFit":false,"run":null,"source":"Terminal-Bench-Science Team","sourceUrl":"https://www.terminal-bench-science.ai/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:cyber:overall","benchmarkName":"CyberBench v1.1","category":null,"domain":"Other","eligible":false,"ranked":true,"score":59.584,"unit":"percent","family":"vals-ai:cyber:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/cyber","measuredAt":"2026-11-05","stderr":5.212,"costPerTest":2.910647,"latency":2032.625},{"benchmarkId":"livebench:livebench-data-analysis:overall","benchmarkName":"LiveBench Data Analysis","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":59.48766666666668,"unit":"percent","family":"livebench","index":38.68206708685372,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Data%20Analysis","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.5306346666666666,"latency":null},{"benchmarkId":"frontierCode11Extended","benchmarkName":"FrontierCode 1.1 Extended","category":"coding","score":59.1,"unit":"percent","family":"frontierCode11Extended","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"singleCellBench","benchmarkName":"LatchBio SingleCellBench","category":"knowledge","score":59.1,"unit":"percent","family":"singleCellBench","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-chess-puzzles:overall","benchmarkName":"Chess Puzzles","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":59,"unit":"percent","family":"epoch-ai:epoch-chess-puzzles:overall","index":78.76657223899463,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":4.943110704237103,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:fabv2:overall","benchmarkName":"Finance Agent (v2)","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":58.103,"unit":"percent","family":"vals-ai:fabv2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/fabv2","measuredAt":"2026-10-29","stderr":0.673,"costPerTest":6.39361,"latency":1903.336},{"benchmarkId":"aaAnalystAgent","benchmarkName":"Artificial Analysis AnalystAgent","category":"agentic","score":57.5,"unit":"percent","family":"aaAnalystAgent","index":80.48680917642174,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/aa-analyst-agent","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hleNoTools","benchmarkName":"Humanity's Last Exam without tools","category":"knowledge","score":56.9,"unit":"percent","family":"hle","index":81.47400150550877,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-instruction-following:overall","benchmarkName":"LiveBench Instruction Following","category":"instruction","domain":null,"eligible":true,"ranked":true,"score":56.78775,"unit":"percent","family":"livebench:livebench-instruction-following:overall","index":41.49291295155261,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=IF","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.9137304999999999,"latency":null},{"benchmarkId":"livebench:livebench-agentic-coding:overall","benchmarkName":"LiveBench Agentic Coding","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":56.31333333333333,"unit":"percent","family":"livebench:livebench-agentic-coding:overall","index":67.23767191858906,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Agentic%20Coding","measuredAt":"2026-06-25","stderr":null,"costPerTest":5.566768055555555,"latency":null},{"benchmarkId":"cursorBench40","benchmarkName":"cursorBench40","category":"coding","score":55.5,"unit":"percent","family":"cursorBench40","index":79.44545186638453,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":55,"unit":"percent","family":"hle","index":79.56774528995217,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"proteinDesignLibraryRanking","benchmarkName":"Anthropic protein-design library-ranking task","category":"knowledge","score":54.8,"unit":"percent","family":"proteinDesignLibraryRanking","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hermesIndex","benchmarkName":"Hermes Index: agentic capability and cost per task inside Hermes Agent","category":"agentic","score":53.14,"unit":"percent","family":"hermesIndex","index":77.78749375369213,"aboveFit":false,"run":null,"source":"Nous Research","sourceUrl":"https://portal.nousresearch.com/bench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:medcode:overall","benchmarkName":"MedCode","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":52.92,"unit":"percent","family":"vals-ai:medcode:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medcode","measuredAt":"2026-10-29","stderr":2.119,"costPerTest":0.391592,"latency":239.171},{"benchmarkId":"mka","benchmarkName":"Multimodal Knowledge Agent Benchmark","category":"agentic","score":52,"unit":"percent","family":"mka","index":79.65352791500243,"aboveFit":false,"run":null,"source":"Google DeepMind, in partnership with Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mka","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"omniscienceAccuracy","benchmarkName":"Artificial Analysis Omniscience Accuracy","category":"knowledge","score":51.5,"unit":"percent","family":"omniscienceAccuracy","index":75.5931336456907,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"bug-hunt-bench","benchmarkName":"Bug Hunt Bench","category":"coding","score":51.3,"unit":"percent","family":"bug-hunt-bench","index":83.77738519412462,"aboveFit":false,"run":null,"source":"Pawel Huryn","sourceUrl":"https://bughunt.productcompass.pm/method","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"proteinDesign","benchmarkName":"Anthropic Protein Design evaluation","category":"knowledge","score":51,"unit":"percent","family":"proteinDesign","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:mysterymechanism:overall","benchmarkName":"MysteryMechanism","category":null,"domain":"Science","eligible":false,"ranked":true,"score":49.099,"unit":"percent","family":"vals-ai:mysterymechanism:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mysterymechanism","measuredAt":"2026-11-11","stderr":3.363,"costPerTest":2.984657,"latency":1410.236},{"benchmarkId":"vals-ai:legal_research:overall","benchmarkName":"Legal Research Bench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":48.077,"unit":"percent","family":"vals-ai:legal_research:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_research","measuredAt":"2026-11-09","stderr":3.473,"costPerTest":13.663179,"latency":5588.989},{"benchmarkId":"vals-ai:terminal-bench-science:overall","benchmarkName":"Terminal-Bench-Science","category":null,"domain":"Science","eligible":false,"ranked":true,"score":47.143,"unit":"percent","family":"vals-ai:terminal-bench-science:overall","index":null,"aboveFit":false,"run":"Mini-SWE-agent","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-science","measuredAt":"2026-11-09","stderr":6.009,"costPerTest":19.568918,"latency":9980.148},{"benchmarkId":"epoch-ai:epoch-simpleqa-verified:overall","benchmarkName":"SimpleQA Verified","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":46.5,"unit":"percent","family":"epoch-ai:epoch-simpleqa-verified:overall","index":62.657657249153644,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.5780495050030086,"costPerTest":null,"latency":null},{"benchmarkId":"frontierCode","benchmarkName":"FrontierCode 1.1 Main","category":"coding","score":46.2,"unit":"percent","family":"frontierCode","index":70.60608621095703,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"automationBenchZapier106","benchmarkName":"AutomationBench, Zapier private evaluation set release 1.0.6","category":"agentic","score":44.7,"unit":"percent","family":"automationBenchZapier106","index":null,"aboveFit":false,"run":null,"source":"Daniel Shepard and Robin Salimans","sourceUrl":"https://arxiv.org/abs/2604.18934","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"bioMysteryBenchHumanDifficult","benchmarkName":"BioMysteryBench Human Difficult","category":"knowledge","score":44.7,"unit":"percent","family":"bioMysteryBenchHumanDifficult","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:tax_agent_bench:overall","benchmarkName":"Tax Agent Bench","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":41.959,"unit":"percent","family":"vals-ai:tax_agent_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench","measuredAt":"2026-11-09","stderr":3.561,"costPerTest":8.730268,"latency":3560.542},{"benchmarkId":"aaMlcr","benchmarkName":"Medical Long Context Reasoning (MLCR-AA)","category":"reasoning","score":41.7,"unit":"percent","family":"aaMlcr","index":85.43885612748115,"aboveFit":false,"run":null,"source":"Wisedocs and Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mlcr-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"frontierCode2","benchmarkName":"FrontierCode 2.0","category":"coding","score":41,"unit":"percent","family":"frontierCode2","index":82.42339661562346,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"extractBenchPro","benchmarkName":"ExtractBench Pro: long-tail document extraction challenges (LlamaIndex)","category":"multimodal","score":32,"unit":"percent","family":"extractBenchPro","index":null,"aboveFit":false,"run":null,"source":"LlamaIndex","sourceUrl":"https://www.extractbench.ai/pro","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"toolathlonVerifiedAvgTurns","benchmarkName":"Toolathlon Verified average assistant turns","category":"agentic","score":31.6,"unit":"percent","family":"toolathlonVerifiedAvgTurns","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":31.4,"unit":"percent","family":"critpt","index":95,"aboveFit":true,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:srebench:overall","benchmarkName":"SRE Bench","category":null,"domain":"Other","eligible":false,"ranked":true,"score":30.153,"unit":"percent","family":"vals-ai:srebench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/srebench","measuredAt":"2026-11-05","stderr":2.841,"costPerTest":17.274362,"latency":6881.468},{"benchmarkId":"aaGdpPdf","benchmarkName":"Artificial Analysis GDP.pdf","category":"agentic","score":25.8,"unit":"percent","family":"aaGdpPdf","index":73.01291050133496,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdp-pdf","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"morphologyToMolecule","benchmarkName":"Axiom Bio morphology-to-molecule matching","category":"knowledge","score":25,"unit":"percent","family":"morphologyToMolecule","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"arc-prize:v3-semi-private:overall","benchmarkName":"ARC-AGI-3 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":24.310408006681435,"unit":"percent","family":"arc-prize:v3-semi-private:overall","index":75.63417401790217,"aboveFit":false,"run":"xhigh effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lmarena:agent-task-outcome:overall","benchmarkName":"Agent Arena task outcome","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":19.504243849067215,"unit":"index","family":"agent-arena","index":82.56634746077434,"aboveFit":false,"run":"max effort","source":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset","measuredAt":"2026-11-11","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lmarena:agent-bash-recovery:overall","benchmarkName":"Agent Arena command recovery","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":15.035238150522567,"unit":"index","family":"agent-arena","index":78.89212416461376,"aboveFit":false,"run":"max effort","source":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset","measuredAt":"2026-10-08","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lmarena:agent-steerability:overall","benchmarkName":"Agent Arena steerability","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":14.525636783574164,"unit":"index","family":"agent-arena","index":78.47315195009928,"aboveFit":false,"run":"max effort","source":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset","measuredAt":"2026-11-11","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:hlab:overall","benchmarkName":"HLAB-Vals-1.1","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":10.417,"unit":"percent","family":"vals-ai:hlab:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/hlab","measuredAt":"2026-11-09","stderr":2.572,"costPerTest":17.713451,"latency":4397.953},{"benchmarkId":"legalAgentBenchHeldoutAllPass","benchmarkName":"Legal Agent Benchmark all-pass rate — Harvey held-out set","category":"agentic","score":10,"unit":"percent","family":"legalAgentBenchHeldoutAllPass","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mazeBench","benchmarkName":"MazeBench: visual spatial reasoning in a 3D open world","category":"agentic","score":8,"unit":"percent","family":"mazeBench","index":76.53853898097661,"aboveFit":false,"run":null,"source":"Jonathan Pappas and David Pappas","sourceUrl":"https://mazebench.com/blog?post=maze-bench-results","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:programbench:overall","benchmarkName":"ProgramBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":6.5,"unit":"percent","family":"vals-ai:programbench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/programbench","measuredAt":"2026-11-09","stderr":1.748,"costPerTest":22.620087,"latency":7602.078},{"benchmarkId":"epoch-ai:epoch-frontiermath-erdos:overall","benchmarkName":"FrontierMath-Erdos","category":"math","domain":null,"eligible":true,"ranked":true,"score":2.9000000000000004,"unit":"percent","family":"epoch-ai:epoch-frontiermath-erdos:overall","index":null,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.1,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-spireclimb-slay-the-spire-2-ascension-10:overall","benchmarkName":"SpireClimb (Slay the Spire 2, Ascension 10)","category":null,"domain":"Other","eligible":false,"ranked":true,"score":2.666666666666667,"unit":"percent","family":"epoch-ai:epoch-spireclimb-slay-the-spire-2-ascension-10:overall","index":null,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.8728340843511757,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-spireclimb:overall","benchmarkName":"SpireClimb","category":null,"domain":"Other","eligible":false,"ranked":true,"score":1.3333333333333335,"unit":"percent","family":"epoch-ai:epoch-spireclimb:overall","index":null,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0.9364170421755879,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-spireclimb-slay-the-spire-1-ascension-20-heart:overall","benchmarkName":"SpireClimb (Slay the Spire 1, Ascension 20 Heart)","category":null,"domain":"Other","eligible":false,"ranked":true,"score":0,"unit":"percent","family":"epoch-ai:epoch-spireclimb-slay-the-spire-1-ascension-20-heart:overall","index":null,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":2,"outputPerMillion":10,"batch":{"inputPerMillion":1,"outputPerMillion":5},"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":107,"timeToFirstToken":1.613,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":{"elo":1468.8173447748545,"eloUpper":1474.9436402480587,"eloLower":1462.6910493016503,"votes":13821,"rank":34,"rankUpper":null,"rankLower":null,"arenaName":"Claude Sonnet 5.5-xhigh","organization":"anthropic","updatedAt":"2026-11-09"},"quality":{"coveragePercent":95,"benchmarkCount":59,"categoryCount":7,"confidence":"high","state":"available"},"hosts":[{"host":"amazon-bedrock","name":"Amazon Bedrock","kind":"cloud","url":"https://docs.aws.amazon.com/bedrock/latest/userguide/models-supported.html","inputPerMillion":2,"outputPerMillion":10,"cacheReadPerMillion":0.1,"contextTokens":1000000,"maxOutputTokens":128000,"deprecated":false},{"host":"anthropic","name":"Anthropic","kind":"maker","url":"https://docs.anthropic.com/en/docs/about-claude/models","inputPerMillion":2,"outputPerMillion":10,"cacheReadPerMillion":0.1,"contextTokens":1000000,"maxOutputTokens":128000,"deprecated":false},{"host":"digitalocean","name":"DigitalOcean","kind":"cloud","url":"https://docs.digitalocean.com/products/gradient-ai-platform/details/models/","inputPerMillion":2,"outputPerMillion":10,"cacheReadPerMillion":0.2,"contextTokens":1000000,"maxOutputTokens":128000,"deprecated":false},{"host":"google-vertex","name":"Google Vertex AI","kind":"cloud","url":"https://cloud.google.com/vertex-ai/generative-ai/docs/models","inputPerMillion":2,"outputPerMillion":10,"cacheReadPerMillion":0.1,"contextTokens":1000000,"maxOutputTokens":128000,"deprecated":false},{"host":"azure","name":"Microsoft Azure","kind":"cloud","url":"https://learn.microsoft.com/en-us/azure/ai-services/openai/concepts/models","inputPerMillion":2,"outputPerMillion":10,"cacheReadPerMillion":0.2,"contextTokens":1000000,"maxOutputTokens":128000,"deprecated":false},{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":2,"outputPerMillion":10,"cacheReadPerMillion":0.1,"contextTokens":1000000,"maxOutputTokens":128000,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:arc-prize-foundation","name":"ARC Prize Foundation","url":"https://arcprize.org","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:livebench","name":"LiveBench","url":"https://livebench.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:terminal-bench","name":"Terminal-Bench","url":"https://www.tbench.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:lmarena","name":"LMArena","url":"https://huggingface.co","license":"CC BY 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gpt-6-1-terra","providerId":"openai","slug":"gpt-6-1-terra","name":"GPT-6.1 Terra","provider":"OpenAI","description":"GPT-6.1 Terra is a reasoning model from OpenAI in the GPT-6 family. 34 benchmarks count toward its index, in 6 categories.","logoUrl":null,"releaseDate":"2026-10-26","contextWindowTokens":1050000,"modalities":["file","image","text"],"openWeights":false,"details":{"apiId":"openai/gpt-6.1-terra","maxOutputTokens":128000,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.049999999999999996,"weightsUrl":null},"reasoning":true,"overallScore":77.1059168915487,"overallScoreRange":{"low":73.33278476979433,"high":80.87904901330307},"badges":[],"capabilityScores":{"agentic":78.58298982014212,"coding":74.35155273963736,"reasoning":81.32557904547224,"multimodal":73.19012063007239,"knowledge":70.45339927606405,"multilingual":null,"instruction":null,"math":76.90845123235516},"capabilityCounts":{"math":3,"agentic":11,"knowledge":6,"multimodal":3,"coding":4,"reasoning":7},"benchmarks":[{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":100,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":67.09849544987334,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-browser:overall","benchmarkName":"LiveBench Browser Use","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":100,"unit":"percent","family":"livebench-2-agentic","index":91.7225810744159,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Browser","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.927516,"latency":null},{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":94.31818181818183,"unit":"percent","family":"gpqa-diamond","index":63.71515984960199,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.4911672683567374,"costPerTest":null,"latency":null},{"benchmarkId":"extractBenchLlamaIndex","benchmarkName":"ExtractBench: schema-guided enterprise document extraction (LlamaIndex leaderboard)","category":"multimodal","score":94.28,"unit":"percent","family":"extractBenchLlamaIndex","index":77.20149747046362,"aboveFit":false,"run":null,"source":"Boyang Zhang, Adrian Lyjak, Eli Stewart, Zhaoqi Li and Simon Suo (LlamaIndex)","sourceUrl":"https://arxiv.org/abs/2607.29677","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tier-4-v2-private:overall","benchmarkName":"FrontierMath-Tier-4-v2-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":92.6829268292683,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tier-4-v2-private:overall","index":84.89879589752957,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":4.117547077105886,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tiers-1-3-v2-private:overall","benchmarkName":"FrontierMath-Tiers-1-3-v2-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":90.87719298245615,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tiers-1-3-v2-private:overall","index":78.72806234966255,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.7085690144166794,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-coding-2:overall","benchmarkName":"LiveBench Coding 2.0","category":"coding","domain":null,"eligible":true,"ranked":true,"score":90.8,"unit":"percent","family":"livebench:livebench-coding-2:overall","index":73.94063577996118,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Coding","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.8610379310344827,"latency":null},{"benchmarkId":"livebench:livebench-reasoning:overall","benchmarkName":"LiveBench Reasoning","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":88.7,"unit":"percent","family":"livebench","index":76.52471679361898,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Reasoning","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.040296396039603964,"latency":null},{"benchmarkId":"mmmuPro","benchmarkName":"Massive Multi-discipline Multimodal Understanding Pro","category":"multimodal","score":85.5,"unit":"percent","family":"mmmu-pro","index":66.4235636955243,"aboveFit":false,"run":null,"source":"MMMU-Pro authors","sourceUrl":"https://arxiv.org/abs/2409.02813","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaMmmuPro","benchmarkName":"Artificial Analysis MMMU-Pro","category":"multimodal","score":84.7,"unit":"percent","family":"mmmu-pro","index":65.38243388342897,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmmu-pro","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:vibe-code:overall","benchmarkName":"Vibe Code Bench v1.1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":83.997,"unit":"percent","family":"vals-ai:vibe-code:overall","index":72.55695988280766,"aboveFit":false,"run":"OpenHands","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","measuredAt":"2026-11-09","stderr":2.865,"costPerTest":5.996953,"latency":2250.002},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":82.7,"unit":"percent","family":"lcr","index":64.0369089251048,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-data-analysis:overall","benchmarkName":"LiveBench Data Analysis","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":82.3,"unit":"percent","family":"livebench","index":68.233939218281,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Data%20Analysis","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.06284672000000001,"latency":null},{"benchmarkId":"livebench:livebench-design:overall","benchmarkName":"LiveBench Design","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":82.3,"unit":"percent","family":"livebench-2-agentic","index":78.77802069432381,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Design","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.270236,"latency":null},{"benchmarkId":"epoch-ai:epoch-furniture-assembly:overall","benchmarkName":"Furniture Assembly","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":79.16666666666666,"unit":"percent","family":"epoch-ai:epoch-furniture-assembly:overall","index":83.5691511767157,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":5.08288179896971,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-ebr-bench:overall","benchmarkName":"EBR-bench","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":78.0952380952381,"unit":"percent","family":"epoch-ai:epoch-ebr-bench:overall","index":88.266822018259,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":5.128728387747146,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-mirrorcode:overall","benchmarkName":"MirrorCode","category":"coding","domain":null,"eligible":true,"ranked":true,"score":76.66666666666667,"unit":"percent","family":"epoch-ai:epoch-mirrorcode:overall","index":83.23543468581754,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":9.315175080076958,"costPerTest":null,"latency":null},{"benchmarkId":"osWorld21Offline","benchmarkName":"OSWorld 2.1 offline subset (82 tasks)","category":"agentic","score":74.2,"unit":"percent","family":"osWorld21Offline","index":76.57028895920669,"aboveFit":false,"run":null,"source":"Mengqi Yuan, Zilong Zhou, Xinzhuang Xiong, Weiming Wu, Jiayang Sun, Jiamin Song, Kaiqian Cui, Bowen Wang, Haoyuan Wu, Yitong Li, Dunjie Lu, Haikong Lu, Qi Zhen, Xinyuan Wang, Jiaqi Deng, Yuhao Yang, Cheng Chen, Boyuan Zheng, Alex Su, Xiao Yu, Hao Zou, Saaket Agashe, Xing Han Lu, Manpreet Kaur, Zhengyang Qi, Vincent Sunn Chen, Frederic Sala, Dayiheng Liu, Junyang Lin, Zhou Yu, Yu Su, Siva Reddy, Xin Eric Wang, Peng Qi, Tianbao Xie, Tao Yu","sourceUrl":"https://github.com/xlang-ai/OSWorld-V2/releases","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-mystery-game-puzzles:overall","benchmarkName":"Mystery Game Puzzles","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":74,"unit":"percent","family":"epoch-ai:epoch-mystery-game-puzzles:overall","index":95,"aboveFit":true,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":4.4084400227680804,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-research:overall","benchmarkName":"LiveBench Research","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":73.9,"unit":"percent","family":"livebench-2-agentic","index":72.63483949699197,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Research","measuredAt":"2026-10-14","stderr":null,"costPerTest":1.334681818181818,"latency":null},{"benchmarkId":"parseBenchLlamaIndex","benchmarkName":"ParseBench: document parsing for AI agents (LlamaIndex leaderboard)","category":"multimodal","score":73.64,"unit":"percent","family":"parseBenchLlamaIndex","index":76.4658656302769,"aboveFit":false,"run":null,"source":"Boyang Zhang, Sebastián G. Acosta, Preston Carlson, Sacha Bron, Pierre-Loïc Doulcet, Daniel B. Ospina and Simon Suo (LlamaIndex)","sourceUrl":"https://arxiv.org/abs/2604.08538","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"healthBenchProfessionalRaw","benchmarkName":"HealthBench Professional raw score","category":"knowledge","score":64.2,"unit":"percent","family":"healthBenchProfessionalRaw","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"healthBenchProfessional","benchmarkName":"HealthBench Professional","category":"knowledge","score":62.7,"unit":"percent","family":"healthBenchProfessional","index":75.10378059062096,"aboveFit":false,"run":null,"source":"Rebecca Soskin Hicks, Mikhail Trofimov, Dominick Lim, Rahul K. Arora, Foivos Tsimpourlas, Preston Bowman, Michael Sharman, Chi Tong, Kavin Karthik, Arnav Dugar, Akshay Jagadeesh, Khaled Saab, Johannes Heidecke, Ashley Alexander, Nate Gross, Karan Singhal","sourceUrl":"https://arxiv.org/abs/2604.27470","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaAutomationBench","benchmarkName":"Artificial Analysis AutomationBench","category":"agentic","score":60.2,"unit":"percent","family":"aaAutomationBench","index":70.55477711673352,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/automationbench-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"agentsLastExam","benchmarkName":"Agents' Last Exam","category":"agentic","score":60,"unit":"percent","family":"agentsLastExam","index":90.86824442128302,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"terminalBenchScience","benchmarkName":"Terminal-Bench-Science 0.1","category":"agentic","score":58.3,"unit":"percent","family":"terminalBenchScience","index":77.83572995012798,"aboveFit":false,"run":null,"source":"Terminal-Bench-Science Team","sourceUrl":"https://www.terminal-bench-science.ai/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-simpleqa-verified:overall","benchmarkName":"SimpleQA Verified","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":57.3,"unit":"percent","family":"epoch-ai:epoch-simpleqa-verified:overall","index":71.65501065504529,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.5649789644462189,"costPerTest":null,"latency":null},{"benchmarkId":"aaTerminalBench4","benchmarkName":"Artificial Analysis Terminal-Bench v4.0","category":"agentic","score":55.6,"unit":"percent","family":"aaTerminalBench4","index":77.59982004993445,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/terminalbench-v4-0","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaSciCode","benchmarkName":"Artificial Analysis SciCode","category":"coding","score":55.1,"unit":"percent","family":"scicode-family","index":67.67318060996305,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/scicode","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"healthBenchLengthAdjusted","benchmarkName":"HealthBench length-adjusted score","category":"knowledge","score":55.1,"unit":"percent","family":"healthBenchLengthAdjusted","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"healthBench","benchmarkName":"HealthBench raw score","category":"knowledge","score":51.1,"unit":"percent","family":"healthBench","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:terminal-bench-4:overall","benchmarkName":"Terminal-Bench 4.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":49.495,"unit":"percent","family":"terminal-bench-4","index":78.01815152793199,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-4","measuredAt":"2026-11-09","stderr":1.821,"costPerTest":2.15663,"latency":2032.802},{"benchmarkId":"omniscienceAccuracy","benchmarkName":"Artificial Analysis Omniscience Accuracy","category":"knowledge","score":47.6,"unit":"percent","family":"omniscienceAccuracy","index":70.67813530785851,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":46.2,"unit":"percent","family":"hle","index":70.73876913368997,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mka","benchmarkName":"Multimodal Knowledge Agent Benchmark","category":"agentic","score":43.8,"unit":"percent","family":"mka","index":72.589842562553,"aboveFit":false,"run":null,"source":"Google DeepMind, in partnership with Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mka","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-long-horizon:overall","benchmarkName":"LiveBench Long Agentic","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":39.6,"unit":"percent","family":"livebench-2-agentic","index":47.5501829412203,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Long%20Agentic","measuredAt":"2026-10-14","stderr":null,"costPerTest":4.750444444444444,"latency":null},{"benchmarkId":"epoch-ai:epoch-spireclimb-slay-the-spire-2-ascension-10:overall","benchmarkName":"SpireClimb (Slay the Spire 2, Ascension 10)","category":null,"domain":"Other","eligible":false,"ranked":true,"score":34.66666666666667,"unit":"percent","family":"epoch-ai:epoch-spireclimb-slay-the-spire-2-ascension-10:overall","index":null,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":5.532323798062311,"costPerTest":null,"latency":null},{"benchmarkId":"healthBenchHard","benchmarkName":"HealthBench Hard","category":"knowledge","score":31.4,"unit":"percent","family":"healthBenchHard","index":70.82954011956754,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaMlcr","benchmarkName":"Medical Long Context Reasoning (MLCR-AA)","category":"reasoning","score":30.6,"unit":"percent","family":"aaMlcr","index":74.97299431193446,"aboveFit":false,"run":null,"source":"Wisedocs and Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mlcr-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaGdpPdf","benchmarkName":"Artificial Analysis GDP.pdf","category":"agentic","score":28.6,"unit":"percent","family":"aaGdpPdf","index":76.24319870032707,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdp-pdf","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":28,"unit":"percent","family":"critpt","index":91.05384888034175,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"extractBenchPro","benchmarkName":"ExtractBench Pro: long-tail document extraction challenges (LlamaIndex)","category":"multimodal","score":27,"unit":"percent","family":"extractBenchPro","index":null,"aboveFit":false,"run":null,"source":"LlamaIndex","sourceUrl":"https://www.extractbench.ai/pro","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:legal_research:overall","benchmarkName":"Legal Research Bench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":25.962,"unit":"percent","family":"vals-ai:legal_research:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_research","measuredAt":"2026-11-09","stderr":3.047,"costPerTest":3.220614,"latency":4202.728},{"benchmarkId":"epoch-ai:epoch-spireclimb:overall","benchmarkName":"SpireClimb","category":null,"domain":"Other","eligible":false,"ranked":true,"score":22.333333333333332,"unit":"percent","family":"epoch-ai:epoch-spireclimb:overall","index":null,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":3.386031744341088,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:tax_agent_bench:overall","benchmarkName":"Tax Agent Bench","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":17.081,"unit":"percent","family":"vals-ai:tax_agent_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench","measuredAt":"2026-11-09","stderr":2.714,"costPerTest":2.159819,"latency":2751.35},{"benchmarkId":"lmarena:agent-steerability:overall","benchmarkName":"Agent Arena steerability","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":16.600454816893475,"unit":"index","family":"agent-arena","index":80.17897764546558,"aboveFit":false,"run":"max effort","source":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset","measuredAt":"2026-11-11","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mazeBench","benchmarkName":"MazeBench: visual spatial reasoning in a 3D open world","category":"agentic","score":16,"unit":"percent","family":"mazeBench","index":91.09841061360541,"aboveFit":false,"run":null,"source":"Jonathan Pappas and David Pappas","sourceUrl":"https://mazebench.com/blog?post=maze-bench-results","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-spireclimb-slay-the-spire-1-ascension-20-heart:overall","benchmarkName":"SpireClimb (Slay the Spire 1, Ascension 20 Heart)","category":null,"domain":"Other","eligible":false,"ranked":true,"score":10,"unit":"percent","family":"epoch-ai:epoch-spireclimb-slay-the-spire-1-ascension-20-heart:overall","index":null,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":3.9056673294247153,"costPerTest":null,"latency":null},{"benchmarkId":"lmarena:agent-task-outcome:overall","benchmarkName":"Agent Arena task outcome","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":9.665077719264549,"unit":"index","family":"agent-arena","index":74.47701040293887,"aboveFit":false,"run":"max effort","source":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset","measuredAt":"2026-11-11","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:hlab:overall","benchmarkName":"HLAB-Vals-1.1","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":1.667,"unit":"percent","family":"vals-ai:hlab:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/hlab","measuredAt":"2026-11-09","stderr":1.013,"costPerTest":1.477673,"latency":821.505}],"price":{"inputPerMillion":1,"outputPerMillion":5,"batch":{"inputPerMillion":0.5,"outputPerMillion":2.5},"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":46,"timeToFirstToken":3.247,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":90,"benchmarkCount":34,"categoryCount":6,"confidence":"medium","state":"available"},"hosts":[{"host":"amazon-bedrock","name":"Amazon Bedrock","kind":"cloud","url":"https://docs.aws.amazon.com/bedrock/latest/userguide/models-supported.html","inputPerMillion":1,"outputPerMillion":5,"cacheReadPerMillion":0.05,"contextTokens":1050000,"maxOutputTokens":128000,"deprecated":false},{"host":"digitalocean","name":"DigitalOcean","kind":"cloud","url":"https://docs.digitalocean.com/products/gradient-ai-platform/details/models/","inputPerMillion":1,"outputPerMillion":5,"cacheReadPerMillion":0.05,"contextTokens":1050000,"maxOutputTokens":128000,"deprecated":false},{"host":"azure","name":"Microsoft Azure","kind":"cloud","url":"https://learn.microsoft.com/en-us/azure/ai-services/openai/concepts/models","inputPerMillion":1,"outputPerMillion":5,"cacheReadPerMillion":0.05,"contextTokens":1050000,"maxOutputTokens":128000,"deprecated":false},{"host":"openai","name":"OpenAI","kind":"maker","url":"https://platform.openai.com/docs/models","inputPerMillion":1,"outputPerMillion":5,"cacheReadPerMillion":0.05,"contextTokens":1050000,"maxOutputTokens":128000,"deprecated":false},{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":1,"outputPerMillion":5,"cacheReadPerMillion":0.05,"contextTokens":1050000,"maxOutputTokens":128000,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:livebench","name":"LiveBench","url":"https://livebench.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:lmarena","name":"LMArena","url":"https://huggingface.co","license":"CC BY 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"claude-fable-5-1","providerId":"anthropic","slug":"claude-fable-5-1","name":"Claude Fable 5.1","provider":"Anthropic","description":"Claude Fable 5.1 is a reasoning model from Anthropic in the Claude Fable family. 71 benchmarks count toward its index, in 7 categories.","logoUrl":null,"releaseDate":"2026-09-01","contextWindowTokens":1000000,"modalities":["text","image","file"],"openWeights":false,"details":{"apiId":"anthropic/claude-fable-5.1","maxOutputTokens":128000,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.25,"weightsUrl":null},"reasoning":true,"overallScore":76.15615747293195,"overallScoreRange":{"low":73.73713153163249,"high":78.57518341423142},"badges":[],"capabilityScores":{"agentic":71.0672245775348,"coding":77.71840190601472,"reasoning":75.82855755788577,"multimodal":77.90567938221508,"knowledge":77.17409603643989,"multilingual":null,"instruction":74.2729628607804,"math":77.01914728782165},"capabilityCounts":{"math":5,"agentic":28,"coding":18,"reasoning":10,"knowledge":6,"multimodal":3,"instruction":1},"benchmarks":[{"benchmarkId":"vals-ai:proof_bench:overall","benchmarkName":"ProofBench v1.1","category":"math","domain":null,"eligible":true,"ranked":true,"score":100,"unit":"percent","family":"vals-ai:proof_bench:overall","index":81.68109068983412,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/proof_bench","measuredAt":"2026-11-09","stderr":0,"costPerTest":2.70347,"latency":478.303},{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":100,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":67.09849544987334,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-browser:overall","benchmarkName":"LiveBench Browser Use","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":100,"unit":"percent","family":"livebench-2-agentic","index":91.7225810744159,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Browser","measuredAt":"2026-10-14","stderr":null,"costPerTest":3.746228,"latency":null},{"benchmarkId":"arc-prize:v1-public-eval:overall","benchmarkName":"ARC-AGI-1 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":99,"unit":"percent","family":"arc-prize:v1-public-eval:overall","index":null,"aboveFit":false,"run":"max effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.892544475000001,"latency":null},{"benchmarkId":"arc-prize:v2-public-eval:overall","benchmarkName":"ARC-AGI-2 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":98.33333333333333,"unit":"percent","family":"arc-prize:v2-public-eval:overall","index":null,"aboveFit":false,"run":"max effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":4.331563333333333,"latency":null},{"benchmarkId":"livebench:livebench-coding-2:overall","benchmarkName":"LiveBench Coding 2.0","category":"coding","domain":null,"eligible":true,"ranked":true,"score":97.7,"unit":"percent","family":"livebench:livebench-coding-2:overall","index":78.93847880445804,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Coding","measuredAt":"2026-10-14","stderr":null,"costPerTest":5.28236551724138,"latency":null},{"benchmarkId":"arc-prize:v1-semi-private:overall","benchmarkName":"ARC-AGI-1 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":97.5,"unit":"percent","family":"arc-prize:v1-semi-private:overall","index":71.81162890019444,"aboveFit":false,"run":"max effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":1.3997611,"latency":null},{"benchmarkId":"livebench:livebench-math:overall","benchmarkName":"LiveBench Mathematics","category":"math","domain":null,"eligible":true,"ranked":true,"score":97.00675,"unit":"percent","family":"livebench:livebench-math:overall","index":74.97500332681895,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Mathematics","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.589025857519789,"latency":null},{"benchmarkId":"gmmlu","benchmarkName":"Global MMLU","category":"multilingual","score":94,"unit":"percent","family":"gmmlu","index":null,"aboveFit":false,"run":null,"source":"Singh et al.","sourceUrl":"https://arxiv.org/abs/2412.03304","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":93.7,"unit":"percent","family":"gpqa-diamond","index":63.14818037783339,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:gpqa:overall","benchmarkName":"GPQA Diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":93.434,"unit":"percent","family":"gpqa-diamond","index":62.904212446304726,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","measuredAt":"2026-09-01","stderr":1.863,"costPerTest":0.204523,"latency":45.857},{"benchmarkId":"epoch-ai:epoch-mirrorcode:overall","benchmarkName":"MirrorCode","category":"coding","domain":null,"eligible":true,"ranked":true,"score":93.33333333333333,"unit":"percent","family":"epoch-ai:epoch-mirrorcode:overall","index":90.55495408267049,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":3.9171310092866882,"costPerTest":null,"latency":null},{"benchmarkId":"aaHarveyLab","benchmarkName":"Artificial Analysis Harvey LAB-AA v1.0","category":"agentic","score":93,"unit":"percent","family":"aaHarveyLab","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"milu","benchmarkName":"Multi-task Indic Language Understanding Benchmark","category":"multilingual","score":93,"unit":"percent","family":"milu","index":null,"aboveFit":false,"run":null,"source":"Verma et al.","sourceUrl":"https://arxiv.org/abs/2411.02538","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:mmlu_pro:overall","benchmarkName":"MMLU Pro","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":92.375,"unit":"percent","family":"mmlu-pro","index":60.86217142367394,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","measuredAt":"2026-09-01","stderr":0.266,"costPerTest":0.097724,"latency":20.589},{"benchmarkId":"livebench:livebench-reasoning:overall","benchmarkName":"LiveBench Reasoning","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":91.69225,"unit":"percent","family":"livebench","index":80.4009791685885,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Reasoning","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.46013564356435643,"latency":null},{"benchmarkId":"aaTerminalBench21","benchmarkName":"Artificial Analysis Terminal-Bench v2.1","category":"agentic","score":91.4,"unit":"percent","family":"aaTerminalBench21","index":74.09846187701987,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/terminalbench-v2-1","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:medscribe:overall","benchmarkName":"MedScribe","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":91.294,"unit":"percent","family":"vals-ai:medscribe:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medscribe","measuredAt":"2026-10-29","stderr":1.953,"costPerTest":0.9635,"latency":188.363},{"benchmarkId":"vals-ai:ioi:overall","benchmarkName":"IOI","category":"coding","domain":null,"eligible":true,"ranked":true,"score":90.778,"unit":"percent","family":"vals-ai:ioi:overall","index":80.19672887300182,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/ioi","measuredAt":"2026-10-29","stderr":4.647,"costPerTest":11.196959,"latency":1805.905},{"benchmarkId":"vals-ai:mmmu:overall","benchmarkName":"MMMU Pro","category":"multimodal","domain":null,"eligible":true,"ranked":true,"score":90.636,"unit":"percent","family":"mmmu-pro","index":73.10761708917633,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmmu","measuredAt":"2026-09-01","stderr":0.7,"costPerTest":0.166285,"latency":37.47},{"benchmarkId":"vals-ai:lcb:overall","benchmarkName":"LiveCodeBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":90.515,"unit":"percent","family":"livecodebench","index":64.44779384668172,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","measuredAt":"2026-09-01","stderr":0.863,"costPerTest":0.260604,"latency":59.738},{"benchmarkId":"vals-ai:vibe-code:overall","benchmarkName":"Vibe Code Bench v1.1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":90.263,"unit":"percent","family":"vals-ai:vibe-code:overall","index":74.81783112905107,"aboveFit":false,"run":"OpenHands","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","measuredAt":"2026-11-09","stderr":1.566,"costPerTest":33.367507,"latency":3459.994},{"benchmarkId":"epoch-ai:epoch-frontiermath-tiers-1-3-v2-private:overall","benchmarkName":"FrontierMath-Tiers-1-3-v2-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":90.17543859649123,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tiers-1-3-v2-private:overall","index":78.35869272769853,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.7662067789031757,"costPerTest":null,"latency":null},{"benchmarkId":"arc-prize:v2-semi-private:overall","benchmarkName":"ARC-AGI-2 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":90,"unit":"percent","family":"arc-prize:v2-semi-private:overall","index":80.5106384900966,"aboveFit":false,"run":"max effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":4.492348833333334,"latency":null},{"benchmarkId":"livebench:livebench-language:overall","benchmarkName":"LiveBench Language","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":89.50099999999999,"unit":"percent","family":"livebench:livebench-language:overall","index":75.76733348485851,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Language","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.21783599999999997,"latency":null},{"benchmarkId":"vals-ai:legal_bench:overall","benchmarkName":"LegalBench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":88.511,"unit":"percent","family":"vals-ai:legal_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","measuredAt":"2026-10-13","stderr":0.42,"costPerTest":0.051606,"latency":12.405},{"benchmarkId":"epoch-ai:epoch-frontiermath-tier-4-v2-private:overall","benchmarkName":"FrontierMath-Tier-4-v2-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":87.8048780487805,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tier-4-v2-private:overall","index":82.98245424488331,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":5.173952057462543,"costPerTest":null,"latency":null},{"benchmarkId":"draco","benchmarkName":"Data Research and Analysis with Complex Operations","category":"agentic","score":87.7,"unit":"percent","family":"draco","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"programBench","benchmarkName":"ProgramBench: Can Language Models Rebuild Programs From Scratch?","category":"coding","score":87.6,"unit":"percent","family":"programBench","index":95,"aboveFit":false,"run":null,"source":"John Yang, Kilian Lieret, Jeffrey Ma, Parth Thakkar, Dmitrii Pedchenko, Sten Sootla, Emily McMilin, Pengcheng Yin, Rui Hou, Gabriel Synnaeve, Diyi Yang, Ofir Press","sourceUrl":"https://programbench.com/static/paper.pdf","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-coding:overall","benchmarkName":"LiveBench Coding","category":"coding","domain":null,"eligible":true,"ranked":true,"score":86.375,"unit":"percent","family":"livebench:livebench-coding:overall","index":76.30260612752578,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Coding","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.4012735042735043,"latency":null},{"benchmarkId":"chartographyWithTools","benchmarkName":"Chartography with image and code tools","category":"multimodal","score":86.2,"unit":"percent","family":"chartographyWithTools","index":null,"aboveFit":false,"run":null,"source":"Surge AI and Anthropic","sourceUrl":"https://surgehq.ai/blog/chartography","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gdpPdf","benchmarkName":"GDP.pdf mean criteria pass rate without tools","category":"multimodal","score":85.4,"unit":"percent","family":"gdpPdf","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":85.3,"unit":"percent","family":"lcr","index":65.82071829162751,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gdpPdfWithTools","benchmarkName":"GDP.pdf mean criteria pass rate with tools","category":"multimodal","score":85.1,"unit":"percent","family":"gdpPdfWithTools","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:terminal-bench-2-1:overall","benchmarkName":"Terminal-Bench 2.1","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":85.019,"unit":"percent","family":"terminal-bench-2-1","index":70.8649394394831,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2-1","measuredAt":"2026-10-01","stderr":0.991,"costPerTest":3.074796,"latency":500.355},{"benchmarkId":"benchCadVision2CodeWithTools","benchmarkName":"BenchCAD Vision2Code voxel IoU with tools","category":"multimodal","score":84.3,"unit":"percent","family":"benchCadVision2CodeWithTools","index":null,"aboveFit":false,"run":null,"source":"Zhang et al. and Anthropic","sourceUrl":"https://arxiv.org/abs/2605.10865","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"osWorld21Offline","benchmarkName":"OSWorld 2.1 offline subset (82 tasks)","category":"agentic","score":83.5,"unit":"percent","family":"osWorld21Offline","index":82.44806961008622,"aboveFit":false,"run":null,"source":"Mengqi Yuan, Zilong Zhou, Xinzhuang Xiong, Weiming Wu, Jiayang Sun, Jiamin Song, Kaiqian Cui, Bowen Wang, Haoyuan Wu, Yitong Li, Dunjie Lu, Haikong Lu, Qi Zhen, Xinyuan Wang, Jiaqi Deng, Yuhao Yang, Cheng Chen, Boyuan Zheng, Alex Su, Xiao Yu, Hao Zou, Saaket Agashe, Xing Han Lu, Manpreet Kaur, Zhengyang Qi, Vincent Sunn Chen, Frederic Sala, Dayiheng Liu, Junyang Lin, Zhou Yu, Yu Su, Siva Reddy, Xin Eric Wang, Peng Qi, Tianbao Xie, Tao Yu","sourceUrl":"https://github.com/xlang-ai/OSWorld-V2/releases","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-design:overall","benchmarkName":"LiveBench Design","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":83.4,"unit":"percent","family":"livebench-2-agentic","index":79.58248489873631,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Design","measuredAt":"2026-10-14","stderr":null,"costPerTest":7.375544,"latency":null},{"benchmarkId":"livebench:livebench-research:overall","benchmarkName":"LiveBench Research","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":82,"unit":"percent","family":"livebench-2-agentic","index":78.55862136584767,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Research","measuredAt":"2026-10-14","stderr":null,"costPerTest":20.88601818181818,"latency":null},{"benchmarkId":"toolathlonVerifiedPass3","benchmarkName":"Toolathlon Verified Pass@3","category":"agentic","score":81.5,"unit":"percent","family":"toolathlonVerifiedPass3","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"swePro","benchmarkName":"SWE-bench Pro","category":"coding","score":81.2,"unit":"percent","family":"swePro","index":77.99011414637773,"aboveFit":false,"run":null,"source":"Xiang Deng, Jeff Da, Edwin Pan, Yannis Yiming He, Charles Ide, Kanak Garg, Niklas Lauffer, Andrew Park, Nitin Pasari, Chetan Rane, Karmini Sampath, Maya Krishnan, Srivatsa Kundurthy, Sean Hendryx, Zifan Wang, Chen Bo Calvin Zhang, Noah Jacobson, Bing Liu, Brad Kenstler","sourceUrl":"https://arxiv.org/abs/2509.16941","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-long-horizon:overall","benchmarkName":"LiveBench Long Agentic","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":80.8,"unit":"percent","family":"livebench-2-agentic","index":77.68102405194313,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Long%20Agentic","measuredAt":"2026-10-14","stderr":null,"costPerTest":51.57278888888889,"latency":null},{"benchmarkId":"osWorld21","benchmarkName":"OSWorld 2.1 (all 108 tasks)","category":"agentic","score":80.7,"unit":"percent","family":"osWorld21","index":78.0639329948257,"aboveFit":false,"run":null,"source":"Mengqi Yuan, Zilong Zhou, Xinzhuang Xiong, Weiming Wu, Jiayang Sun, Jiamin Song, Kaiqian Cui, Bowen Wang, Haoyuan Wu, Yitong Li, Dunjie Lu, Haikong Lu, Qi Zhen, Xinyuan Wang, Jiaqi Deng, Yuhao Yang, Cheng Chen, Boyuan Zheng, Alex Su, Xiao Yu, Hao Zou, Saaket Agashe, Xing Han Lu, Manpreet Kaur, Zhengyang Qi, Vincent Sunn Chen, Frederic Sala, Dayiheng Liu, Junyang Lin, Zhou Yu, Yu Su, Siva Reddy, Xin Eric Wang, Peng Qi, Tianbao Xie, Tao Yu","sourceUrl":"https://github.com/xlang-ai/OSWorld-V2/releases","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-data-analysis:overall","benchmarkName":"LiveBench Data Analysis","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":80.27566666666667,"unit":"percent","family":"livebench","index":65.61154899874832,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Data%20Analysis","measuredAt":"2026-10-14","stderr":null,"costPerTest":1.3331426666666668,"latency":null},{"benchmarkId":"officeQa","benchmarkName":"OfficeQA","category":"multimodal","score":80.2,"unit":"percent","family":"officeQa","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"parseBenchLlamaIndex","benchmarkName":"ParseBench: document parsing for AI agents (LlamaIndex leaderboard)","category":"multimodal","score":77.81,"unit":"percent","family":"parseBenchLlamaIndex","index":82.28447255080884,"aboveFit":false,"run":null,"source":"Boyang Zhang, Sebastián G. Acosta, Preston Carlson, Sacha Bron, Pierre-Loïc Doulcet, Daniel B. Ospina and Simon Suo (LlamaIndex)","sourceUrl":"https://arxiv.org/abs/2604.08538","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"toolathlonVerified","benchmarkName":"Toolathlon-Verified","category":"agentic","score":77.8,"unit":"percent","family":"toolathlonVerified","index":71.6099643123515,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:emb:overall","benchmarkName":"EMB","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":76.67,"unit":"percent","family":"vals-ai:emb:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/emb","measuredAt":"2026-10-20","stderr":2.085,"costPerTest":15.93249,"latency":1771.558},{"benchmarkId":"vals-ai:tax_eval_v2:overall","benchmarkName":"TaxEval v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":75.961,"unit":"percent","family":"vals-ai:tax_eval_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","measuredAt":"2026-09-01","stderr":0.83,"costPerTest":0.346549,"latency":85.564},{"benchmarkId":"vals-ai:public-benefits-bench:overall","benchmarkName":"Public Benefits Bench v1.1","category":null,"domain":"Public benefits","eligible":false,"ranked":true,"score":74.899,"unit":"percent","family":"vals-ai:public-benefits-bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/public-benefits-bench","measuredAt":"2026-10-12","stderr":1.128,"costPerTest":7.481573,"latency":2882.401},{"benchmarkId":"cursorBench32","benchmarkName":"cursorBench32","category":"coding","score":73.4,"unit":"percent","family":"cursorBench32","index":76.14844680866653,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"toolathlonVerifiedPass3All","benchmarkName":"Toolathlon Verified Pass cubed","category":"agentic","score":73.1,"unit":"percent","family":"toolathlonVerifiedPass3All","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-instruction-following:overall","benchmarkName":"LiveBench Instruction Following","category":"instruction","domain":null,"eligible":true,"ranked":true,"score":72.98775,"unit":"percent","family":"livebench:livebench-instruction-following:overall","index":74.2729628607804,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=IF","measuredAt":"2026-06-25","stderr":null,"costPerTest":1.2004130000000002,"latency":null},{"benchmarkId":"apprenticeBench","benchmarkName":"ApprenticeBench: end-to-end computer use, continual learning, and long-horizon agency on a real accounts-payable job","category":"agentic","score":72,"unit":"percent","family":"apprenticeBench","index":94.70888176688149,"aboveFit":true,"run":null,"source":"NeoCognition","sourceUrl":"https://neocognition.io/blog/apprentice-bench/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-simpleqa-verified:overall","benchmarkName":"SimpleQA Verified","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":70.8,"unit":"percent","family":"epoch-ai:epoch-simpleqa-verified:overall","index":82.90170241240985,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.4385511563477433,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:mortgage_tax:overall","benchmarkName":"MortgageTax","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":70.786,"unit":"percent","family":"vals-ai:mortgage_tax:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mortgage_tax","measuredAt":"2026-09-01","stderr":0.896,"costPerTest":0.171632,"latency":14.98},{"benchmarkId":"vals-ai:cyber:overall","benchmarkName":"CyberBench v1.1","category":null,"domain":"Other","eligible":false,"ranked":true,"score":70.416,"unit":"percent","family":"vals-ai:cyber:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/cyber","measuredAt":"2026-11-05","stderr":5.43,"costPerTest":4.010688,"latency":1112.095},{"benchmarkId":"epoch-ai:epoch-furniture-assembly:overall","benchmarkName":"Furniture Assembly","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":70,"unit":"percent","family":"epoch-ai:epoch-furniture-assembly:overall","index":79.29586957446455,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":5.724365605193373,"costPerTest":null,"latency":null},{"benchmarkId":"aaHealthBenchProfessionalRaw","benchmarkName":"Artificial Analysis HealthBench Professional raw score","category":"knowledge","score":69.2,"unit":"percent","family":"aaHealthBenchProfessionalRaw","index":null,"aboveFit":false,"run":null,"source":"OpenAI (benchmark); Artificial Analysis (evaluation)","sourceUrl":"https://artificialanalysis.ai/evaluations/healthbench-professional","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"officeQaPro","benchmarkName":"OfficeQA Pro","category":"multimodal","score":69,"unit":"percent","family":"officeQaPro","index":78.32494850666009,"aboveFit":false,"run":null,"source":"OfficeQA Pro authors","sourceUrl":"https://arxiv.org/abs/2603.08655","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"wandr","benchmarkName":"WANDR (Wide ANd Deep Research)","category":"agentic","score":68.7,"unit":"percent","family":"wandr","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"omniscienceAccuracy","benchmarkName":"Artificial Analysis Omniscience Accuracy","category":"knowledge","score":68.4,"unit":"percent","family":"omniscienceAccuracy","index":95,"aboveFit":true,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"deepSwe","benchmarkName":"DeepSWE","category":"agentic","score":67.4,"unit":"percent","family":"deepSwe","index":70.07463909405871,"aboveFit":false,"run":null,"source":"Datacurve AI","sourceUrl":"https://deepswe.datacurve.ai/blog","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-agentic-coding:overall","benchmarkName":"LiveBench Agentic Coding","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":66.06066666666668,"unit":"percent","family":"livebench:livebench-agentic-coding:overall","index":77.28993077178805,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Agentic%20Coding","measuredAt":"2026-06-25","stderr":null,"costPerTest":6.219509722222223,"latency":null},{"benchmarkId":"graphwalksBfs256k1m","benchmarkName":"GraphWalks BFS 256K–1M","category":"reasoning","score":65,"unit":"percent","family":"graphwalksBfs256k1m","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hle","benchmarkName":"Humanity's Last Exam","category":"knowledge","score":65,"unit":"percent","family":"hle","index":89.6006727402501,"aboveFit":false,"run":null,"source":"Center for AI Safety, Scale AI, and thousands of expert contributors","sourceUrl":"https://lastexam.ai/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaHealthBench","benchmarkName":"Artificial Analysis HealthBench raw score","category":"knowledge","score":63.9,"unit":"percent","family":"aaHealthBench","index":null,"aboveFit":false,"run":null,"source":"OpenAI (benchmark); Artificial Analysis (evaluation)","sourceUrl":"https://artificialanalysis.ai/evaluations/healthbench","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:time_horizon_index:overall","benchmarkName":"Time Horizon Index: KSP","category":null,"domain":"Other","eligible":false,"ranked":true,"score":63.333,"unit":"percent","family":"vals-ai:time_horizon_index:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/time_horizon_index","measuredAt":"2026-10-07","stderr":0,"costPerTest":4655.011028,"latency":0},{"benchmarkId":"aaSciCode","benchmarkName":"Artificial Analysis SciCode","category":"coding","score":63.1,"unit":"percent","family":"scicode-family","index":78.10270978128092,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/scicode","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gdpvalAaNormalized","benchmarkName":"GDPval-AA normalized","category":"agentic","score":62.7,"unit":"percent","family":"gdpvalAaNormalized","index":78.86281787527065,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:skillsbench:overall","benchmarkName":"SkillsBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":61.552,"unit":"percent","family":"vals-ai:skillsbench:overall","index":70.95653771088944,"aboveFit":false,"run":"OpenHands","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/skillsbench","measuredAt":"2026-10-29","stderr":4.76,"costPerTest":4.516559,"latency":661.978},{"benchmarkId":"aaBriefcaseRubric","benchmarkName":"AA-Briefcase v1.1, rubric pass rate","category":"agentic","score":61.5,"unit":"percent","family":"aaBriefcaseRubric","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"physicianBench","benchmarkName":"PhysicianBench","category":"knowledge","score":61,"unit":"percent","family":"physicianBench","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hleNoTools","benchmarkName":"Humanity's Last Exam without tools","category":"knowledge","score":60.9,"unit":"percent","family":"hle","index":85.48717248562795,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaAutomationBench","benchmarkName":"Artificial Analysis AutomationBench","category":"agentic","score":59.4,"unit":"percent","family":"aaAutomationBench","index":69.82465236093942,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/automationbench-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":59.1,"unit":"percent","family":"hle","index":83.68124554457432,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:fabv2:overall","benchmarkName":"Finance Agent (v2)","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":58.877,"unit":"percent","family":"vals-ai:fabv2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/fabv2","measuredAt":"2026-10-29","stderr":2.058,"costPerTest":8.346696,"latency":1094.165},{"benchmarkId":"vals-ai:terminal-bench-4:overall","benchmarkName":"Terminal-Bench 4.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":58.081,"unit":"percent","family":"terminal-bench-4","index":82.09511601729429,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-4","measuredAt":"2026-11-09","stderr":3.312,"costPerTest":17.175337,"latency":3623.762},{"benchmarkId":"cweBenchV1","benchmarkName":"CWE-bench v1","category":"agentic","score":58,"unit":"percent","family":"cweBenchV1","index":71.07229074770696,"aboveFit":false,"run":null,"source":"Collinear AI","sourceUrl":"https://cwe-bench.com/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-mystery-game-puzzles:overall","benchmarkName":"Mystery Game Puzzles","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":57.99999999999999,"unit":"percent","family":"epoch-ai:epoch-mystery-game-puzzles:overall","index":83.59706221097015,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":4.9604496374885825,"costPerTest":null,"latency":null},{"benchmarkId":"terminal-bench:terminal-bench-4-0-0:overall","benchmarkName":"Terminal-Bench 4.0.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":57.88,"unit":"percent","family":"terminal-bench-4","index":81.9996734523791,"aboveFit":false,"run":"max effort · Claude Code","source":"Terminal-Bench","sourceUrl":"https://www.tbench.ai/leaderboard","measuredAt":"2026-09-29","stderr":1.9183673469387754,"costPerTest":18.919696969696968,"latency":3893.5},{"benchmarkId":"aaHealthBenchProfessional","benchmarkName":"Artificial Analysis HealthBench Professional length-adjusted score","category":"knowledge","score":57.8,"unit":"percent","family":"aaHealthBenchProfessional","index":null,"aboveFit":false,"run":null,"source":"OpenAI (benchmark); Artificial Analysis (evaluation)","sourceUrl":"https://artificialanalysis.ai/evaluations/healthbench-professional","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaAnalystAgent","benchmarkName":"Artificial Analysis AnalystAgent","category":"agentic","score":57.5,"unit":"percent","family":"aaAnalystAgent","index":80.48680917642174,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/aa-analyst-agent","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"terminal-bench:terminal-bench-4-1-0:overall","benchmarkName":"Terminal-Bench 4.1.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":57.27,"unit":"percent","family":"terminal-bench:terminal-bench-4-1-0:overall","index":79.66976369396711,"aboveFit":false,"run":"max effort · Claude Code","source":"Terminal-Bench","sourceUrl":"https://www.tbench.ai/leaderboard","measuredAt":"2026-09-29","stderr":1.9540816326530612,"costPerTest":18.919696969696968,"latency":3893.5},{"benchmarkId":"aaHealthBenchLengthAdjusted","benchmarkName":"Artificial Analysis HealthBench length-adjusted score","category":"knowledge","score":57.2,"unit":"percent","family":"aaHealthBenchLengthAdjusted","index":null,"aboveFit":false,"run":null,"source":"OpenAI (benchmark); Artificial Analysis (evaluation)","sourceUrl":"https://artificialanalysis.ai/evaluations/healthbench","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-ebr-bench:overall","benchmarkName":"EBR-bench","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":57.14285714285714,"unit":"percent","family":"epoch-ai:epoch-ebr-bench:overall","index":78.65120061232378,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":6.023386019368341,"costPerTest":null,"latency":null},{"benchmarkId":"frontierSweV2","benchmarkName":"FrontierSWE v2","category":"coding","score":56.3,"unit":"percent","family":"frontierSweV2","index":80.36365760306427,"aboveFit":false,"run":null,"source":"Proximal","sourceUrl":"https://www.frontierswe.com/blog/v2","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:legal_research:overall","benchmarkName":"Legal Research Bench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":55.288,"unit":"percent","family":"vals-ai:legal_research:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_research","measuredAt":"2026-11-09","stderr":3.456,"costPerTest":23.055789,"latency":3449.382},{"benchmarkId":"vals-ai:code-migration:overall","benchmarkName":"Code Migration","category":"coding","domain":null,"eligible":true,"ranked":true,"score":54.607,"unit":"percent","family":"vals-ai:code-migration:overall","index":74.19856524138766,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/code-migration","measuredAt":"2026-10-29","stderr":4.812,"costPerTest":70.973455,"latency":15610.474},{"benchmarkId":"vals-ai:medcode:overall","benchmarkName":"MedCode","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":53.509,"unit":"percent","family":"vals-ai:medcode:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medcode","measuredAt":"2026-10-29","stderr":2.165,"costPerTest":1.116862,"latency":215.494},{"benchmarkId":"terminalBenchScience","benchmarkName":"Terminal-Bench-Science 0.1","category":"agentic","score":52.6,"unit":"percent","family":"terminalBenchScience","index":74.62593723336671,"aboveFit":false,"run":null,"source":"Terminal-Bench-Science Team","sourceUrl":"https://www.terminal-bench-science.ai/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaTerminalBench4","benchmarkName":"Artificial Analysis Terminal-Bench v4.0","category":"agentic","score":52,"unit":"percent","family":"aaTerminalBench4","index":75.86071958648239,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/terminalbench-v4-0","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"cursorBench40","benchmarkName":"cursorBench40","category":"coding","score":51.8,"unit":"percent","family":"cursorBench40","index":76.4678260372115,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaCweBench","benchmarkName":"Artificial Analysis CWE-Bench-AA","category":"agentic","score":50.8,"unit":"percent","family":"aaCweBench","index":64.07622346971608,"aboveFit":false,"run":null,"source":"Artificial Analysis; dataset by Collinear AI","sourceUrl":"https://artificialanalysis.ai/evaluations/cwe-bench-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:tax_agent_bench:overall","benchmarkName":"Tax Agent Bench","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":50.749,"unit":"percent","family":"vals-ai:tax_agent_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench","measuredAt":"2026-11-09","stderr":3.606,"costPerTest":13.180073,"latency":1714.017},{"benchmarkId":"aaItbench","benchmarkName":"Artificial Analysis ITBench-AA","category":"agentic","score":49.5,"unit":"percent","family":"aaItbench","index":71.1373841286965,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/itbench-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-chess-puzzles:overall","benchmarkName":"Chess Puzzles","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":48,"unit":"percent","family":"epoch-ai:epoch-chess-puzzles:overall","index":70.41740509234356,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":5.021167315686783,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:mysterymechanism:overall","benchmarkName":"MysteryMechanism","category":null,"domain":"Science","eligible":false,"ranked":true,"score":47.748,"unit":"percent","family":"vals-ai:mysterymechanism:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mysterymechanism","measuredAt":"2026-11-11","stderr":3.36,"costPerTest":5.631518,"latency":1060.876},{"benchmarkId":"aaTau3Banking","benchmarkName":"Artificial Analysis Tau3-Banking","category":"agentic","score":47.2,"unit":"percent","family":"aaTau3Banking","index":72.91487670348333,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/tau3-banking","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"benchCad21WithTools","benchmarkName":"BenchCAD 2.1 Core, with tools","category":"multimodal","score":44.4,"unit":"percent","family":"benchCad21WithTools","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"benchCadVision2Code","benchmarkName":"BenchCAD Vision2Code voxel IoU without tools","category":"multimodal","score":43.7,"unit":"percent","family":"benchCadVision2Code","index":null,"aboveFit":false,"run":null,"source":"Zhang et al. and Anthropic","sourceUrl":"https://arxiv.org/abs/2605.10865","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"bug-hunt-bench","benchmarkName":"Bug Hunt Bench","category":"coding","score":43,"unit":"percent","family":"bug-hunt-bench","index":78.42711761959501,"aboveFit":false,"run":null,"source":"Pawel Huryn","sourceUrl":"https://bughunt.productcompass.pm/method","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"chartography","benchmarkName":"Chartography without tools","category":"multimodal","score":42.6,"unit":"percent","family":"chartography","index":null,"aboveFit":false,"run":null,"source":"Surge AI and Anthropic","sourceUrl":"https://surgehq.ai/blog/chartography","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"osWorld2","benchmarkName":"OSWorld 2.0","category":"agentic","score":41.7,"unit":"percent","family":"osWorld2","index":70.93177121450782,"aboveFit":false,"run":null,"source":"Mengqi Yuan, Zilong Zhou, Xinzhuang Xiong, Weiming Wu, Jiayang Sun, Jiamin Song, Kaiqian Cui, Bowen Wang, Haoyuan Wu, Yitong Li, Dunjie Lu, Haikong Lu, Qi Zhen, Xinyuan Wang, Jiaqi Deng, Yuhao Yang, Cheng Chen, Boyuan Zheng, Alex Su, Xiao Yu, Hao Zou, Saaket Agashe, Xing Han Lu, Manpreet Kaur, Zhengyang Qi, Vincent Sunn Chen, Frederic Sala, Dayiheng Liu, Junyang Lin, Zhou Yu, Yu Su, Siva Reddy, Xin Eric Wang, Peng Qi, Tianbao Xie, Tao Yu","sourceUrl":"https://arxiv.org/abs/2606.29537","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"postTrainBench11","benchmarkName":"PostTrainBench v1.1","category":"coding","score":40.2,"unit":"percent","family":"postTrainBench11","index":74.25287392501038,"aboveFit":false,"run":null,"source":"PostTrainBench team","sourceUrl":"https://posttrainbench.com/?version=v1.1","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:terminal-bench-science:overall","benchmarkName":"Terminal-Bench-Science","category":null,"domain":"Science","eligible":false,"ranked":true,"score":40,"unit":"percent","family":"vals-ai:terminal-bench-science:overall","index":null,"aboveFit":false,"run":"Mini-SWE-agent","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-science","measuredAt":"2026-11-09","stderr":5.898,"costPerTest":38.010073,"latency":9376.226},{"benchmarkId":"benchCad21","benchmarkName":"BenchCAD 2.1 Core, without tools","category":"multimodal","score":36.3,"unit":"percent","family":"benchCad21","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"frontierCode2","benchmarkName":"FrontierCode 2.0","category":"coding","score":32.4,"unit":"percent","family":"frontierCode2","index":76.04768426661789,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"automationBench","benchmarkName":"AutomationBench","category":"agentic","score":31.4,"unit":"percent","family":"automationBench","index":41.81626246459745,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"stemProfessorBench","benchmarkName":"Anthropic STEM Professor Bench","category":"reasoning","score":29.9,"unit":"percent","family":"stemProfessorBench","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":29.7,"unit":"percent","family":"critpt","index":94.34485051872987,"aboveFit":true,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:vcb-1-100:overall","benchmarkName":"Vibe Code Bench 1-100","category":"coding","domain":null,"eligible":true,"ranked":true,"score":28,"unit":"percent","family":"vals-ai:vcb-1-100:overall","index":75.7173083047748,"aboveFit":false,"run":"OpenHands","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vcb-1-100","measuredAt":"2026-10-29","stderr":4.488,"costPerTest":149.664612,"latency":10359.295},{"benchmarkId":"roboDojoSim","benchmarkName":"RoboDojo Sim","category":"agentic","score":27.6,"unit":"percent","family":"roboDojoSim","index":null,"aboveFit":false,"run":null,"source":"Anthropic (reported run)","sourceUrl":"https://www.anthropic.com/claude-fable-and-mythos-5-2","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaGdpPdf","benchmarkName":"Artificial Analysis GDP.pdf","category":"agentic","score":26.2,"unit":"percent","family":"aaGdpPdf","index":73.47438024404812,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdp-pdf","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaCyberIndex","benchmarkName":"Artificial Analysis Cyber Index","category":"agentic","score":25,"unit":"percent","family":"aaCyberIndex","index":52.912530758910655,"aboveFit":false,"run":null,"source":"Artificial Analysis with the Cyber Index Alliance (Collinear AI, IBM, NVIDIA, Pasta Lab, Vercel)","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-cyber-index","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"toolathlonVerifiedAvgTurns","benchmarkName":"Toolathlon Verified average assistant turns","category":"agentic","score":23.7,"unit":"percent","family":"toolathlonVerifiedAvgTurns","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaDeepsecBench","benchmarkName":"Artificial Analysis DeepsecBench-AA","category":"agentic","score":23.5,"unit":"percent","family":"aaDeepsecBench","index":68.3613094470664,"aboveFit":false,"run":null,"source":"Artificial Analysis; dataset by Vercel","sourceUrl":"https://artificialanalysis.ai/evaluations/deepsecbench-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:srebench:overall","benchmarkName":"SRE Bench","category":null,"domain":"Other","eligible":false,"ranked":true,"score":22.901,"unit":"percent","family":"vals-ai:srebench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/srebench","measuredAt":"2026-11-05","stderr":2.601,"costPerTest":32.231825,"latency":7375.578},{"benchmarkId":"rliAutomationRate","benchmarkName":"Remote Labor Index (RLI)","category":"agentic","score":17.9,"unit":"percent","family":"rliAutomationRate","index":null,"aboveFit":false,"run":null,"source":"Center for AI Safety; Scale AI","sourceUrl":"https://scale.com/leaderboard/rli","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaMlcr","benchmarkName":"Medical Long Context Reasoning (MLCR-AA)","category":"reasoning","score":15.6,"unit":"percent","family":"aaMlcr","index":60.82993780443897,"aboveFit":false,"run":null,"source":"Wisedocs and Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mlcr-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lmarena:agent-steerability:overall","benchmarkName":"Agent Arena steerability","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":14.800582050233018,"unit":"index","family":"agent-arena","index":78.69920006282734,"aboveFit":false,"run":"max effort","source":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset","measuredAt":"2026-11-11","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lmarena:agent-task-outcome:overall","benchmarkName":"Agent Arena task outcome","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":14.435790771670332,"unit":"index","family":"agent-arena","index":78.39928444303575,"aboveFit":false,"run":"max effort","source":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset","measuredAt":"2026-11-11","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:cua_bench:overall","benchmarkName":"CUA-bench","category":null,"domain":"Other","eligible":false,"ranked":true,"score":13.166667,"unit":"percent","family":"vals-ai:cua_bench:overall","index":null,"aboveFit":false,"run":"claude-code","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/cua_bench","measuredAt":"2026-10-13","stderr":0,"costPerTest":323.776608,"latency":37.44},{"benchmarkId":"lmarena:agent-bash-recovery:overall","benchmarkName":"Agent Arena command recovery","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":11.603170489005432,"unit":"index","family":"agent-arena","index":76.0704264970426,"aboveFit":false,"run":"max effort","source":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset","measuredAt":"2026-10-08","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mazeBench","benchmarkName":"MazeBench: visual spatial reasoning in a 3D open world","category":"agentic","score":10,"unit":"percent","family":"mazeBench","index":80.17850688913381,"aboveFit":false,"run":null,"source":"Jonathan Pappas and David Pappas","sourceUrl":"https://mazebench.com/blog?post=maze-bench-results","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:programbench:overall","benchmarkName":"ProgramBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":7,"unit":"percent","family":"vals-ai:programbench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/programbench","measuredAt":"2026-11-09","stderr":1.809,"costPerTest":58.51711,"latency":8791.194},{"benchmarkId":"aaHarveyLabV11","benchmarkName":"Harvey LAB-AA v1.1","category":"agentic","score":6.4,"unit":"percent","family":"aaHarveyLabV11","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-spireclimb-slay-the-spire-2-ascension-10:overall","benchmarkName":"SpireClimb (Slay the Spire 2, Ascension 10)","category":null,"domain":"Other","eligible":false,"ranked":true,"score":4,"unit":"percent","family":"epoch-ai:epoch-spireclimb-slay-the-spire-2-ascension-10:overall","index":null,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.2779791898059973,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-spireclimb:overall","benchmarkName":"SpireClimb","category":null,"domain":"Other","eligible":false,"ranked":true,"score":2,"unit":"percent","family":"epoch-ai:epoch-spireclimb:overall","index":null,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.1389895949029987,"costPerTest":null,"latency":null},{"benchmarkId":"aaCyberGymE2e","benchmarkName":"Artificial Analysis CyberGym-E2E-AA","category":"agentic","score":0.8,"unit":"percent","family":"aaCyberGymE2e","index":5,"aboveFit":false,"run":null,"source":"Artificial Analysis; dataset by Berkeley RDI","sourceUrl":"https://artificialanalysis.ai/evaluations/cybergym-e2e-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-erdos:overall","benchmarkName":"FrontierMath-Erdos","category":"math","domain":null,"eligible":true,"ranked":true,"score":0,"unit":"percent","family":"epoch-ai:epoch-frontiermath-erdos:overall","index":null,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-spireclimb-slay-the-spire-1-ascension-20-heart:overall","benchmarkName":"SpireClimb (Slay the Spire 1, Ascension 20 Heart)","category":null,"domain":"Other","eligible":false,"ranked":true,"score":0,"unit":"percent","family":"epoch-ai:epoch-spireclimb-slay-the-spire-1-ascension-20-heart:overall","index":null,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":10,"outputPerMillion":50,"batch":{"inputPerMillion":5,"outputPerMillion":25},"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":69,"timeToFirstToken":3.007,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":{"elo":1512.3784035606031,"eloUpper":1517.9161209749818,"eloLower":1506.840686146225,"votes":20087,"rank":4,"rankUpper":null,"rankLower":null,"arenaName":"Claude Fable 5.1-max","organization":"anthropic","updatedAt":"2026-11-09"},"quality":{"coveragePercent":95,"benchmarkCount":71,"categoryCount":7,"confidence":"high","state":"available"},"hosts":[{"host":"amazon-bedrock","name":"Amazon Bedrock","kind":"cloud","url":"https://docs.aws.amazon.com/bedrock/latest/userguide/models-supported.html","inputPerMillion":10,"outputPerMillion":50,"cacheReadPerMillion":0.25,"contextTokens":1000000,"maxOutputTokens":128000,"deprecated":false},{"host":"anthropic","name":"Anthropic","kind":"maker","url":"https://docs.anthropic.com/en/docs/about-claude/models","inputPerMillion":10,"outputPerMillion":50,"cacheReadPerMillion":0.25,"contextTokens":1000000,"maxOutputTokens":128000,"deprecated":false},{"host":"digitalocean","name":"DigitalOcean","kind":"cloud","url":"https://docs.digitalocean.com/products/gradient-ai-platform/details/models/","inputPerMillion":10,"outputPerMillion":50,"cacheReadPerMillion":0.25,"contextTokens":1000000,"maxOutputTokens":128000,"deprecated":false},{"host":"google-vertex","name":"Google Vertex AI","kind":"cloud","url":"https://cloud.google.com/vertex-ai/generative-ai/docs/models","inputPerMillion":10,"outputPerMillion":50,"cacheReadPerMillion":0.25,"contextTokens":1000000,"maxOutputTokens":128000,"deprecated":false},{"host":"azure","name":"Microsoft Azure","kind":"cloud","url":"https://learn.microsoft.com/en-us/azure/ai-services/openai/concepts/models","inputPerMillion":10,"outputPerMillion":50,"cacheReadPerMillion":0.25,"contextTokens":1000000,"maxOutputTokens":128000,"deprecated":false},{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":10,"outputPerMillion":50,"cacheReadPerMillion":0.25,"contextTokens":1000000,"maxOutputTokens":128000,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:livebench","name":"LiveBench","url":"https://livebench.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:arc-prize-foundation","name":"ARC Prize Foundation","url":"https://arcprize.org","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:terminal-bench","name":"Terminal-Bench","url":"https://www.tbench.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:lmarena","name":"LMArena","url":"https://huggingface.co","license":"CC BY 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"kimi-3-1-pro","providerId":"moonshot-ai","slug":"kimi-3-1-pro","name":"Kimi K3.1 Pro","provider":"Moonshot AI","description":"Kimi K3.1 Pro is a reasoning model from Moonshot AI in the Kimi K3.1 family. 35 benchmarks count toward its index, in 5 categories.","logoUrl":null,"releaseDate":"2026-10-23","contextWindowTokens":1050000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":true,"overallScore":75.70562584326319,"overallScoreRange":{"low":70.97239698347117,"high":80.43885470305521},"badges":["open"],"capabilityScores":{"agentic":76.7335267414424,"coding":71.85910527976644,"reasoning":77.73724702440919,"multimodal":71.98241651750192,"knowledge":70.99424940993563,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"agentic":21,"knowledge":3,"multimodal":5,"reasoning":3,"coding":3},"benchmarks":[{"benchmarkId":"babyVisionPython","benchmarkName":"BabyVision with Python","category":"multimodal","score":95.4,"unit":"percent","family":"babyVisionPython","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaCyberGymE2e","benchmarkName":"Artificial Analysis CyberGym-E2E-AA","category":"agentic","score":94.7,"unit":"percent","family":"aaCyberGymE2e","index":95,"aboveFit":true,"run":null,"source":"Artificial Analysis; dataset by Berkeley RDI","sourceUrl":"https://artificialanalysis.ai/evaluations/cybergym-e2e-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gpqa","benchmarkName":"Graduate-Level Google-Proof Q&A","category":"knowledge","score":94.2,"unit":"percent","family":"gpqa-diamond","index":63.60676671529328,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gpqaDiamond","benchmarkName":"GPQA Diamond","category":"knowledge","score":94.2,"unit":"percent","family":"gpqa-diamond","index":63.60676671529328,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"charxiv","benchmarkName":"CharXiv Reasoning","category":"multimodal","score":93.4,"unit":"percent","family":"charxiv","index":70.51995397932768,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"benchCadVision2CodeWithTools","benchmarkName":"BenchCAD Vision2Code voxel IoU with tools","category":"multimodal","score":92.7,"unit":"percent","family":"benchCadVision2CodeWithTools","index":null,"aboveFit":false,"run":null,"source":"Zhang et al. and Anthropic","sourceUrl":"https://arxiv.org/abs/2605.10865","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"browseComp","benchmarkName":"BrowseComp","category":"agentic","score":92.1,"unit":"percent","family":"browseComp","index":72.66981818313499,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"chartographyWithTools","benchmarkName":"Chartography with image and code tools","category":"multimodal","score":90.2,"unit":"percent","family":"chartographyWithTools","index":null,"aboveFit":false,"run":null,"source":"Surge AI and Anthropic","sourceUrl":"https://surgehq.ai/blog/chartography","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmmuProPython","benchmarkName":"MMMU-Pro with Python","category":"multimodal","score":90,"unit":"percent","family":"mmmuProPython","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"charxivNoTools","benchmarkName":"CharXiv Reasoning without tools","category":"multimodal","score":89,"unit":"percent","family":"charxivNoTools","index":71.17221425583207,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":86.3,"unit":"percent","family":"lcr","index":66.50679881721318,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gdpPdfPythonGeminiJudge","benchmarkName":"GDP.pdf task-macro rubric pass rate with Python, Gemini judge","category":"multimodal","score":85.7,"unit":"percent","family":"gdpPdfPythonGeminiJudge","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmmuPro","benchmarkName":"Massive Multi-discipline Multimodal Understanding Pro","category":"multimodal","score":84.1,"unit":"percent","family":"mmmu-pro","index":64.60158652435746,"aboveFit":false,"run":null,"source":"MMMU-Pro authors","sourceUrl":"https://arxiv.org/abs/2409.02813","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaMmmuPro","benchmarkName":"Artificial Analysis MMMU-Pro","category":"multimodal","score":83.9,"unit":"percent","family":"mmmu-pro","index":64.34130407133364,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmmu-pro","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"bioMysteryBench","benchmarkName":"BioMysteryBench (overall, v11 release)","category":"agentic","score":80.6,"unit":"percent","family":"bioMysteryBench","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"osWorld2","benchmarkName":"OSWorld 2.0","category":"agentic","score":79.6,"unit":"percent","family":"osWorld2","index":86.25594039849611,"aboveFit":false,"run":null,"source":"Mengqi Yuan, Zilong Zhou, Xinzhuang Xiong, Weiming Wu, Jiayang Sun, Jiamin Song, Kaiqian Cui, Bowen Wang, Haoyuan Wu, Yitong Li, Dunjie Lu, Haikong Lu, Qi Zhen, Xinyuan Wang, Jiaqi Deng, Yuhao Yang, Cheng Chen, Boyuan Zheng, Alex Su, Xiao Yu, Hao Zou, Saaket Agashe, Xing Han Lu, Manpreet Kaur, Zhengyang Qi, Vincent Sunn Chen, Frederic Sala, Dayiheng Liu, Junyang Lin, Zhou Yu, Yu Su, Siva Reddy, Xin Eric Wang, Peng Qi, Tianbao Xie, Tao Yu","sourceUrl":"https://arxiv.org/abs/2606.29537","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"osWorld21","benchmarkName":"OSWorld 2.1 (all 108 tasks)","category":"agentic","score":79.6,"unit":"percent","family":"osWorld21","index":77.19903191238438,"aboveFit":false,"run":null,"source":"Mengqi Yuan, Zilong Zhou, Xinzhuang Xiong, Weiming Wu, Jiayang Sun, Jiamin Song, Kaiqian Cui, Bowen Wang, Haoyuan Wu, Yitong Li, Dunjie Lu, Haikong Lu, Qi Zhen, Xinyuan Wang, Jiaqi Deng, Yuhao Yang, Cheng Chen, Boyuan Zheng, Alex Su, Xiao Yu, Hao Zou, Saaket Agashe, Xing Han Lu, Manpreet Kaur, Zhengyang Qi, Vincent Sunn Chen, Frederic Sala, Dayiheng Liu, Junyang Lin, Zhou Yu, Yu Su, Siva Reddy, Xin Eric Wang, Peng Qi, Tianbao Xie, Tao Yu","sourceUrl":"https://github.com/xlang-ai/OSWorld-V2/releases","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"kimiCodeBenchV2","benchmarkName":"Kimi Code Bench v2","category":"coding","score":77.1,"unit":"percent","family":"kimiCodeBenchV2","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"deepSwe","benchmarkName":"DeepSWE","category":"agentic","score":75.5,"unit":"percent","family":"deepSwe","index":75.96311297440974,"aboveFit":false,"run":null,"source":"Datacurve AI","sourceUrl":"https://deepswe.datacurve.ai/blog","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaAutomationBench","benchmarkName":"Artificial Analysis AutomationBench","category":"agentic","score":73.3,"unit":"percent","family":"aaAutomationBench","index":82.5105699928617,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/automationbench-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"benchCadVision2Code","benchmarkName":"BenchCAD Vision2Code voxel IoU without tools","category":"multimodal","score":72.8,"unit":"percent","family":"benchCadVision2Code","index":null,"aboveFit":false,"run":null,"source":"Zhang et al. and Anthropic","sourceUrl":"https://arxiv.org/abs/2605.10865","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"kimiFinancialResearch","benchmarkName":"Financial Research (Internal)","category":"agentic","score":70.4,"unit":"percent","family":"kimiFinancialResearch","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hleWithTools","benchmarkName":"Humanity's Last Exam with tools","category":"agentic","score":70,"unit":"percent","family":"hleWithTools","index":77.08323021767217,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hle","benchmarkName":"Humanity's Last Exam","category":"knowledge","score":70,"unit":"percent","family":"hle","index":94.61713646539908,"aboveFit":true,"run":null,"source":"Center for AI Safety, Scale AI, and thousands of expert contributors","sourceUrl":"https://lastexam.ai/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"apexAgents","benchmarkName":"APEX-Agents","category":"agentic","score":69.4,"unit":"percent","family":"apexAgents","index":81.02402770704057,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"perceptionBench","benchmarkName":"PerceptionBench (Internal)","category":"multimodal","score":68.3,"unit":"percent","family":"perceptionBench","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"officeQaPro","benchmarkName":"OfficeQA Pro","category":"multimodal","score":67.4,"unit":"percent","family":"officeQaPro","index":76.79836488754722,"aboveFit":false,"run":null,"source":"OfficeQA Pro authors","sourceUrl":"https://arxiv.org/abs/2603.08655","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gdpvalAaNormalized","benchmarkName":"GDPval-AA normalized","category":"agentic","score":67.3,"unit":"percent","family":"gdpvalAaNormalized","index":82.15171097905306,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaTerminalBench4","benchmarkName":"Artificial Analysis Terminal-Bench v4.0","category":"agentic","score":67.2,"unit":"percent","family":"aaTerminalBench4","index":83.20358820994664,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/terminalbench-v4-0","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"officeQaProV2","benchmarkName":"OfficeQA Pro v2","category":"multimodal","score":64.4,"unit":"percent","family":"officeQaProV2","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaCyberIndex","benchmarkName":"Artificial Analysis Cyber Index","category":"agentic","score":64.1,"unit":"percent","family":"aaCyberIndex","index":93.02661196380174,"aboveFit":true,"run":null,"source":"Artificial Analysis with the Cyber Index Alliance (Collinear AI, IBM, NVIDIA, Pasta Lab, Vercel)","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-cyber-index","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"zeroBenchPython","benchmarkName":"ZeroBench_main with Python","category":"multimodal","score":63,"unit":"percent","family":"zeroBenchPython","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"kimiCodeBench3LongHorizon","benchmarkName":"Kimi Code Bench 3.0 Long-horizon (Internal)","category":"coding","score":61.2,"unit":"percent","family":"kimiCodeBench3LongHorizon","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaCweBench","benchmarkName":"Artificial Analysis CWE-Bench-AA","category":"agentic","score":60.8,"unit":"percent","family":"aaCweBench","index":72.55904309654791,"aboveFit":false,"run":null,"source":"Artificial Analysis; dataset by Collinear AI","sourceUrl":"https://artificialanalysis.ai/evaluations/cwe-bench-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaSciCode","benchmarkName":"Artificial Analysis SciCode","category":"coding","score":60.8,"unit":"percent","family":"scicode-family","index":75.10422014452703,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/scicode","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"frontierFinance","benchmarkName":"Frontier Finance","category":"agentic","score":58.6,"unit":"percent","family":"frontierFinance","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"kimiFinancialModeling","benchmarkName":"Financial Modeling (Internal)","category":"agentic","score":57.6,"unit":"percent","family":"kimiFinancialModeling","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":52.5,"unit":"percent","family":"hle","index":77.05951342737768,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"sweMarathon","benchmarkName":"SWE-Marathon","category":"coding","score":51.7,"unit":"percent","family":"sweMarathon","index":null,"aboveFit":false,"run":null,"source":"Abundant AI and BenchFlow","sourceUrl":"https://www.swe-marathon.org/","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hleNoTools","benchmarkName":"Humanity's Last Exam without tools","category":"knowledge","score":51.1,"unit":"percent","family":"hle","index":75.65490358433597,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"omniscienceAccuracy","benchmarkName":"Artificial Analysis Omniscience Accuracy","category":"knowledge","score":48.9,"unit":"percent","family":"omniscienceAccuracy","index":72.31646808713592,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"cweBenchV2","benchmarkName":"CWE-bench v2","category":"agentic","score":45.3,"unit":"percent","family":"cweBenchV2","index":62.19074783836952,"aboveFit":false,"run":null,"source":"Collinear AI","sourceUrl":"https://cwe-bench.com/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"handbookMd","benchmarkName":"HANDBOOK.md","category":"agentic","score":44.6,"unit":"percent","family":"handbookMd","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"agentsLastExamCli","benchmarkName":"Agents' Last Exam v1.1, CLI subset","category":"agentic","score":44.2,"unit":"percent","family":"agentsLastExamCli","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mka","benchmarkName":"Multimodal Knowledge Agent Benchmark","category":"agentic","score":43.9,"unit":"percent","family":"mka","index":72.67598506685118,"aboveFit":false,"run":null,"source":"Google DeepMind, in partnership with Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mka","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"agentsLastExam","benchmarkName":"Agents' Last Exam","category":"agentic","score":43.2,"unit":"percent","family":"agentsLastExam","index":76.92155204124535,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"zeroBench","benchmarkName":"ZeroBench","category":"multimodal","score":39,"unit":"percent","family":"zeroBench","index":76.95010416695709,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaAnalystAgent","benchmarkName":"Artificial Analysis AnalystAgent","category":"agentic","score":38.8,"unit":"percent","family":"aaAnalystAgent","index":59.353993343245605,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/aa-analyst-agent","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaMlcr","benchmarkName":"Medical Long Context Reasoning (MLCR-AA)","category":"reasoning","score":38.3,"unit":"percent","family":"aaMlcr","index":82.23309665244882,"aboveFit":false,"run":null,"source":"Wisedocs and Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mlcr-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"terminalBenchScience","benchmarkName":"Terminal-Bench-Science 0.1","category":"agentic","score":38.1,"unit":"percent","family":"terminalBenchScience","index":66.46067505914944,"aboveFit":false,"run":null,"source":"Terminal-Bench-Science Team","sourceUrl":"https://www.terminal-bench-science.ai/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"kimiBusinessOperation","benchmarkName":"Business Operation (Internal)","category":"agentic","score":37.8,"unit":"percent","family":"kimiBusinessOperation","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaDeepsecBench","benchmarkName":"Artificial Analysis DeepsecBench-AA","category":"agentic","score":36.7,"unit":"percent","family":"aaDeepsecBench","index":78.51543015475806,"aboveFit":false,"run":null,"source":"Artificial Analysis; dataset by Vercel","sourceUrl":"https://artificialanalysis.ai/evaluations/deepsecbench-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"postTrainBench12","benchmarkName":"PostTrainBench v1.2","category":"coding","score":36.5,"unit":"percent","family":"postTrainBench12","index":null,"aboveFit":false,"run":null,"source":"PostTrainBench authors","sourceUrl":"https://posttrainbench.com/","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"bug-hunt-bench","benchmarkName":"Bug Hunt Bench","category":"coding","score":36,"unit":"percent","family":"bug-hunt-bench","index":73.91484376155799,"aboveFit":false,"run":null,"source":"Pawel Huryn","sourceUrl":"https://bughunt.productcompass.pm/method","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gdpPdfGeminiJudge","benchmarkName":"GDP.pdf strict pass rate, page images plus text, Gemini judge","category":"multimodal","score":34.6,"unit":"percent","family":"gdpPdfGeminiJudge","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaGdpPdf","benchmarkName":"Artificial Analysis GDP.pdf","category":"agentic","score":31.4,"unit":"percent","family":"aaGdpPdf","index":79.47348689931917,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdp-pdf","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":24.6,"unit":"percent","family":"critpt","index":84.47184560356554,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"harveyLegalAgentPublic","benchmarkName":"Harvey's Legal Agent Benchmark, public 1,251-task set, all-pass","category":"agentic","score":23.7,"unit":"percent","family":"harveyLegalAgentPublic","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"exploitGym","benchmarkName":"ExploitGym","category":"agentic","score":20.7,"unit":"percent","family":"exploitGym","index":71.54687027549818,"aboveFit":false,"run":null,"source":"Zhun Wang, Nico Schiller, Hongwei Li, Srijiith Sesha Narayana, Milad Nasr, Nicholas Carlini, Xiangyu Qi, Eric Wallace, Elie Bursztein, Luca Invernizzi, Kurt Thomas, Yan Shoshitaishvili, Wenbo Guo, Jingxuan He, Thorsten Holz, Dawn Song","sourceUrl":"https://arxiv.org/abs/2605.11086","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"frontierCode2","benchmarkName":"FrontierCode 2.0","category":"coding","score":19.6,"unit":"percent","family":"frontierCode2","index":66.5582519332143,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"kimiHalluBenchHardTools","benchmarkName":"Kimi Hallucination Benchmark Hard with tools (Internal)","category":"agentic","score":8.3,"unit":"percent","family":"kimiHalluBenchHardTools","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"kimiHalluBench","benchmarkName":"Kimi Hallucination Benchmark without tools (Internal)","category":"knowledge","score":5.4,"unit":"percent","family":"kimiHalluBench","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mazeBench","benchmarkName":"MazeBench: visual spatial reasoning in a 3D open world","category":"agentic","score":2,"unit":"percent","family":"mazeBench","index":65.61863525650502,"aboveFit":false,"run":null,"source":"Jonathan Pappas and David Pappas","sourceUrl":"https://mazebench.com/blog?post=maze-bench-results","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":2,"outputPerMillion":10,"batch":{"inputPerMillion":0.7,"outputPerMillion":6},"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":43,"timeToFirstToken":49.6,"source":"BenchLM","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":80,"benchmarkCount":35,"categoryCount":5,"confidence":"medium","state":"available"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gpt-5-5-pro","providerId":"openai","slug":"gpt-5-5-pro","name":"GPT-5.5 Pro","provider":"OpenAI","description":"GPT-5.5 Pro is a reasoning model from OpenAI in the GPT-5.5 family. 12 benchmarks count toward its index, in 4 categories.","logoUrl":null,"releaseDate":"2026-04-23","contextWindowTokens":1050000,"modalities":["file","image","text"],"openWeights":false,"details":{"apiId":"openai/gpt-5.5-pro","maxOutputTokens":128000,"knowledgeCutoff":"2025-12-01","retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":null,"weightsUrl":null},"reasoning":true,"overallScore":74.83748789356885,"overallScoreRange":{"low":66.16961096717073,"high":83.50536481996697},"badges":[],"capabilityScores":{"agentic":71.08977857113635,"coding":null,"reasoning":81.1337995758998,"multimodal":null,"knowledge":69.0264889577386,"multilingual":null,"instruction":null,"math":76.33467792121475},"capabilityCounts":{"math":5,"reasoning":4,"knowledge":2,"agentic":1},"benchmarks":[{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":100,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":67.09849544987334,"aboveFit":false,"run":"xhigh effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0,"costPerTest":null,"latency":null},{"benchmarkId":"arc-prize:v1-public-eval:overall","benchmarkName":"ARC-AGI-1 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":97.99,"unit":"percent","family":"arc-prize:v1-public-eval:overall","index":null,"aboveFit":false,"run":"xhigh effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":3.27,"latency":null},{"benchmarkId":"arc-prize:v1-semi-private:overall","benchmarkName":"ARC-AGI-1 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":95,"unit":"percent","family":"arc-prize:v1-semi-private:overall","index":70.66228864275845,"aboveFit":false,"run":"xhigh effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":4.52,"latency":null},{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":93.92135642135642,"unit":"percent","family":"gpqa-diamond","index":63.35120243891953,"aboveFit":false,"run":"xhigh effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.5832823810775416,"costPerTest":null,"latency":null},{"benchmarkId":"arc-prize:v2-public-eval:overall","benchmarkName":"ARC-AGI-2 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":90.42,"unit":"percent","family":"arc-prize:v2-public-eval:overall","index":null,"aboveFit":false,"run":"xhigh effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":11.1,"latency":null},{"benchmarkId":"browseComp","benchmarkName":"BrowseComp","category":"agentic","score":90.1,"unit":"percent","family":"browseComp","index":71.08977857113635,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tiers-1-3-v2-private:overall","benchmarkName":"FrontierMath-Tiers-1-3-v2-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":87.71929824561403,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tiers-1-3-v2-private:overall","index":77.06589905082447,"aboveFit":false,"run":"xhigh effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.9476010341530077,"costPerTest":null,"latency":null},{"benchmarkId":"arc-prize:v2-semi-private:overall","benchmarkName":"ARC-AGI-2 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":84.16,"unit":"percent","family":"arc-prize:v2-semi-private:overall","index":77.82929183639584,"aboveFit":false,"run":"xhigh effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":10.76,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tier-4-v2-private:overall","benchmarkName":"FrontierMath-Tier-4-v2-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":78.04878048780488,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tier-4-v2-private:overall","index":79.14977093959078,"aboveFit":false,"run":"xhigh effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":6.5445892024384085,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-chess-puzzles:overall","benchmarkName":"Chess Puzzles","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":62,"unit":"percent","family":"epoch-ai:epoch-chess-puzzles:overall","index":81.04361782444491,"aboveFit":false,"run":"xhigh effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":4.878317312145634,"costPerTest":null,"latency":null},{"benchmarkId":"hle","benchmarkName":"Humanity's Last Exam","category":"knowledge","score":57.2,"unit":"percent","family":"hle","index":81.77498932901773,"aboveFit":false,"run":null,"source":"Center for AI Safety, Scale AI, and thousands of expert contributors","sourceUrl":"https://lastexam.ai/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-2025-02-28-private:overall","benchmarkName":"FrontierMath-2025-02-28-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":51,"unit":"percent","family":"epoch-ai:epoch-frontiermath-2025-02-28-private:overall","index":74.93583230224641,"aboveFit":false,"run":"xhigh effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.9000000000000004,"costPerTest":null,"latency":null},{"benchmarkId":"hleNoTools","benchmarkName":"Humanity's Last Exam without tools","category":"knowledge","score":43.1,"unit":"percent","family":"hle","index":67.62856162409761,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-private:overall","benchmarkName":"FrontierMath-Tier-4-2025-07-01-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":39.6,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-private:overall","index":83.42339186353868,"aboveFit":false,"run":"xhigh effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":7.1,"costPerTest":null,"latency":null},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":30.6,"unit":"percent","family":"critpt","index":95,"aboveFit":true,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":30,"outputPerMillion":180,"batch":{"inputPerMillion":15,"outputPerMillion":90},"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":19,"timeToFirstToken":9.148,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":60,"benchmarkCount":12,"categoryCount":4,"confidence":"low","state":"available"},"hosts":[{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":30,"outputPerMillion":180,"cacheReadPerMillion":null,"contextTokens":1050000,"maxOutputTokens":128000,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:arc-prize-foundation","name":"ARC Prize Foundation","url":"https://arcprize.org","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"claude-fable-5","providerId":"anthropic","slug":"claude-fable","name":"Claude Fable 5","provider":"Anthropic","description":"Claude Fable 5 is a reasoning model from Anthropic in the Claude Fable family. 55 benchmarks count toward its index, in 7 categories.","logoUrl":null,"releaseDate":"2026-06-09","contextWindowTokens":1000000,"modalities":["text","image","file"],"openWeights":false,"details":{"apiId":"anthropic/claude-fable-5","maxOutputTokens":128000,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":1,"weightsUrl":null},"reasoning":true,"overallScore":73.99912481115072,"overallScoreRange":{"low":71.3187624559446,"high":76.67948716635685},"badges":[],"capabilityScores":{"agentic":71.3525734023447,"coding":74.73464763101724,"reasoning":73.64170230902674,"multimodal":70.06826131247752,"knowledge":75.83419078529154,"multilingual":null,"instruction":64.13465255767234,"math":76.21273530619132},"capabilityCounts":{"math":5,"agentic":16,"reasoning":9,"coding":14,"knowledge":6,"multimodal":3,"instruction":2},"benchmarks":[{"benchmarkId":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-public:overall","benchmarkName":"FrontierMath-Tier-4-2025-07-01-Public","category":"math","domain":null,"eligible":false,"ranked":true,"score":100,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-public:overall","index":null,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-2025-02-28-public:overall","benchmarkName":"FrontierMath-2025-02-28-Public","category":"math","domain":null,"eligible":false,"ranked":true,"score":100,"unit":"percent","family":"epoch-ai:epoch-frontiermath-2025-02-28-public:overall","index":null,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":99.72222222222223,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":66.93688599272527,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0.2777777777777778,"costPerTest":null,"latency":null},{"benchmarkId":"tau2Bench","benchmarkName":"τ²-Bench Tool-Agent-User Evaluation","category":"agentic","score":98.5,"unit":"percent","family":"tau2-bench","index":68.15139166961706,"aboveFit":false,"run":null,"source":"Victor Barres, Honghua Dong, Soham Ray, Xujie Si, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2506.07982","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"arc-prize:v1-semi-private:overall","benchmarkName":"ARC-AGI-1 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":98.5,"unit":"percent","family":"arc-prize:v1-semi-private:overall","index":72.27136500316884,"aboveFit":false,"run":"max effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":2.1101577000000007,"latency":null},{"benchmarkId":"livebench:livebench-coding-2:overall","benchmarkName":"LiveBench Coding 2.0","category":"coding","domain":null,"eligible":true,"ranked":true,"score":98.3,"unit":"percent","family":"livebench:livebench-coding-2:overall","index":79.37307385006645,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Coding","measuredAt":"2026-10-14","stderr":null,"costPerTest":8.927262068965517,"latency":null},{"benchmarkId":"arc-prize:v1-public-eval:overall","benchmarkName":"ARC-AGI-1 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":97.625,"unit":"percent","family":"arc-prize:v1-public-eval:overall","index":null,"aboveFit":false,"run":"max effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":1.4118586999999994,"latency":null},{"benchmarkId":"arc-prize:v2-public-eval:overall","benchmarkName":"ARC-AGI-2 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":96.38888888888889,"unit":"percent","family":"arc-prize:v2-public-eval:overall","index":null,"aboveFit":false,"run":"max effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":5.6868593333333335,"latency":null},{"benchmarkId":"livebench:livebench-math:overall","benchmarkName":"LiveBench Mathematics","category":"math","domain":null,"eligible":true,"ranked":true,"score":95.98575,"unit":"percent","family":"livebench:livebench-math:overall","index":73.55201592816199,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Mathematics","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.7513469656992083,"latency":null},{"benchmarkId":"sweVerified","benchmarkName":"Software Engineering Benchmark Verified","category":"coding","score":95,"unit":"percent","family":"swe-bench-verified","index":71.29589741892238,"aboveFit":false,"run":null,"source":"Carlos E. Jimenez, John Yang, Alexander Wettig, Shunyu Yao, Kexin Pei, Ofir Press, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2310.06770","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:proof_bench:overall","benchmarkName":"ProofBench v1.1","category":"math","domain":null,"eligible":true,"ranked":true,"score":95,"unit":"percent","family":"vals-ai:proof_bench:overall","index":79.95486718487626,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/proof_bench","measuredAt":"2026-11-09","stderr":2.19,"costPerTest":5.229282,"latency":822.739},{"benchmarkId":"vals-ai:swebench:overall","benchmarkName":"SWE-bench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":95,"unit":"percent","family":"swe-bench-verified","index":71.29589741892238,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/swebench","measuredAt":"2026-09-01","stderr":0.976,"costPerTest":2.046736,"latency":356.182},{"benchmarkId":"aaHarveyLab","benchmarkName":"Artificial Analysis Harvey LAB-AA v1.0","category":"agentic","score":93.6,"unit":"percent","family":"aaHarveyLab","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:gpqa:overall","benchmarkName":"GPQA Diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":93.182,"unit":"percent","family":"gpqa-diamond","index":62.67308493222494,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","measuredAt":"2026-09-01","stderr":1.939,"costPerTest":0.450473,"latency":99.902},{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":92.6,"unit":"percent","family":"gpqa-diamond","index":62.139290435421614,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-browser:overall","benchmarkName":"LiveBench Browser Use","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":92,"unit":"percent","family":"livebench-2-agentic","index":85.87193231505225,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Browser","measuredAt":"2026-10-14","stderr":null,"costPerTest":10.301584,"latency":null},{"benchmarkId":"vals-ai:mmlu_pro:overall","benchmarkName":"MMLU Pro","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":91.502,"unit":"percent","family":"mmlu-pro","index":59.762295919087826,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","measuredAt":"2026-09-01","stderr":0.278,"costPerTest":0.082276,"latency":24.998},{"benchmarkId":"livebench:livebench-language:overall","benchmarkName":"LiveBench Language","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":90.68400000000001,"unit":"percent","family":"livebench:livebench-language:overall","index":77.10508128607692,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Language","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.44591333333333333,"latency":null},{"benchmarkId":"vals-ai:vibe-code:overall","benchmarkName":"Vibe Code Bench v1.1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":90.352,"unit":"percent","family":"vals-ai:vibe-code:overall","index":74.84994372734594,"aboveFit":false,"run":"OpenHands","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","measuredAt":"2026-11-09","stderr":2.097,"costPerTest":41.714694,"latency":3711.026},{"benchmarkId":"epoch-ai:epoch-frontiermath-tier-4-v2-private:overall","benchmarkName":"FrontierMath-Tier-4-v2-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":90.2,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tier-4-v2-private:overall","index":83.92337799633262,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":4.6,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:lcb:overall","benchmarkName":"LiveCodeBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":89.778,"unit":"percent","family":"livecodebench","index":63.8362745609067,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","measuredAt":"2026-09-01","stderr":0.892,"costPerTest":0.42816,"latency":118.53},{"benchmarkId":"livebench:livebench-reasoning:overall","benchmarkName":"LiveBench Reasoning","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":89.65375,"unit":"percent","family":"livebench","index":77.76023696744375,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Reasoning","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.6023960396039604,"latency":null},{"benchmarkId":"vulcanBench","benchmarkName":"VulcanBench v3","category":"coding","score":89.5,"unit":"percent","family":"vulcanBench","index":71.7640549295391,"aboveFit":false,"run":null,"source":"VulcanBench contributors","sourceUrl":"https://github.com/morganlinton/VulcanBench/tree/main","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:mmmu:overall","benchmarkName":"MMMU Pro","category":"multimodal","domain":null,"eligible":true,"ranked":true,"score":89.306,"unit":"percent","family":"mmmu-pro","index":71.37673877656783,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmmu","measuredAt":"2026-09-01","stderr":0.743,"costPerTest":0.16064,"latency":61.44},{"benchmarkId":"arc-prize:v2-semi-private:overall","benchmarkName":"ARC-AGI-2 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":89.16666666666667,"unit":"percent","family":"arc-prize:v2-semi-private:overall","index":80.12802623928313,"aboveFit":false,"run":"max effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":5.449076000000001,"latency":null},{"benchmarkId":"vals-ai:legal_bench:overall","benchmarkName":"LegalBench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":88.561,"unit":"percent","family":"vals-ai:legal_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","measuredAt":"2026-10-13","stderr":0.331,"costPerTest":0.030639,"latency":8.964},{"benchmarkId":"vals-ai:medscribe:overall","benchmarkName":"MedScribe","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":88.522,"unit":"percent","family":"vals-ai:medscribe:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medscribe","measuredAt":"2026-10-29","stderr":1.945,"costPerTest":0.583239,"latency":119.475},{"benchmarkId":"epoch-ai:epoch-frontiermath-tiers-1-3-v2-private:overall","benchmarkName":"FrontierMath-Tiers-1-3-v2-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":87.01754385964912,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tiers-1-3-v2-private:overall","index":76.69652942886046,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.9944477920133024,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-research:overall","benchmarkName":"LiveBench Research","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":86.9,"unit":"percent","family":"livebench-2-agentic","index":82.14214373095791,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Research","measuredAt":"2026-10-14","stderr":null,"costPerTest":17.274277272727275,"latency":null},{"benchmarkId":"livebench:livebench-coding:overall","benchmarkName":"LiveBench Coding","category":"coding","domain":null,"eligible":true,"ranked":true,"score":85.992,"unit":"percent","family":"livebench:livebench-coding:overall","index":75.76395967771005,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Coding","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.21157264957264954,"latency":null},{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":85.85858585858585,"unit":"percent","family":"gpqa-diamond","index":55.956249594598745,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.4825909793343355,"costPerTest":null,"latency":null},{"benchmarkId":"osWorldVerified","benchmarkName":"OSWorld-Verified","category":"agentic","score":85,"unit":"percent","family":"osWorldVerified","index":69.09786078177461,"aboveFit":false,"run":null,"source":"Tianbao Xie, Danyang Zhang, Jixuan Chen, Xiaochuan Li, Siheng Zhao, Ruisheng Cao, Toh Jing Hua, Zhoujun Cheng, Dongchan Shin, Fangyu Lei, Yitao Liu, Yiheng Xu, Shuyan Zhou, Silvio Savarese, Caiming Xiong, Victor Zhong, Tao Yu","sourceUrl":"https://os-world.github.io/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-design:overall","benchmarkName":"LiveBench Design","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":84.7,"unit":"percent","family":"livebench-2-agentic","index":80.53321532213292,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Design","measuredAt":"2026-10-14","stderr":null,"costPerTest":8.549472,"latency":null},{"benchmarkId":"aaTerminalBench21","benchmarkName":"Artificial Analysis Terminal-Bench v2.1","category":"agentic","score":84.6,"unit":"percent","family":"aaTerminalBench21","index":67.20045463795731,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/terminalbench-v2-1","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"terminalBench21","benchmarkName":"Terminal-Bench 2.1","category":"agentic","score":84.3,"unit":"percent","family":"terminal-bench-2-1","index":70.44442843242702,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":82.3,"unit":"percent","family":"lcr","index":63.76247671487054,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-data-analysis:overall","benchmarkName":"LiveBench Data Analysis","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":80.53766666666667,"unit":"percent","family":"livebench","index":65.95095270573873,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Data%20Analysis","measuredAt":"2026-10-14","stderr":null,"costPerTest":1.300688,"latency":null},{"benchmarkId":"vals-ai:terminal-bench-2-1:overall","benchmarkName":"Terminal-Bench 2.1","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":80.524,"unit":"percent","family":"terminal-bench-2-1","index":68.23601457617707,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2-1","measuredAt":"2026-10-01","stderr":1.35,"costPerTest":1.429025,"latency":504.537},{"benchmarkId":"swePro","benchmarkName":"SWE-bench Pro","category":"coding","score":80,"unit":"percent","family":"swePro","index":76.96818216895215,"aboveFit":false,"run":null,"source":"Xiang Deng, Jeff Da, Edwin Pan, Yannis Yiming He, Charles Ide, Kanak Garg, Niklas Lauffer, Andrew Park, Nitin Pasari, Chetan Rane, Karmini Sampath, Maya Krishnan, Srivatsa Kundurthy, Sean Hendryx, Zifan Wang, Chen Bo Calvin Zhang, Noah Jacobson, Bing Liu, Brad Kenstler","sourceUrl":"https://arxiv.org/abs/2509.16941","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:tax_eval_v2:overall","benchmarkName":"TaxEval v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":76.942,"unit":"percent","family":"vals-ai:tax_eval_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","measuredAt":"2026-09-01","stderr":0.821,"costPerTest":0.220133,"latency":56.865},{"benchmarkId":"livebench:livebench-instruction-following:overall","benchmarkName":"LiveBench Instruction Following","category":"instruction","domain":null,"eligible":true,"ranked":true,"score":75.77074999999999,"unit":"percent","family":"livebench:livebench-instruction-following:overall","index":79.90425168160635,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=IF","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.92417,"latency":null},{"benchmarkId":"vals-ai:emb:overall","benchmarkName":"EMB","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":73.666,"unit":"percent","family":"vals-ai:emb:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/emb","measuredAt":"2026-10-20","stderr":2.468,"costPerTest":12.354419,"latency":1594.153},{"benchmarkId":"vals-ai:ioi-v1:overall","benchmarkName":"IOI v1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":72.25,"unit":"percent","family":"vals-ai:ioi-v1:overall","index":76.08888811913693,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/ioi-v1","measuredAt":"2026-08-09","stderr":9.357,"costPerTest":15.101678,"latency":3318.969},{"benchmarkId":"vals-ai:corp_fin_v2:overall","benchmarkName":"CorpFin v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":71.834,"unit":"percent","family":"vals-ai:corp_fin_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","measuredAt":"2026-08-12","stderr":0.883,"costPerTest":1.737573,"latency":52.99},{"benchmarkId":"vals-ai:public-benefits-bench-v1:overall","benchmarkName":"Public Benefits Bench v1","category":null,"domain":"Public benefits","eligible":false,"ranked":true,"score":71.651,"unit":"percent","family":"vals-ai:public-benefits-bench-v1:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/public-benefits-bench-v1","measuredAt":"2026-06-09","stderr":1.172,"costPerTest":1.054998,"latency":1038.337},{"benchmarkId":"epoch-ai:epoch-simpleqa-verified:overall","benchmarkName":"SimpleQA Verified","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":70.7,"unit":"percent","family":"epoch-ai:epoch-simpleqa-verified:overall","index":82.81839358457754,"aboveFit":false,"run":"xhigh effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.4399942998441293,"costPerTest":null,"latency":null},{"benchmarkId":"cursorBench31","benchmarkName":"cursorBench31","category":"coding","score":70.6,"unit":"percent","family":"cursorBench31","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"cursorBench32","benchmarkName":"cursorBench32","category":"coding","score":70.5,"unit":"percent","family":"cursorBench32","index":73.76519869862952,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:public-benefits-bench:overall","benchmarkName":"Public Benefits Bench v1.1","category":null,"domain":"Public benefits","eligible":false,"ranked":true,"score":70.433,"unit":"percent","family":"vals-ai:public-benefits-bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/public-benefits-bench","measuredAt":"2026-10-12","stderr":1.187,"costPerTest":4.585446,"latency":1276.576},{"benchmarkId":"epoch-ai:epoch-mirrorcode:overall","benchmarkName":"MirrorCode","category":"coding","domain":null,"eligible":true,"ranked":true,"score":70,"unit":"percent","family":"epoch-ai:epoch-mirrorcode:overall","index":80.30762692707636,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":10.440053112638315,"costPerTest":null,"latency":null},{"benchmarkId":"parseBenchLlamaIndex","benchmarkName":"ParseBench: document parsing for AI agents (LlamaIndex leaderboard)","category":"multimodal","score":69.79,"unit":"percent","family":"parseBenchLlamaIndex","index":71.09377051180016,"aboveFit":false,"run":null,"source":"Boyang Zhang, Sebastián G. Acosta, Preston Carlson, Sacha Bron, Pierre-Loïc Doulcet, Daniel B. Ospina and Simon Suo (LlamaIndex)","sourceUrl":"https://arxiv.org/abs/2604.08538","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:mortgage_tax:overall","benchmarkName":"MortgageTax","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":68.919,"unit":"percent","family":"vals-ai:mortgage_tax:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mortgage_tax","measuredAt":"2026-09-01","stderr":0.909,"costPerTest":0.175013,"latency":16.096},{"benchmarkId":"omniscienceAccuracy","benchmarkName":"Artificial Analysis Omniscience Accuracy","category":"knowledge","score":65.4,"unit":"percent","family":"omniscienceAccuracy","index":93.11069182411826,"aboveFit":true,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaIfBench","benchmarkName":"Artificial Analysis IFBench","category":"instruction","score":63.5,"unit":"percent","family":"ifbench-family","index":48.365053433738346,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"terminalBenchHard","benchmarkName":"Terminal-Bench Hard","category":"agentic","score":62.9,"unit":"percent","family":"terminalBenchHard","index":70.72190918189499,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/terminalbench-hard","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-agentic-coding:overall","benchmarkName":"LiveBench Agentic Coding","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":62.17166666666666,"unit":"percent","family":"livebench:livebench-agentic-coding:overall","index":73.27927130799375,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Agentic%20Coding","measuredAt":"2026-06-25","stderr":null,"costPerTest":8.864393055555556,"latency":null},{"benchmarkId":"aaSciCode","benchmarkName":"Artificial Analysis SciCode","category":"coding","score":61,"unit":"percent","family":"scicode-family","index":75.36495837380998,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/scicode","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"officeQaPro","benchmarkName":"OfficeQA Pro","category":"multimodal","score":57.9,"unit":"percent","family":"officeQaPro","index":67.73427464906457,"aboveFit":false,"run":null,"source":"OfficeQA Pro authors","sourceUrl":"https://arxiv.org/abs/2603.08655","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:fabv2:overall","benchmarkName":"Finance Agent (v2)","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":56.314,"unit":"percent","family":"vals-ai:fabv2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/fabv2","measuredAt":"2026-10-29","stderr":0.84,"costPerTest":8.061563,"latency":610.952},{"benchmarkId":"vals-ai:medcode:overall","benchmarkName":"MedCode","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":56.07,"unit":"percent","family":"vals-ai:medcode:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medcode","measuredAt":"2026-10-29","stderr":2.203,"costPerTest":0.591071,"latency":91.439},{"benchmarkId":"gdpvalAaNormalized","benchmarkName":"GDPval-AA normalized","category":"agentic","score":55.5,"unit":"percent","family":"gdpvalAaNormalized","index":73.71498519108945,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":55.5,"unit":"percent","family":"hle","index":80.06939166246707,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:code-migration:overall","benchmarkName":"Code Migration","category":"coding","domain":null,"eligible":true,"ranked":true,"score":55.065,"unit":"percent","family":"vals-ai:code-migration:overall","index":74.44187320430291,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/code-migration","measuredAt":"2026-10-29","stderr":4.605,"costPerTest":112.098109,"latency":6559.388},{"benchmarkId":"frontierCode","benchmarkName":"FrontierCode 1.1 Main","category":"coding","score":53.5,"unit":"percent","family":"frontierCode","index":76.5864736123374,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-long-horizon:overall","benchmarkName":"LiveBench Long Agentic","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":53.2,"unit":"percent","family":"livebench-2-agentic","index":57.49628583213851,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Long%20Agentic","measuredAt":"2026-10-14","stderr":null,"costPerTest":59.73659444444445,"latency":null},{"benchmarkId":"epoch-ai:epoch-mystery-game-puzzles:overall","benchmarkName":"Mystery Game Puzzles","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":52,"unit":"percent","family":"epoch-ai:epoch-mystery-game-puzzles:overall","index":78.95057283073913,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":5.021167315686783,"costPerTest":null,"latency":null},{"benchmarkId":"aaEnterpriseOpsGym","benchmarkName":"Artificial Analysis EnterpriseOps-Gym","category":"agentic","score":51.1,"unit":"percent","family":"aaEnterpriseOpsGym","index":null,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/enterprise-ops-gym-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:legal_research:overall","benchmarkName":"Legal Research Bench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":49.519,"unit":"percent","family":"vals-ai:legal_research:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_research","measuredAt":"2026-11-09","stderr":3.475,"costPerTest":9.791426,"latency":1337.604},{"benchmarkId":"aaAnalystAgent","benchmarkName":"Artificial Analysis AnalystAgent","category":"agentic","score":48.8,"unit":"percent","family":"aaAnalystAgent","index":70.65496437702963,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/aa-analyst-agent","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"frontierSweV2","benchmarkName":"FrontierSWE v2","category":"coding","score":47,"unit":"percent","family":"frontierSweV2","index":75.87866156550551,"aboveFit":false,"run":null,"source":"Proximal","sourceUrl":"https://www.frontierswe.com/blog/v2","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-chess-puzzles:overall","benchmarkName":"Chess Puzzles","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":47,"unit":"percent","family":"epoch-ai:epoch-chess-puzzles:overall","index":69.65838989719346,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":5.016135580465917,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:web_search:overall","benchmarkName":"Vals Web Search Index","category":null,"domain":"Other","eligible":false,"ranked":true,"score":46.93805588941155,"unit":"percent","family":"vals-ai:web_search:overall","index":null,"aboveFit":false,"run":"Native","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/web_search","measuredAt":"2026-11-04","stderr":2.099818988794454,"costPerTest":4.368756,"latency":742.4488012222541},{"benchmarkId":"terminal-bench:terminal-bench-4-1-0:overall","benchmarkName":"Terminal-Bench 4.1.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":44.85,"unit":"percent","family":"terminal-bench:terminal-bench-4-1-0:overall","index":73.8831575713767,"aboveFit":false,"run":"max effort · Claude Code","source":"Terminal-Bench","sourceUrl":"https://www.tbench.ai/leaderboard","measuredAt":"2026-09-29","stderr":1.9540816326530612,"costPerTest":22.01518181818182,"latency":4202.8},{"benchmarkId":"terminal-bench:terminal-bench-4-0-0:overall","benchmarkName":"Terminal-Bench 4.0.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":44.55,"unit":"percent","family":"terminal-bench-4","index":75.67007449556576,"aboveFit":false,"run":"max effort · Claude Code","source":"Terminal-Bench","sourceUrl":"https://www.tbench.ai/leaderboard","measuredAt":"2026-09-29","stderr":1.9642857142857144,"costPerTest":22.01518181818182,"latency":4202.8},{"benchmarkId":"epoch-ai:epoch-oeis-open-lite:overall","benchmarkName":"OEIS Open Lite","category":"math","domain":null,"eligible":true,"ranked":true,"score":44,"unit":"percent","family":"epoch-ai:epoch-oeis-open-lite:overall","index":null,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":4.988876515698589,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:terminal-bench-4:overall","benchmarkName":"Terminal-Bench 4.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":41.414,"unit":"percent","family":"terminal-bench-4","index":74.18098054743622,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-4","measuredAt":"2026-11-09","stderr":1.336,"costPerTest":30.339756,"latency":4093.25},{"benchmarkId":"tau2-bench:banking-knowledge:overall","benchmarkName":"τ²-bench Banking","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":39.69072164948454,"unit":"percent","family":"tau2-bench","index":29.12713582470158,"aboveFit":false,"run":"max effort · Sierra","source":"Sierra Research","sourceUrl":"https://github.com/sierra-research/tau2-bench","measuredAt":"2026-10-20","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-ebr-bench:overall","benchmarkName":"EBR-bench","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":39.523799999999994,"unit":"percent","family":"epoch-ai:epoch-ebr-bench:overall","index":70.56533278659577,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.0141,"costPerTest":null,"latency":null},{"benchmarkId":"blueprintBench2","benchmarkName":"Blueprint-Bench 2","category":"multimodal","score":38.6,"unit":"percent","family":"blueprintBench2","index":null,"aboveFit":false,"run":null,"source":"Google DeepMind","sourceUrl":"https://x.com/GoogleDeepMind","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:tax_agent_bench:overall","benchmarkName":"Tax Agent Bench","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":38.281,"unit":"percent","family":"vals-ai:tax_agent_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench","measuredAt":"2026-11-09","stderr":3.503,"costPerTest":6.696087,"latency":866.628},{"benchmarkId":"aaTau3Banking","benchmarkName":"Artificial Analysis Tau3-Banking","category":"agentic","score":38.1,"unit":"percent","family":"aaTau3Banking","index":61.65482546119393,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/tau3-banking","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-furniture-assembly:overall","benchmarkName":"Furniture Assembly","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":35.833333333333336,"unit":"percent","family":"epoch-ai:epoch-furniture-assembly:overall","index":63.368183602437426,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":6.185876325418331,"costPerTest":null,"latency":null},{"benchmarkId":"frontierBench","benchmarkName":"Terminal-Bench 3.0","category":"agentic","score":34.05,"unit":"percent","family":"frontierBench","index":76.73467437567163,"aboveFit":false,"run":null,"source":"Ryan Marten, Alex Shaw, Andy Konwinski, Harbor, and the Laude Institute","sourceUrl":"https://www.frontierbench.ai/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"apprenticeBench","benchmarkName":"ApprenticeBench: end-to-end computer use, continual learning, and long-horizon agency on a real accounts-payable job","category":"agentic","score":34,"unit":"percent","family":"apprenticeBench","index":75.44483433015336,"aboveFit":false,"run":null,"source":"NeoCognition","sourceUrl":"https://neocognition.io/blog/apprentice-bench/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":28.6,"unit":"percent","family":"critpt","index":92.21537887036109,"aboveFit":true,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:terminal-bench-science:overall","benchmarkName":"Terminal-Bench-Science","category":null,"domain":"Science","eligible":false,"ranked":true,"score":15.714,"unit":"percent","family":"vals-ai:terminal-bench-science:overall","index":null,"aboveFit":false,"run":"Mini-SWE-agent","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-science","measuredAt":"2026-11-09","stderr":4.381,"costPerTest":51.990885,"latency":7157.669},{"benchmarkId":"mazeBench","benchmarkName":"MazeBench: visual spatial reasoning in a 3D open world","category":"agentic","score":11,"unit":"percent","family":"mazeBench","index":81.99849084321241,"aboveFit":false,"run":null,"source":"Jonathan Pappas and David Pappas","sourceUrl":"https://mazebench.com/blog?post=maze-bench-results","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lmarena:agent-bash-recovery:overall","benchmarkName":"Agent Arena command recovery","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":9.597561240468977,"unit":"index","family":"agent-arena","index":74.42150127272089,"aboveFit":false,"run":"high effort","source":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset","measuredAt":"2026-10-08","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lmarena:agent-steerability:overall","benchmarkName":"Agent Arena steerability","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":7.884448555419027,"unit":"index","family":"agent-arena","index":73.01305407865969,"aboveFit":false,"run":"high effort","source":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset","measuredAt":"2026-11-11","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lmarena:agent-task-outcome:overall","benchmarkName":"Agent Arena task outcome","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":5.110401002371701,"unit":"index","family":"agent-arena","index":70.73235209893679,"aboveFit":false,"run":"high effort","source":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset","measuredAt":"2026-11-11","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:programbench:overall","benchmarkName":"ProgramBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":2,"unit":"percent","family":"vals-ai:programbench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/programbench","measuredAt":"2026-11-09","stderr":0.992,"costPerTest":75.675797,"latency":9447.46},{"benchmarkId":"epoch-ai:epoch-frontiermath-erdos:overall","benchmarkName":"FrontierMath-Erdos","category":"math","domain":null,"eligible":true,"ranked":true,"score":0,"unit":"percent","family":"epoch-ai:epoch-frontiermath-erdos:overall","index":null,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":10,"outputPerMillion":50,"batch":{"inputPerMillion":5,"outputPerMillion":25},"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":56,"timeToFirstToken":3.762,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":{"elo":1491.5868575384075,"eloUpper":1495.7659402249778,"eloLower":1487.4077748518366,"votes":45524,"rank":12,"rankUpper":null,"rankLower":null,"arenaName":"Claude Fable 5-high","organization":"anthropic","updatedAt":"2026-11-09"},"quality":{"coveragePercent":95,"benchmarkCount":55,"categoryCount":7,"confidence":"high","state":"available"},"hosts":[{"host":"amazon-bedrock","name":"Amazon Bedrock","kind":"cloud","url":"https://docs.aws.amazon.com/bedrock/latest/userguide/models-supported.html","inputPerMillion":10,"outputPerMillion":50,"cacheReadPerMillion":1,"contextTokens":1000000,"maxOutputTokens":128000,"deprecated":false},{"host":"deepinfra","name":"Deep Infra","kind":"host","url":"https://deepinfra.com/models","inputPerMillion":10,"outputPerMillion":50,"cacheReadPerMillion":null,"contextTokens":1000000,"maxOutputTokens":128000,"deprecated":false},{"host":"digitalocean","name":"DigitalOcean","kind":"cloud","url":"https://docs.digitalocean.com/products/gradient-ai-platform/details/models/","inputPerMillion":10,"outputPerMillion":50,"cacheReadPerMillion":1,"contextTokens":1000000,"maxOutputTokens":128000,"deprecated":false},{"host":"google-vertex","name":"Google Vertex AI","kind":"cloud","url":"https://cloud.google.com/vertex-ai/generative-ai/docs/models","inputPerMillion":10,"outputPerMillion":50,"cacheReadPerMillion":1,"contextTokens":1000000,"maxOutputTokens":128000,"deprecated":false},{"host":"azure","name":"Microsoft Azure","kind":"cloud","url":"https://learn.microsoft.com/en-us/azure/ai-services/openai/concepts/models","inputPerMillion":10,"outputPerMillion":50,"cacheReadPerMillion":1,"contextTokens":1000000,"maxOutputTokens":128000,"deprecated":false},{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":10,"outputPerMillion":50,"cacheReadPerMillion":1,"contextTokens":1000000,"maxOutputTokens":128000,"deprecated":false},{"host":"snowflake","name":"Snowflake Cortex","kind":"cloud","url":"https://docs.snowflake.com/en/user-guide/snowflake-cortex/cortex-rest-api","inputPerMillion":10,"outputPerMillion":50,"cacheReadPerMillion":1,"contextTokens":1000000,"maxOutputTokens":128000,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:arc-prize-foundation","name":"ARC Prize Foundation","url":"https://arcprize.org","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:livebench","name":"LiveBench","url":"https://livebench.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:terminal-bench","name":"Terminal-Bench","url":"https://www.tbench.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:sierra-research","name":"Sierra Research","url":"https://github.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:lmarena","name":"LMArena","url":"https://huggingface.co","license":"CC BY 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"claude-opus-5","providerId":"anthropic","slug":"claude-opus-5","name":"Claude Opus 5","provider":"Anthropic","description":"Claude Opus 5 is a reasoning model from Anthropic. 72 benchmarks count toward its index, in 8 categories.","logoUrl":null,"releaseDate":"2026-07-24","contextWindowTokens":1000000,"modalities":["text","image","file"],"openWeights":false,"details":{"apiId":"anthropic/claude-opus-5","maxOutputTokens":128000,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.5,"weightsUrl":null},"reasoning":true,"overallScore":73.83590330525331,"overallScoreRange":{"low":71.93025052995557,"high":75.74155608055105},"badges":[],"capabilityScores":{"agentic":71.59852923010456,"coding":74.80430931747671,"reasoning":75.39312961741729,"multimodal":72.53850099059615,"knowledge":73.13296580509073,"multilingual":63.61211793504711,"instruction":55.61413352587505,"math":74.83521780402901},"capabilityCounts":{"math":5,"reasoning":10,"coding":23,"agentic":23,"knowledge":7,"multimodal":2,"multilingual":1,"instruction":1},"benchmarks":[{"benchmarkId":"vals-ai:proof_bench:overall","benchmarkName":"ProofBench v1.1","category":"math","domain":null,"eligible":true,"ranked":true,"score":99,"unit":"percent","family":"vals-ai:proof_bench:overall","index":81.33584598884255,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/proof_bench","measuredAt":"2026-11-09","stderr":1,"costPerTest":1.786125,"latency":590.886},{"benchmarkId":"arc-prize:v1-public-eval:overall","benchmarkName":"ARC-AGI-1 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":99,"unit":"percent","family":"arc-prize:v1-public-eval:overall","index":null,"aboveFit":false,"run":"max effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.42,"latency":null},{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":98.88888888888889,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":66.45205762128103,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.1111111111111112,"costPerTest":null,"latency":null},{"benchmarkId":"arc-prize:v1-semi-private:overall","benchmarkName":"ARC-AGI-1 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":97.5,"unit":"percent","family":"arc-prize:v1-semi-private:overall","index":71.81162890019444,"aboveFit":false,"run":"max effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.7,"latency":null},{"benchmarkId":"arc-prize:v2-public-eval:overall","benchmarkName":"ARC-AGI-2 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":97.06,"unit":"percent","family":"arc-prize:v2-public-eval:overall","index":null,"aboveFit":false,"run":"max effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":2.07,"latency":null},{"benchmarkId":"vals-ai:swebench:overall","benchmarkName":"SWE-bench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":97,"unit":"percent","family":"swe-bench-verified","index":72.77101257448976,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/swebench","measuredAt":"2026-09-01","stderr":0.764,"costPerTest":1.290782,"latency":576.988},{"benchmarkId":"vulcanCiiV1","benchmarkName":"VulcanBench Coding Intelligence Index v1","category":"coding","score":96.4,"unit":"percent","family":"vulcanCiiV1","index":null,"aboveFit":false,"run":null,"source":"VulcanBench contributors","sourceUrl":"https://github.com/morganlinton/VulcanBench/blob/main/docs/results/cii-v1-2026-08/README.md","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"sweVerified","benchmarkName":"Software Engineering Benchmark Verified","category":"coding","score":96,"unit":"percent","family":"swe-bench-verified","index":72.03345499670607,"aboveFit":false,"run":null,"source":"Carlos E. Jimenez, John Yang, Alexander Wettig, Shunyu Yao, Kexin Pei, Ofir Press, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2310.06770","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-math:overall","benchmarkName":"LiveBench Mathematics","category":"math","domain":null,"eligible":true,"ranked":true,"score":95.731,"unit":"percent","family":"livebench:livebench-math:overall","index":73.1969659381445,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Mathematics","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.2568934210526316,"latency":null},{"benchmarkId":"deepSearchQa","benchmarkName":"DeepSearchQA","category":"agentic","score":95,"unit":"percent","family":"deepSearchQa","index":69.44790025841915,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"legalAgentBenchHeldoutCriterionPass","benchmarkName":"Legal Agent Benchmark mean criterion-pass rate — Harvey held-out set","category":"agentic","score":94.1,"unit":"percent","family":"legalAgentBenchHeldoutCriterionPass","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":93.87626262626263,"unit":"percent","family":"gpqa-diamond","index":63.30984364225107,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.4782015056089346,"costPerTest":null,"latency":null},{"benchmarkId":"legalAgentBenchCriterionPass","benchmarkName":"Legal Agent Benchmark mean criterion-pass rate — Anthropic harness","category":"agentic","score":93.74,"unit":"percent","family":"legalAgentBenchCriterionPass","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"multiAgentBrowseCompPrerelease","benchmarkName":"Multi-Agent BrowseComp — 10-agent team prerelease configuration","category":"agentic","score":93.6,"unit":"percent","family":"multiAgentBrowseCompPrerelease","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaHarveyLab","benchmarkName":"Artificial Analysis Harvey LAB-AA v1.0","category":"agentic","score":93.5,"unit":"percent","family":"aaHarveyLab","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:gpqa:overall","benchmarkName":"GPQA Diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":93.434,"unit":"percent","family":"gpqa-diamond","index":62.904212446304726,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","measuredAt":"2026-09-01","stderr":1.244,"costPerTest":0.095481,"latency":39.588},{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":93.2,"unit":"percent","family":"gpqa-diamond","index":62.6895940403735,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"programBench","benchmarkName":"ProgramBench: Can Language Models Rebuild Programs From Scratch?","category":"coding","score":93,"unit":"percent","family":"programBench","index":95,"aboveFit":false,"run":null,"source":"John Yang, Kilian Lieret, Jeffrey Ma, Parth Thakkar, Dmitrii Pedchenko, Sten Sootla, Emily McMilin, Pengcheng Yin, Rui Hou, Gabriel Synnaeve, Diyi Yang, Ofir Press","sourceUrl":"https://programbench.com/static/paper.pdf","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gmmlu","benchmarkName":"Global MMLU","category":"multilingual","score":92.5,"unit":"percent","family":"gmmlu","index":null,"aboveFit":false,"run":null,"source":"Singh et al.","sourceUrl":"https://arxiv.org/abs/2412.03304","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"milu","benchmarkName":"Multi-task Indic Language Understanding Benchmark","category":"multilingual","score":92.1,"unit":"percent","family":"milu","index":null,"aboveFit":false,"run":null,"source":"Verma et al.","sourceUrl":"https://arxiv.org/abs/2411.02538","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:ioi-v1:overall","benchmarkName":"IOI v1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":91.667,"unit":"percent","family":"vals-ai:ioi-v1:overall","index":85.74995461578902,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/ioi-v1","measuredAt":"2026-08-09","stderr":8.333,"costPerTest":5.541014,"latency":1693.04},{"benchmarkId":"vals-ai:mmlu_pro:overall","benchmarkName":"MMLU Pro","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":91.591,"unit":"percent","family":"mmlu-pro","index":59.874425266061664,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","measuredAt":"2026-09-01","stderr":0.276,"costPerTest":0.027843,"latency":9.6},{"benchmarkId":"arxivMathJune2026WithTools","benchmarkName":"ArXivMath June 2026 with tools","category":"math","score":91.3,"unit":"percent","family":"arxivMathJune2026WithTools","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-reasoning:overall","benchmarkName":"LiveBench Reasoning","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":91.2115,"unit":"percent","family":"livebench","index":79.77819927494104,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Reasoning","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.16844554455445543,"latency":null},{"benchmarkId":"vals-ai:medscribe:overall","benchmarkName":"MedScribe","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":90.985,"unit":"percent","family":"vals-ai:medscribe:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medscribe","measuredAt":"2026-10-29","stderr":1.916,"costPerTest":0.236275,"latency":76.563},{"benchmarkId":"browseComp","benchmarkName":"BrowseComp","category":"agentic","score":90.8,"unit":"percent","family":"browseComp","index":71.64279243533588,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"arxivMathJune2026","benchmarkName":"ArXivMath June 2026 without tools","category":"math","score":90.8,"unit":"percent","family":"arxivMathJune2026","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-coding-2:overall","benchmarkName":"LiveBench Coding 2.0","category":"coding","domain":null,"eligible":true,"ranked":true,"score":90.8,"unit":"percent","family":"livebench:livebench-coding-2:overall","index":73.94063577996118,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Coding","measuredAt":"2026-10-14","stderr":null,"costPerTest":3.9217655172413792,"latency":null},{"benchmarkId":"arc-prize:v2-semi-private:overall","benchmarkName":"ARC-AGI-2 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":90.42,"unit":"percent","family":"arc-prize:v2-semi-private:overall","index":80.70347506450658,"aboveFit":false,"run":"max effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":2.06,"latency":null},{"benchmarkId":"bioMysteryBenchHumanSolvable","benchmarkName":"BioMysteryBench Human Solvable","category":"knowledge","score":90.1,"unit":"percent","family":"bioMysteryBenchHumanSolvable","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:mmmu:overall","benchmarkName":"MMMU Pro","category":"multimodal","domain":null,"eligible":true,"ranked":true,"score":89.884,"unit":"percent","family":"mmmu-pro","index":72.12895506580672,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmmu","measuredAt":"2026-09-01","stderr":0.725,"costPerTest":0.069014,"latency":30.028},{"benchmarkId":"include","benchmarkName":"INCLUDE","category":"multilingual","score":89.8,"unit":"percent","family":"include","index":63.61211793504711,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mcpAtlasClaimCoverage","benchmarkName":"MCP-Atlas mean claim coverage","category":"agentic","score":89.1,"unit":"percent","family":"mcpAtlasClaimCoverage","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaTerminalBench21","benchmarkName":"Artificial Analysis Terminal-Bench v2.1","category":"agentic","score":89.1,"unit":"percent","family":"aaTerminalBench21","index":71.76531236968987,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/terminalbench-v2-1","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:lcb:overall","benchmarkName":"LiveCodeBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":89.033,"unit":"percent","family":"livecodebench","index":63.21811734529965,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","measuredAt":"2026-09-01","stderr":0.913,"costPerTest":0.134205,"latency":59.22},{"benchmarkId":"livebench:livebench-research:overall","benchmarkName":"LiveBench Research","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":88.9,"unit":"percent","family":"livebench-2-agentic","index":83.60480592079884,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Research","measuredAt":"2026-10-14","stderr":null,"costPerTest":16.97036818181818,"latency":null},{"benchmarkId":"livebench:livebench-language:overall","benchmarkName":"LiveBench Language","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":88.68799999999999,"unit":"percent","family":"livebench:livebench-language:overall","index":74.84798524953258,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Language","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.09630266666666668,"latency":null},{"benchmarkId":"draco","benchmarkName":"Data Research and Analysis with Complex Operations","category":"agentic","score":88.6,"unit":"percent","family":"draco","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:vibe-code:overall","benchmarkName":"Vibe Code Bench v1.1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":88.404,"unit":"percent","family":"vals-ai:vibe-code:overall","index":74.14707472197055,"aboveFit":false,"run":"OpenHands","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","measuredAt":"2026-11-09","stderr":3.001,"costPerTest":33.876235,"latency":5301.22},{"benchmarkId":"epoch-ai:epoch-mirrorcode:overall","benchmarkName":"MirrorCode","category":"coding","domain":null,"eligible":true,"ranked":true,"score":87.77777777777777,"unit":"percent","family":"epoch-ai:epoch-mirrorcode:overall","index":88.11511428371952,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":7.176288606639567,"costPerTest":null,"latency":null},{"benchmarkId":"toolathlonVerifiedPass3","benchmarkName":"Toolathlon Verified Pass@3","category":"agentic","score":87,"unit":"percent","family":"toolathlonVerifiedPass3","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vulcanBench","benchmarkName":"VulcanBench v3","category":"coding","score":87,"unit":"percent","family":"vulcanBench","index":68.4175418120868,"aboveFit":false,"run":null,"source":"VulcanBench contributors","sourceUrl":"https://github.com/morganlinton/VulcanBench/tree/main","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:legal_bench:overall","benchmarkName":"LegalBench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":86.974,"unit":"percent","family":"vals-ai:legal_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","measuredAt":"2026-10-13","stderr":0.417,"costPerTest":0.011976,"latency":4.415},{"benchmarkId":"mcpAtlas","benchmarkName":"MCP Atlas","category":"agentic","score":85.8,"unit":"percent","family":"mcpAtlas","index":68.83962545456788,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tiers-1-3-v2-private:overall","benchmarkName":"FrontierMath-Tiers-1-3-v2-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":85.6140350877193,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tiers-1-3-v2-private:overall","index":75.95779018493243,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.082489457534394,"costPerTest":null,"latency":null},{"benchmarkId":"gdpPdfWithTools","benchmarkName":"GDP.pdf mean criteria pass rate with tools","category":"multimodal","score":85.5,"unit":"percent","family":"gdpPdfWithTools","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaMmmuPro","benchmarkName":"Artificial Analysis MMMU-Pro","category":"multimodal","score":84.7,"unit":"percent","family":"mmmu-pro","index":65.38243388342897,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmmu-pro","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:terminal-bench-2-1:overall","benchmarkName":"Terminal-Bench 2.1","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":84.644,"unit":"percent","family":"terminal-bench-2-1","index":70.64561867780573,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2-1","measuredAt":"2026-10-01","stderr":0.991,"costPerTest":0.886936,"latency":752.932},{"benchmarkId":"vals-ai:ioi:overall","benchmarkName":"IOI","category":"coding","domain":null,"eligible":true,"ranked":true,"score":84.333,"unit":"percent","family":"vals-ai:ioi:overall","index":77.77713149985266,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/ioi","measuredAt":"2026-10-29","stderr":9.964,"costPerTest":16.477241,"latency":3402.045},{"benchmarkId":"labBench2","benchmarkName":"LABBench2: An Improved Benchmark for AI Systems Performing Biology Research","category":"knowledge","score":84.2,"unit":"percent","family":"labBench2","index":null,"aboveFit":false,"run":null,"source":"Jon M. Laurent, Albert Bou, Michael Pieler, Conor Igoe, Alex Andonian, Siddharth Narayanan, James Braza, Alexandros Sanchez Vassopoulos, Jacob L. Steenwyk, Blake Lash, Andrew D. White, Samuel G. Rodriques","sourceUrl":"https://arxiv.org/abs/2604.09554","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-browser:overall","benchmarkName":"LiveBench Browser Use","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":84,"unit":"percent","family":"livebench-2-agentic","index":80.02128355568858,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Browser","measuredAt":"2026-10-14","stderr":null,"costPerTest":2.965252,"latency":null},{"benchmarkId":"gdpPdf","benchmarkName":"GDP.pdf mean criteria pass rate without tools","category":"multimodal","score":83.4,"unit":"percent","family":"gdpPdf","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"programBenchEpisode1","benchmarkName":"ProgramBench hidden-test pass rate after episode 1","category":"coding","score":83,"unit":"percent","family":"programBenchEpisode1","index":null,"aboveFit":false,"run":null,"source":"Yang et al.","sourceUrl":"https://arxiv.org/abs/2605.03546","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"chartographyWithTools","benchmarkName":"Chartography with image and code tools","category":"multimodal","score":83,"unit":"percent","family":"chartographyWithTools","index":null,"aboveFit":false,"run":null,"source":"Surge AI and Anthropic","sourceUrl":"https://surgehq.ai/blog/chartography","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-design:overall","benchmarkName":"LiveBench Design","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":83,"unit":"percent","family":"livebench-2-agentic","index":79.28995246076813,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Design","measuredAt":"2026-10-14","stderr":null,"costPerTest":5.452636,"latency":null},{"benchmarkId":"benchCadVision2CodeWithTools","benchmarkName":"BenchCAD Vision2Code voxel IoU with tools","category":"multimodal","score":82.1,"unit":"percent","family":"benchCadVision2CodeWithTools","index":null,"aboveFit":false,"run":null,"source":"Zhang et al. and Anthropic","sourceUrl":"https://arxiv.org/abs/2605.10865","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-coding:overall","benchmarkName":"LiveBench Coding","category":"coding","domain":null,"eligible":true,"ranked":true,"score":81.4455,"unit":"percent","family":"livebench:livebench-coding:overall","index":69.36981846599397,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Coding","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.033035042735042736,"latency":null},{"benchmarkId":"toolathlonVerified","benchmarkName":"Toolathlon-Verified","category":"agentic","score":80.6,"unit":"percent","family":"toolathlonVerified","index":73.7763492000903,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":79.3,"unit":"percent","family":"lcr","index":61.70423513811355,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:biomysterybench:overall","benchmarkName":"BioMysteryBench","category":null,"domain":"Science","eligible":false,"ranked":true,"score":79.259,"unit":"percent","family":"vals-ai:biomysterybench:overall","index":null,"aboveFit":false,"run":"Terminus 2","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/biomysterybench","measuredAt":"2026-11-09","stderr":2.062,"costPerTest":3.292774,"latency":2195.964},{"benchmarkId":"swePro","benchmarkName":"SWE-bench Pro","category":"coding","score":79.2,"unit":"percent","family":"swePro","index":76.28689418400177,"aboveFit":false,"run":null,"source":"Xiang Deng, Jeff Da, Edwin Pan, Yannis Yiming He, Charles Ide, Kanak Garg, Niklas Lauffer, Andrew Park, Nitin Pasari, Chetan Rane, Karmini Sampath, Maya Krishnan, Srivatsa Kundurthy, Sean Hendryx, Zifan Wang, Chen Bo Calvin Zhang, Noah Jacobson, Bing Liu, Brad Kenstler","sourceUrl":"https://arxiv.org/abs/2509.16941","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"riemannBenchWithTools","benchmarkName":"RiemannBench with tools","category":"math","score":79,"unit":"percent","family":"riemannBenchWithTools","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"protocolsUnderstanding","benchmarkName":"Benchling Molecular Biology Protocols Understanding","category":"knowledge","score":78.4,"unit":"percent","family":"protocolsUnderstanding","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"officeQa","benchmarkName":"OfficeQA","category":"multimodal","score":78.1,"unit":"percent","family":"officeQa","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:public-benefits-bench:overall","benchmarkName":"Public Benefits Bench v1.1","category":null,"domain":"Public benefits","eligible":false,"ranked":true,"score":76.928,"unit":"percent","family":"vals-ai:public-benefits-bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/public-benefits-bench","measuredAt":"2026-10-12","stderr":1.096,"costPerTest":3.949099,"latency":1808.881},{"benchmarkId":"vals-ai:tax_eval_v2:overall","benchmarkName":"TaxEval v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":75.143,"unit":"percent","family":"vals-ai:tax_eval_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","measuredAt":"2026-09-01","stderr":0.832,"costPerTest":0.114274,"latency":58.339},{"benchmarkId":"livebench:livebench-data-analysis:overall","benchmarkName":"LiveBench Data Analysis","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":74.55033333333334,"unit":"percent","family":"livebench","index":58.19475755947725,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Data%20Analysis","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.7455293333333333,"latency":null},{"benchmarkId":"osWorld21","benchmarkName":"OSWorld 2.1 (all 108 tasks)","category":"agentic","score":74,"unit":"percent","family":"osWorld21","index":72.79589912904675,"aboveFit":false,"run":null,"source":"Mengqi Yuan, Zilong Zhou, Xinzhuang Xiong, Weiming Wu, Jiayang Sun, Jiamin Song, Kaiqian Cui, Bowen Wang, Haoyuan Wu, Yitong Li, Dunjie Lu, Haikong Lu, Qi Zhen, Xinyuan Wang, Jiaqi Deng, Yuhao Yang, Cheng Chen, Boyuan Zheng, Alex Su, Xiao Yu, Hao Zou, Saaket Agashe, Xing Han Lu, Manpreet Kaur, Zhengyang Qi, Vincent Sunn Chen, Frederic Sala, Dayiheng Liu, Junyang Lin, Zhou Yu, Yu Su, Siva Reddy, Xin Eric Wang, Peng Qi, Tianbao Xie, Tao Yu","sourceUrl":"https://github.com/xlang-ai/OSWorld-V2/releases","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:emb:overall","benchmarkName":"EMB","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":73.556,"unit":"percent","family":"vals-ai:emb:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/emb","measuredAt":"2026-10-20","stderr":2.24,"costPerTest":8.021851,"latency":1558.673},{"benchmarkId":"healthBenchProfessionalRaw","benchmarkName":"HealthBench Professional raw score","category":"knowledge","score":73.4,"unit":"percent","family":"healthBenchProfessionalRaw","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:corp_fin_v2:overall","benchmarkName":"CorpFin v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":73.194,"unit":"percent","family":"vals-ai:corp_fin_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","measuredAt":"2026-08-12","stderr":0.871,"costPerTest":0.860452,"latency":28.573},{"benchmarkId":"epoch-ai:epoch-frontiermath-tier-4-v2-private:overall","benchmarkName":"FrontierMath-Tier-4-v2-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":73.17073170731707,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tier-4-v2-private:overall","index":77.23342928694453,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":7.005564203095157,"costPerTest":null,"latency":null},{"benchmarkId":"toolathlonVerifiedPass3All","benchmarkName":"Toolathlon Verified Pass cubed","category":"agentic","score":73.1,"unit":"percent","family":"toolathlonVerifiedPass3All","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"spatialBenchVerified","benchmarkName":"LatchBio SpatialBench Verified","category":"knowledge","score":72.5,"unit":"percent","family":"spatialBenchVerified","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:mortgage_tax:overall","benchmarkName":"MortgageTax","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":72.059,"unit":"percent","family":"vals-ai:mortgage_tax:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mortgage_tax","measuredAt":"2026-09-01","stderr":0.875,"costPerTest":0.07375,"latency":7.978},{"benchmarkId":"osWorld2","benchmarkName":"OSWorld 2.0","category":"agentic","score":70.6,"unit":"percent","family":"osWorld2","index":82.6169556318498,"aboveFit":false,"run":null,"source":"Mengqi Yuan, Zilong Zhou, Xinzhuang Xiong, Weiming Wu, Jiayang Sun, Jiamin Song, Kaiqian Cui, Bowen Wang, Haoyuan Wu, Yitong Li, Dunjie Lu, Haikong Lu, Qi Zhen, Xinyuan Wang, Jiaqi Deng, Yuhao Yang, Cheng Chen, Boyuan Zheng, Alex Su, Xiao Yu, Hao Zou, Saaket Agashe, Xing Han Lu, Manpreet Kaur, Zhengyang Qi, Vincent Sunn Chen, Frederic Sala, Dayiheng Liu, Junyang Lin, Zhou Yu, Yu Su, Siva Reddy, Xin Eric Wang, Peng Qi, Tianbao Xie, Tao Yu","sourceUrl":"https://arxiv.org/abs/2606.29537","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"cursorBench32","benchmarkName":"cursorBench32","category":"coding","score":70,"unit":"percent","family":"cursorBench32","index":73.35429385207142,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"scarfBench","benchmarkName":"ScarfBench: Cross-Framework Application Migration in Enterprise Java","category":"coding","score":69.8,"unit":"percent","family":"scarfBench","index":74.21468464445365,"aboveFit":false,"run":null,"source":"Advait Pavuluri, Bridget McGinn, Ashita Saxena, George Safta, Srikanth Tamilselvam, Raju Pavuluri, Michele Merler, Baishakhi Ray, Rahul Krishna (IBM Research)","sourceUrl":"https://arxiv.org/abs/2605.06754","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"deepSwe","benchmarkName":"DeepSWE","category":"agentic","score":68.8,"unit":"percent","family":"deepSwe","index":71.09240001165024,"aboveFit":false,"run":null,"source":"Datacurve AI","sourceUrl":"https://deepswe.datacurve.ai/blog","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"healthBench","benchmarkName":"HealthBench raw score","category":"knowledge","score":67.1,"unit":"percent","family":"healthBench","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"officeQaPro","benchmarkName":"OfficeQA Pro","category":"multimodal","score":66.9,"unit":"percent","family":"officeQaPro","index":76.32130750657446,"aboveFit":false,"run":null,"source":"OfficeQA Pro authors","sourceUrl":"https://arxiv.org/abs/2603.08655","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:cyber:overall","benchmarkName":"CyberBench v1.1","category":null,"domain":"Other","eligible":false,"ranked":true,"score":65.357,"unit":"percent","family":"vals-ai:cyber:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/cyber","measuredAt":"2026-11-05","stderr":5.55,"costPerTest":3.527538,"latency":1165.404},{"benchmarkId":"livebench:livebench-agentic-coding:overall","benchmarkName":"LiveBench Agentic Coding","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":65.202,"unit":"percent","family":"livebench:livebench-agentic-coding:overall","index":76.40440246299107,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Agentic%20Coding","measuredAt":"2026-06-25","stderr":null,"costPerTest":5.3366375,"latency":null},{"benchmarkId":"hleWithTools","benchmarkName":"Humanity's Last Exam with tools","category":"agentic","score":64.7,"unit":"percent","family":"hleWithTools","index":72.567164817454,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hle","benchmarkName":"Humanity's Last Exam","category":"knowledge","score":64.7,"unit":"percent","family":"hle","index":89.29968491674117,"aboveFit":false,"run":null,"source":"Center for AI Safety, Scale AI, and thousands of expert contributors","sourceUrl":"https://lastexam.ai/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-instruction-following:overall","benchmarkName":"LiveBench Instruction Following","category":"instruction","domain":null,"eligible":true,"ranked":true,"score":63.76650000000001,"unit":"percent","family":"livebench:livebench-instruction-following:overall","index":55.61413352587505,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=IF","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.3282615,"latency":null},{"benchmarkId":"frontierCode11Extended","benchmarkName":"FrontierCode 1.1 Extended","category":"coding","score":63.6,"unit":"percent","family":"frontierCode11Extended","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:medcode:overall","benchmarkName":"MedCode","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":63.57,"unit":"percent","family":"vals-ai:medcode:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medcode","measuredAt":"2026-10-29","stderr":1.993,"costPerTest":0.156845,"latency":24.5},{"benchmarkId":"livebench:livebench-long-horizon:overall","benchmarkName":"LiveBench Long Agentic","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":62.1,"unit":"percent","family":"livebench-2-agentic","index":64.00513257693058,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Long%20Agentic","measuredAt":"2026-10-14","stderr":null,"costPerTest":48.75317222222222,"latency":null},{"benchmarkId":"organicChemistryV2","benchmarkName":"Anthropic Organic Chemistry V2 evaluation","category":"knowledge","score":61.6,"unit":"percent","family":"organicChemistryV2","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gdpvalAaNormalized","benchmarkName":"GDPval-AA normalized","category":"agentic","score":61.1,"unit":"percent","family":"gdpvalAaNormalized","index":77.71885505656373,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"protocolsTroubleshooting","benchmarkName":"Molecular Biology Protocols Troubleshooting","category":"knowledge","score":61.1,"unit":"percent","family":"protocolsTroubleshooting","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"omniscienceAccuracy","benchmarkName":"Artificial Analysis Omniscience Accuracy","category":"knowledge","score":60.9,"unit":"percent","family":"omniscienceAccuracy","index":87.43953989585034,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-furniture-assembly:overall","benchmarkName":"Furniture Assembly","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":60.83333333333333,"unit":"percent","family":"epoch-ai:epoch-furniture-assembly:overall","index":75.02258797221336,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":6.185876325418331,"costPerTest":null,"latency":null},{"benchmarkId":"singleCellBench","benchmarkName":"LatchBio SingleCellBench","category":"knowledge","score":60.6,"unit":"percent","family":"singleCellBench","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:skillsbench:overall","benchmarkName":"SkillsBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":60.441,"unit":"percent","family":"vals-ai:skillsbench:overall","index":70.12965133264869,"aboveFit":false,"run":"OpenHands","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/skillsbench","measuredAt":"2026-10-29","stderr":4.578,"costPerTest":2.508594,"latency":505.008},{"benchmarkId":"riemannBench","benchmarkName":"RiemannBench without tools","category":"math","score":60,"unit":"percent","family":"riemannBench","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-simpleqa-verified:overall","benchmarkName":"SimpleQA Verified","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":59.9,"unit":"percent","family":"epoch-ai:epoch-simpleqa-verified:overall","index":73.82104017868588,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.550610974549837,"costPerTest":null,"latency":null},{"benchmarkId":"healthBenchProfessional","benchmarkName":"HealthBench Professional","category":"knowledge","score":59.8,"unit":"percent","family":"healthBenchProfessional","index":72.17153174070901,"aboveFit":false,"run":null,"source":"Rebecca Soskin Hicks, Mikhail Trofimov, Dominick Lim, Rahul K. Arora, Foivos Tsimpourlas, Preston Bowman, Michael Sharman, Chi Tong, Kavin Karthik, Arnav Dugar, Akshay Jagadeesh, Khaled Saab, Johannes Heidecke, Ashley Alexander, Nate Gross, Karan Singhal","sourceUrl":"https://arxiv.org/abs/2604.27470","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-mystery-game-puzzles:overall","benchmarkName":"Mystery Game Puzzles","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":59,"unit":"percent","family":"epoch-ai:epoch-mystery-game-puzzles:overall","index":84.37147710767533,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":4.943110704237103,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:fabv2:overall","benchmarkName":"Finance Agent (v2)","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":58.633,"unit":"percent","family":"vals-ai:fabv2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/fabv2","measuredAt":"2026-10-29","stderr":0.077,"costPerTest":5.124473,"latency":598.424},{"benchmarkId":"healthBenchLengthAdjusted","benchmarkName":"HealthBench length-adjusted score","category":"knowledge","score":57.8,"unit":"percent","family":"healthBenchLengthAdjusted","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:code-migration:overall","benchmarkName":"Code Migration","category":"coding","domain":null,"eligible":true,"ranked":true,"score":57.473,"unit":"percent","family":"vals-ai:code-migration:overall","index":75.7210993499359,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/code-migration","measuredAt":"2026-10-29","stderr":4.371,"costPerTest":60.509587,"latency":10268.729},{"benchmarkId":"aaSciCode","benchmarkName":"Artificial Analysis SciCode","category":"coding","score":56.4,"unit":"percent","family":"scicode-family","index":69.36797910030221,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/scicode","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hleNoTools","benchmarkName":"Humanity's Last Exam without tools","category":"knowledge","score":56.3,"unit":"percent","family":"hle","index":80.8720258584909,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:legal_research:overall","benchmarkName":"Legal Research Bench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":55.288,"unit":"percent","family":"vals-ai:legal_research:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_research","measuredAt":"2026-11-09","stderr":3.456,"costPerTest":6.762498,"latency":1860.719},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":54.9,"unit":"percent","family":"hle","index":79.46741601544919,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hleVerified","benchmarkName":"HLE-Verified","category":"knowledge","score":54.4,"unit":"percent","family":"hleVerified","index":null,"aboveFit":false,"run":null,"source":"Weiqi Zhai, Zhihai Wang, Jinghang Wang, Boyu Yang, Xiaogang Li, Xiang Xu, Bohan Wang, Peng Wang, Xingzhe Wu, Anfeng Li, Qiyuan Feng, Yuhao Zhou, Shoulin Han, Wenjie Luo, Yiyuan Li, Yaxuan Wang, Ruixian Luo, Guojie Lin, Peiyao Xiao, Chengliang Xu, Ben Wang, Zeyu Wang, Zichao Chen, Jianan Ye, Yijie Hu, Jialong Chen, Zongwen Shen, Yuliang Xu, An Yang, Bowen Yu, Dayiheng Liu, Junyang Lin, Hu Wei, Que Shen, Bing Zhao","sourceUrl":"https://arxiv.org/abs/2602.13964","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaAnalystAgent","benchmarkName":"Artificial Analysis AnalystAgent","category":"agentic","score":53.8,"unit":"percent","family":"aaAnalystAgent","index":76.30544989392165,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/aa-analyst-agent","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:terminal-bench-4:overall","benchmarkName":"Terminal-Bench 4.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":53.535,"unit":"percent","family":"terminal-bench-4","index":79.93649959886417,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-4","measuredAt":"2026-11-09","stderr":1.336,"costPerTest":18.596913,"latency":3995.725},{"benchmarkId":"frontierCode","benchmarkName":"FrontierCode 1.1 Main","category":"coding","score":53.4,"unit":"percent","family":"frontierCode","index":76.50455049725,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"terminal-bench:terminal-bench-4-1-0:overall","benchmarkName":"Terminal-Bench 4.1.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":52.12,"unit":"percent","family":"terminal-bench:terminal-bench-4-1-0:overall","index":77.27032556745014,"aboveFit":false,"run":"max effort · Claude Code","source":"Terminal-Bench","sourceUrl":"https://www.tbench.ai/leaderboard","measuredAt":"2026-09-29","stderr":1.729591836734694,"costPerTest":18.08821212121212,"latency":4792.8},{"benchmarkId":"frontierSweV2","benchmarkName":"FrontierSWE v2","category":"coding","score":52,"unit":"percent","family":"frontierSweV2","index":78.2899497577414,"aboveFit":false,"run":null,"source":"Proximal","sourceUrl":"https://www.frontierswe.com/blog/v2","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"terminal-bench:terminal-bench-4-0-0:overall","benchmarkName":"Terminal-Bench 4.0.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":51.82,"unit":"percent","family":"terminal-bench-4","index":79.12215134598082,"aboveFit":false,"run":"max effort · Claude Code","source":"Terminal-Bench","sourceUrl":"https://www.tbench.ai/leaderboard","measuredAt":"2026-09-29","stderr":1.729591836734694,"costPerTest":18.08821212121212,"latency":4792.8},{"benchmarkId":"bioMysteryBenchHumanDifficult","benchmarkName":"BioMysteryBench Human Difficult","category":"knowledge","score":49.4,"unit":"percent","family":"bioMysteryBenchHumanDifficult","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"tau2-bench:banking-knowledge:overall","benchmarkName":"τ²-bench Banking","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":48.71134020618557,"unit":"percent","family":"tau2-bench","index":35.11297575182947,"aboveFit":false,"run":"max effort · Sierra","source":"Sierra Research","sourceUrl":"https://github.com/sierra-research/tau2-bench","measuredAt":"2026-10-20","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"proteinGymHard","benchmarkName":"ProteinGym Hard","category":"knowledge","score":47.7,"unit":"percent","family":"proteinGymHard","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaEnterpriseOpsGym","benchmarkName":"Artificial Analysis EnterpriseOps-Gym","category":"agentic","score":47.5,"unit":"percent","family":"aaEnterpriseOpsGym","index":null,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/enterprise-ops-gym-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:tax_agent_bench:overall","benchmarkName":"Tax Agent Bench","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":47.102,"unit":"percent","family":"vals-ai:tax_agent_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench","measuredAt":"2026-11-09","stderr":3.599,"costPerTest":5.136554,"latency":1008.306},{"benchmarkId":"cursorBench40","benchmarkName":"cursorBench40","category":"coding","score":46.6,"unit":"percent","family":"cursorBench40","index":72.28305460161695,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-ebr-bench:overall","benchmarkName":"EBR-bench","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":45.714285714285715,"unit":"percent","family":"epoch-ai:epoch-ebr-bench:overall","index":73.40631620908638,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":3.5634832254989925,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-chess-puzzles:overall","benchmarkName":"Chess Puzzles","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":43,"unit":"percent","family":"epoch-ai:epoch-chess-puzzles:overall","index":66.62232911659308,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":4.97569851956243,"costPerTest":null,"latency":null},{"benchmarkId":"frontierBench","benchmarkName":"Terminal-Bench 3.0","category":"agentic","score":42.7,"unit":"percent","family":"frontierBench","index":82.65710299030891,"aboveFit":false,"run":null,"source":"Ryan Marten, Alex Shaw, Andy Konwinski, Harbor, and the Laude Institute","sourceUrl":"https://www.frontierbench.ai/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"proteinDesign","benchmarkName":"Anthropic Protein Design evaluation","category":"knowledge","score":42.5,"unit":"percent","family":"proteinDesign","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"imo2026","benchmarkName":"International Mathematical Olympiad 2026","category":"math","score":42,"unit":"percent","family":"imo2026","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:mysterymechanism:overall","benchmarkName":"MysteryMechanism","category":null,"domain":"Science","eligible":false,"ranked":true,"score":37.387,"unit":"percent","family":"vals-ai:mysterymechanism:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mysterymechanism","measuredAt":"2026-11-11","stderr":3.255,"costPerTest":3.280693,"latency":1012.543},{"benchmarkId":"benchCadVision2Code","benchmarkName":"BenchCAD Vision2Code voxel IoU without tools","category":"multimodal","score":36.6,"unit":"percent","family":"benchCadVision2Code","index":null,"aboveFit":false,"run":null,"source":"Zhang et al. and Anthropic","sourceUrl":"https://arxiv.org/abs/2605.10865","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"apprenticeBench","benchmarkName":"ApprenticeBench: end-to-end computer use, continual learning, and long-horizon agency on a real accounts-payable job","category":"agentic","score":36,"unit":"percent","family":"apprenticeBench","index":76.45873156366537,"aboveFit":false,"run":null,"source":"NeoCognition","sourceUrl":"https://neocognition.io/blog/apprentice-bench/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"postTrainBench11","benchmarkName":"PostTrainBench v1.1","category":"coding","score":35.04,"unit":"percent","family":"postTrainBench11","index":70.06192033848991,"aboveFit":false,"run":null,"source":"PostTrainBench team","sourceUrl":"https://posttrainbench.com/?version=v1.1","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"arc-prize:v3-semi-private:overall","benchmarkName":"ARC-AGI-3 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":30.159999999999997,"unit":"percent","family":"arc-prize:v3-semi-private:overall","index":78.1194477198705,"aboveFit":false,"run":"high effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"chartography","benchmarkName":"Chartography without tools","category":"multimodal","score":29.6,"unit":"percent","family":"chartography","index":null,"aboveFit":false,"run":null,"source":"Surge AI and Anthropic","sourceUrl":"https://surgehq.ai/blog/chartography","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":29.1,"unit":"percent","family":"critpt","index":93.18332052871054,"aboveFit":true,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:vcb-1-100:overall","benchmarkName":"Vibe Code Bench 1-100","category":"coding","domain":null,"eligible":true,"ranked":true,"score":28.533,"unit":"percent","family":"vals-ai:vcb-1-100:overall","index":76.21214410130726,"aboveFit":false,"run":"OpenHands","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vcb-1-100","measuredAt":"2026-10-29","stderr":4.333,"costPerTest":41.49277,"latency":4905.208},{"benchmarkId":"vals-ai:terminal-bench-science:overall","benchmarkName":"Terminal-Bench-Science","category":null,"domain":"Science","eligible":false,"ranked":true,"score":27.143,"unit":"percent","family":"vals-ai:terminal-bench-science:overall","index":null,"aboveFit":false,"run":"Mini-SWE-agent","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-science","measuredAt":"2026-11-09","stderr":5.354,"costPerTest":32.543386,"latency":7908.453},{"benchmarkId":"bug-hunt-bench","benchmarkName":"Bug Hunt Bench","category":"coding","score":27,"unit":"percent","family":"bug-hunt-bench","index":68.11334880122467,"aboveFit":false,"run":null,"source":"Pawel Huryn","sourceUrl":"https://bughunt.productcompass.pm/method","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"frontierCode2","benchmarkName":"FrontierCode 2.0","category":"coding","score":26.7,"unit":"percent","family":"frontierCode2","index":71.82192143064911,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"automationBench","benchmarkName":"AutomationBench","category":"agentic","score":26,"unit":"percent","family":"automationBench","index":31.545009002707882,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaDeepsecBench","benchmarkName":"Artificial Analysis DeepsecBench-AA","category":"agentic","score":25.9,"unit":"percent","family":"aaDeepsecBench","index":70.20751321210125,"aboveFit":false,"run":null,"source":"Artificial Analysis; dataset by Vercel","sourceUrl":"https://artificialanalysis.ai/evaluations/deepsecbench-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"legalAgentBenchAllPass","benchmarkName":"Legal Agent Benchmark all-pass rate — Anthropic harness","category":"agentic","score":23.58,"unit":"percent","family":"legalAgentBenchAllPass","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"toolathlonVerifiedAvgTurns","benchmarkName":"Toolathlon Verified average assistant turns","category":"agentic","score":23.5,"unit":"percent","family":"toolathlonVerifiedAvgTurns","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:time_horizon_index:overall","benchmarkName":"Time Horizon Index: KSP","category":null,"domain":"Other","eligible":false,"ranked":true,"score":18.833,"unit":"percent","family":"vals-ai:time_horizon_index:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/time_horizon_index","measuredAt":"2026-10-07","stderr":0,"costPerTest":1348.765783,"latency":0},{"benchmarkId":"lmarena:agent-bash-recovery:overall","benchmarkName":"Agent Arena command recovery","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":13.570115207168133,"unit":"index","family":"agent-arena","index":77.68756341590733,"aboveFit":false,"run":"max effort","source":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset","measuredAt":"2026-10-08","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:srebench:overall","benchmarkName":"SRE Bench","category":null,"domain":"Other","eligible":false,"ranked":true,"score":12.214,"unit":"percent","family":"vals-ai:srebench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/srebench","measuredAt":"2026-11-05","stderr":2.027,"costPerTest":23.633213,"latency":4858.213},{"benchmarkId":"mazeBench","benchmarkName":"MazeBench: visual spatial reasoning in a 3D open world","category":"agentic","score":12,"unit":"percent","family":"mazeBench","index":83.818474797291,"aboveFit":false,"run":null,"source":"Jonathan Pappas and David Pappas","sourceUrl":"https://mazebench.com/blog?post=maze-bench-results","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"legalAgentBenchHeldoutAllPass","benchmarkName":"Legal Agent Benchmark all-pass rate — Harvey held-out set","category":"agentic","score":11.7,"unit":"percent","family":"legalAgentBenchHeldoutAllPass","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lmarena:agent-task-outcome:overall","benchmarkName":"Agent Arena task outcome","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":10.170285653518118,"unit":"index","family":"agent-arena","index":74.89237052701392,"aboveFit":false,"run":"max effort","source":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset","measuredAt":"2026-11-11","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:cua_bench:overall","benchmarkName":"CUA-bench","category":null,"domain":"Other","eligible":false,"ranked":true,"score":9,"unit":"percent","family":"vals-ai:cua_bench:overall","index":null,"aboveFit":false,"run":"claude-code","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/cua_bench","measuredAt":"2026-10-13","stderr":0,"costPerTest":232.233536,"latency":25.29},{"benchmarkId":"lmarena:agent-steerability:overall","benchmarkName":"Agent Arena steerability","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":4.382886210611392,"unit":"index","family":"agent-arena","index":70.13422088664998,"aboveFit":false,"run":"max effort","source":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset","measuredAt":"2026-11-11","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:programbench:overall","benchmarkName":"ProgramBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":3,"unit":"percent","family":"vals-ai:programbench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/programbench","measuredAt":"2026-11-09","stderr":1.209,"costPerTest":60.289268,"latency":10480.572},{"benchmarkId":"epoch-ai:epoch-frontiermath-erdos:overall","benchmarkName":"FrontierMath-Erdos","category":"math","domain":null,"eligible":true,"ranked":true,"score":0,"unit":"percent","family":"epoch-ai:epoch-frontiermath-erdos:overall","index":null,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":5,"outputPerMillion":25,"batch":{"inputPerMillion":2.5,"outputPerMillion":12.5},"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":48,"timeToFirstToken":0.932,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":{"elo":1502.2616064241324,"eloUpper":1505.8693297979344,"eloLower":1498.65388305033,"votes":73697,"rank":7,"rankUpper":null,"rankLower":null,"arenaName":"Claude Opus 5-high","organization":"anthropic","updatedAt":"2026-11-09"},"quality":{"coveragePercent":100,"benchmarkCount":72,"categoryCount":8,"confidence":"high","state":"available"},"hosts":[{"host":"amazon-bedrock","name":"Amazon Bedrock","kind":"cloud","url":"https://docs.aws.amazon.com/bedrock/latest/userguide/models-supported.html","inputPerMillion":5,"outputPerMillion":25,"cacheReadPerMillion":0.5,"contextTokens":1000000,"maxOutputTokens":128000,"deprecated":false},{"host":"anthropic","name":"Anthropic","kind":"maker","url":"https://docs.anthropic.com/en/docs/about-claude/models","inputPerMillion":5,"outputPerMillion":25,"cacheReadPerMillion":0.5,"contextTokens":1000000,"maxOutputTokens":128000,"deprecated":false},{"host":"digitalocean","name":"DigitalOcean","kind":"cloud","url":"https://docs.digitalocean.com/products/gradient-ai-platform/details/models/","inputPerMillion":5,"outputPerMillion":25,"cacheReadPerMillion":0.5,"contextTokens":1000000,"maxOutputTokens":128000,"deprecated":false},{"host":"google-vertex","name":"Google Vertex AI","kind":"cloud","url":"https://cloud.google.com/vertex-ai/generative-ai/docs/models","inputPerMillion":5,"outputPerMillion":25,"cacheReadPerMillion":0.5,"contextTokens":1000000,"maxOutputTokens":128000,"deprecated":false},{"host":"azure","name":"Microsoft Azure","kind":"cloud","url":"https://learn.microsoft.com/en-us/azure/ai-services/openai/concepts/models","inputPerMillion":5,"outputPerMillion":25,"cacheReadPerMillion":0.5,"contextTokens":1000000,"maxOutputTokens":128000,"deprecated":false},{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":5,"outputPerMillion":25,"cacheReadPerMillion":0.5,"contextTokens":1000000,"maxOutputTokens":128000,"deprecated":false},{"host":"snowflake","name":"Snowflake Cortex","kind":"cloud","url":"https://docs.snowflake.com/en/user-guide/snowflake-cortex/cortex-rest-api","inputPerMillion":5,"outputPerMillion":25,"cacheReadPerMillion":0.5,"contextTokens":1000000,"maxOutputTokens":128000,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:arc-prize-foundation","name":"ARC Prize Foundation","url":"https://arcprize.org","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:livebench","name":"LiveBench","url":"https://livebench.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:terminal-bench","name":"Terminal-Bench","url":"https://www.tbench.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:sierra-research","name":"Sierra Research","url":"https://github.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:lmarena","name":"LMArena","url":"https://huggingface.co","license":"CC BY 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gpt-5-4-pro","providerId":"openai","slug":"gpt-5-4-pro","name":"GPT-5.4 Pro","provider":"OpenAI","description":"GPT-5.4 Pro is a reasoning model from OpenAI in the GPT-5.4 family. 12 benchmarks count toward its index, in 4 categories.","logoUrl":null,"releaseDate":"2026-03-05","contextWindowTokens":1050000,"modalities":["text","image","file"],"openWeights":false,"details":{"apiId":"openai/gpt-5.4-pro","maxOutputTokens":128000,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":null,"weightsUrl":null},"reasoning":true,"overallScore":73.29197793418858,"overallScoreRange":{"low":64.62410100779046,"high":81.9598548605867},"badges":[],"capabilityScores":{"agentic":70.4577627263369,"coding":null,"reasoning":80.39320459464776,"multimodal":null,"knowledge":67.23942062169141,"multilingual":null,"instruction":null,"math":75.28088603151576},"capabilityCounts":{"knowledge":3,"reasoning":4,"agentic":1,"math":4},"benchmarks":[{"benchmarkId":"arc-prize:v1-public-eval:overall","benchmarkName":"ARC-AGI-1 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":98.25,"unit":"percent","family":"arc-prize:v1-public-eval:overall","index":null,"aboveFit":false,"run":"xhigh effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":5.38,"latency":null},{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":94.6,"unit":"percent","family":"gpqa-diamond","index":63.97363578526119,"aboveFit":false,"run":"xhigh effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.6,"costPerTest":null,"latency":null},{"benchmarkId":"arc-prize:v1-semi-private:overall","benchmarkName":"ARC-AGI-1 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":94.5,"unit":"percent","family":"arc-prize:v1-semi-private:overall","index":70.43242059127125,"aboveFit":false,"run":"xhigh effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":7.21,"latency":null},{"benchmarkId":"ipho2025Theory","benchmarkName":"International Physics Olympiad 2025 (Theory)","category":"math","score":93.5,"unit":"percent","family":"ipho2025Theory","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"arc-prize:v2-public-eval:overall","benchmarkName":"ARC-AGI-2 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":92.21000000000001,"unit":"percent","family":"arc-prize:v2-public-eval:overall","index":null,"aboveFit":false,"run":"xhigh effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":17.6,"latency":null},{"benchmarkId":"browseComp","benchmarkName":"BrowseComp","category":"agentic","score":89.3,"unit":"percent","family":"browseComp","index":70.4577627263369,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"arc-prize:v2-semi-private:overall","benchmarkName":"ARC-AGI-2 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":83.33,"unit":"percent","family":"arc-prize:v2-semi-private:overall","index":77.44821003458563,"aboveFit":false,"run":"xhigh effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":16.41,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tiers-1-3-v2-private:overall","benchmarkName":"FrontierMath-Tiers-1-3-v2-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":82.45614035087719,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tiers-1-3-v2-private:overall","index":74.29562688609437,"aboveFit":false,"run":"xhigh effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.2569134425265194,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-chess-puzzles:overall","benchmarkName":"Chess Puzzles","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":59,"unit":"percent","family":"epoch-ai:epoch-chess-puzzles:overall","index":78.76657223899463,"aboveFit":false,"run":"xhigh effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":4.943110704237102,"costPerTest":null,"latency":null},{"benchmarkId":"hle","benchmarkName":"Humanity's Last Exam","category":"knowledge","score":58.7,"unit":"percent","family":"hle","index":83.27992844656241,"aboveFit":false,"run":null,"source":"Center for AI Safety, Scale AI, and thousands of expert contributors","sourceUrl":"https://lastexam.ai/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tier-4-v2-private:overall","benchmarkName":"FrontierMath-Tier-4-v2-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":58.536585365853654,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tier-4-v2-private:overall","index":71.48440432900573,"aboveFit":false,"run":"xhigh effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":7.789619230571372,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-2025-02-28-private:overall","benchmarkName":"FrontierMath-2025-02-28-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":50,"unit":"percent","family":"epoch-ai:epoch-frontiermath-2025-02-28-private:overall","index":74.05305467586828,"aboveFit":false,"run":"xhigh effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.9000000000000004,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-simpleqa-verified:overall","benchmarkName":"SimpleQA Verified","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":46.300000000000004,"unit":"percent","family":"epoch-ai:epoch-simpleqa-verified:overall","index":62.49103959348899,"aboveFit":false,"run":"xhigh effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.5775927227262425,"costPerTest":null,"latency":null},{"benchmarkId":"hleNoTools","benchmarkName":"Humanity's Last Exam without tools","category":"knowledge","score":42.7,"unit":"percent","family":"hle","index":67.2272445260857,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-private:overall","benchmarkName":"FrontierMath-Tier-4-2025-07-01-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":37.5,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-private:overall","index":81.29045823509466,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":7.000000000000001,"costPerTest":null,"latency":null},{"benchmarkId":"frontierScience","benchmarkName":"FrontierScience","category":"knowledge","score":36.7,"unit":"percent","family":"frontierScience","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"frontierScienceResearch","benchmarkName":"FrontierScience Research","category":"knowledge","score":36.7,"unit":"percent","family":"frontierScienceResearch","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":30,"unit":"percent","family":"critpt","index":94.92561551373953,"aboveFit":true,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":30,"outputPerMillion":180,"batch":{"inputPerMillion":15,"outputPerMillion":90},"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":21,"timeToFirstToken":87.999,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":60,"benchmarkCount":12,"categoryCount":4,"confidence":"low","state":"available"},"hosts":[{"host":"digitalocean","name":"DigitalOcean","kind":"cloud","url":"https://docs.digitalocean.com/products/gradient-ai-platform/details/models/","inputPerMillion":30,"outputPerMillion":180,"cacheReadPerMillion":null,"contextTokens":1050000,"maxOutputTokens":128000,"deprecated":false},{"host":"azure","name":"Microsoft Azure","kind":"cloud","url":"https://learn.microsoft.com/en-us/azure/ai-services/openai/concepts/models","inputPerMillion":30,"outputPerMillion":180,"cacheReadPerMillion":null,"contextTokens":1050000,"maxOutputTokens":128000,"deprecated":false},{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":30,"outputPerMillion":180,"cacheReadPerMillion":null,"contextTokens":1050000,"maxOutputTokens":128000,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:arc-prize-foundation","name":"ARC Prize Foundation","url":"https://arcprize.org","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"muse-wonder","providerId":"meta","slug":"muse-wonder","name":"Muse Wonder","provider":"Meta","description":"Muse Wonder is a reasoning model from Meta. 43 benchmarks count toward its index, in 6 categories.","logoUrl":null,"releaseDate":"2026-10-12","contextWindowTokens":1048576,"modalities":["text","image","video","file","audio"],"openWeights":false,"details":{"apiId":"meta/muse-wonder","maxOutputTokens":943718,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.19999999999999998,"weightsUrl":null},"reasoning":true,"overallScore":73.13299901867316,"overallScoreRange":{"low":69.16709713639494,"high":77.09890090095138},"badges":[],"capabilityScores":{"agentic":72.97242239118678,"coding":72.29142844311065,"reasoning":73.06907451396286,"multimodal":null,"knowledge":68.61250663960244,"multilingual":null,"instruction":74.35693644542437,"math":73.50925478019258},"capabilityCounts":{"math":5,"reasoning":7,"agentic":14,"coding":12,"knowledge":4,"instruction":1},"benchmarks":[{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":99.44444444444444,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":66.77527653557718,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0.3883473863441843,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-math:overall","benchmarkName":"LiveBench Mathematics","category":"math","domain":null,"eligible":true,"ranked":true,"score":96.47049999999999,"unit":"percent","family":"livebench:livebench-math:overall","index":74.227621355683,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Mathematics","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.21863377308707124,"latency":null},{"benchmarkId":"livebench:livebench-reasoning:overall","benchmarkName":"LiveBench Reasoning","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":90.6345,"unit":"percent","family":"livebench","index":79.03073385916449,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Reasoning","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.29836287128712874,"latency":null},{"benchmarkId":"livebench:livebench-research:overall","benchmarkName":"LiveBench Research","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":88,"unit":"percent","family":"livebench-2-agentic","index":82.94660793537042,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Research","measuredAt":"2026-10-14","stderr":null,"costPerTest":3.5057045454545457,"latency":null},{"benchmarkId":"livebench:livebench-coding-2:overall","benchmarkName":"LiveBench Coding 2.0","category":"coding","domain":null,"eligible":true,"ranked":true,"score":86.2,"unit":"percent","family":"livebench:livebench-coding-2:overall","index":70.60874043029663,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Coding","measuredAt":"2026-10-14","stderr":null,"costPerTest":3.102251724137931,"latency":null},{"benchmarkId":"vals-ai:proof_bench:overall","benchmarkName":"ProofBench v1.1","category":"math","domain":null,"eligible":true,"ranked":true,"score":84,"unit":"percent","family":"vals-ai:proof_bench:overall","index":76.15717547396898,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/proof_bench","measuredAt":"2026-11-09","stderr":3.685,"costPerTest":1.313345,"latency":929.053},{"benchmarkId":"vals-ai:vibe-code:overall","benchmarkName":"Vibe Code Bench v1.1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":83.089,"unit":"percent","family":"vals-ai:vibe-code:overall","index":72.22933921705774,"aboveFit":false,"run":"OpenHands","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","measuredAt":"2026-11-09","stderr":2.49,"costPerTest":1.802723,"latency":350.076},{"benchmarkId":"livebench:livebench-language:overall","benchmarkName":"LiveBench Language","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":82.93233333333333,"unit":"percent","family":"livebench:livebench-language:overall","index":68.33942191235644,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Language","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.23787199999999997,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tiers-1-3-v2-private:overall","benchmarkName":"FrontierMath-Tiers-1-3-v2-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":82.10526315789474,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tiers-1-3-v2-private:overall","index":74.11094207511238,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.274515950332741,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-data-analysis:overall","benchmarkName":"LiveBench Data Analysis","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":81.61733333333333,"unit":"percent","family":"livebench","index":67.34958961024496,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Data%20Analysis","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.36573666666666665,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":80.7,"unit":"percent","family":"lcr","index":62.664747873933486,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-coding:overall","benchmarkName":"LiveBench Coding","category":"coding","domain":null,"eligible":true,"ranked":true,"score":80.68,"unit":"percent","family":"livebench:livebench-coding:overall","index":68.29322876016123,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Coding","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.11916495726495727,"latency":null},{"benchmarkId":"vals-ai:cyber:overall","benchmarkName":"CyberBench v1.1","category":null,"domain":"Other","eligible":false,"ranked":true,"score":80.357,"unit":"percent","family":"vals-ai:cyber:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/cyber","measuredAt":"2026-11-05","stderr":5.133,"costPerTest":3.658792,"latency":824.81},{"benchmarkId":"deepSwe","benchmarkName":"DeepSWE","category":"agentic","score":78.8,"unit":"percent","family":"deepSwe","index":78.3621208515898,"aboveFit":false,"run":null,"source":"Datacurve AI","sourceUrl":"https://deepswe.datacurve.ai/blog","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-design:overall","benchmarkName":"LiveBench Design","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":75.1,"unit":"percent","family":"livebench-2-agentic","index":73.51243681089652,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Design","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.687108,"latency":null},{"benchmarkId":"livebench:livebench-instruction-following:overall","benchmarkName":"LiveBench Instruction Following","category":"instruction","domain":null,"eligible":true,"ranked":true,"score":73.02924999999999,"unit":"percent","family":"livebench:livebench-instruction-following:overall","index":74.35693644542437,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=IF","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.32889450000000003,"latency":null},{"benchmarkId":"osWorld21","benchmarkName":"OSWorld 2.1 (all 108 tasks)","category":"agentic","score":72.6,"unit":"percent","family":"osWorld21","index":71.69511593321232,"aboveFit":false,"run":null,"source":"Mengqi Yuan, Zilong Zhou, Xinzhuang Xiong, Weiming Wu, Jiayang Sun, Jiamin Song, Kaiqian Cui, Bowen Wang, Haoyuan Wu, Yitong Li, Dunjie Lu, Haikong Lu, Qi Zhen, Xinyuan Wang, Jiaqi Deng, Yuhao Yang, Cheng Chen, Boyuan Zheng, Alex Su, Xiao Yu, Hao Zou, Saaket Agashe, Xing Han Lu, Manpreet Kaur, Zhengyang Qi, Vincent Sunn Chen, Frederic Sala, Dayiheng Liu, Junyang Lin, Zhou Yu, Yu Su, Siva Reddy, Xin Eric Wang, Peng Qi, Tianbao Xie, Tao Yu","sourceUrl":"https://github.com/xlang-ai/OSWorld-V2/releases","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-browser:overall","benchmarkName":"LiveBench Browser Use","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":72,"unit":"percent","family":"livebench-2-agentic","index":71.24531041664311,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Browser","measuredAt":"2026-10-14","stderr":null,"costPerTest":4.138132,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tier-4-v2-private:overall","benchmarkName":"FrontierMath-Tier-4-v2-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":70.73170731707317,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tier-4-v2-private:overall","index":76.27525846062139,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":7.194088392074452,"costPerTest":null,"latency":null},{"benchmarkId":"jobBench","benchmarkName":"JobBench","category":"agentic","score":70.5,"unit":"percent","family":"jobBench","index":75.48708540481792,"aboveFit":false,"run":null,"source":"Yuetai Li, Yichen Feng, Zhangchen Xu, Zixian Ma, Kaiyuan Zheng, Fengqing Jiang, Xinghua Sun, Rulin Shao, Zichen Chen, Yue Huang, Xinyang Han, Brian Lee, Kayla Xu, Shenglai Zeng, Hang Hua, Xiangliang Zhang, Basel Alomair, Ranjay Krishna, Luke Zettlemoyer, Pang Wei Koh, Bhaskar Ramasubramanian, Luyao Niu, Xiang Yue, Radha Poovendran","sourceUrl":"https://arxiv.org/abs/2605.26329","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:public-benefits-bench:overall","benchmarkName":"Public Benefits Bench v1.1","category":null,"domain":"Public benefits","eligible":false,"ranked":true,"score":70.23,"unit":"percent","family":"vals-ai:public-benefits-bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/public-benefits-bench","measuredAt":"2026-10-12","stderr":1.189,"costPerTest":2.457181,"latency":1184.49},{"benchmarkId":"vals-ai:emb:overall","benchmarkName":"EMB","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":67.084,"unit":"percent","family":"vals-ai:emb:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/emb","measuredAt":"2026-10-20","stderr":2.891,"costPerTest":5.599337,"latency":1034.437},{"benchmarkId":"vals-ai:biomysterybench:overall","benchmarkName":"BioMysteryBench","category":null,"domain":"Science","eligible":false,"ranked":true,"score":67.037,"unit":"percent","family":"vals-ai:biomysterybench:overall","index":null,"aboveFit":false,"run":"Terminus 2","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/biomysterybench","measuredAt":"2026-11-09","stderr":0.37,"costPerTest":3.228928,"latency":1081.905},{"benchmarkId":"vals-ai:ioi:overall","benchmarkName":"IOI","category":"coding","domain":null,"eligible":true,"ranked":true,"score":66.944,"unit":"percent","family":"vals-ai:ioi:overall","index":71.24891137235991,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/ioi","measuredAt":"2026-10-29","stderr":4.908,"costPerTest":4.541594,"latency":1502.067},{"benchmarkId":"cweBenchV2","benchmarkName":"CWE-bench v2","category":"agentic","score":66.1,"unit":"percent","family":"cweBenchV2","index":79.13025391364795,"aboveFit":false,"run":null,"source":"Collinear AI","sourceUrl":"https://cwe-bench.com/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gdpvalAaNormalized","benchmarkName":"GDPval-AA normalized","category":"agentic","score":63.5,"unit":"percent","family":"gdpvalAaNormalized","index":79.43479928462412,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-agentic-coding:overall","benchmarkName":"LiveBench Agentic Coding","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":62.72733333333334,"unit":"percent","family":"livebench:livebench-agentic-coding:overall","index":73.85232087738993,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Agentic%20Coding","measuredAt":"2026-06-25","stderr":null,"costPerTest":4.454754166666667,"latency":null},{"benchmarkId":"vals-ai:code-migration:overall","benchmarkName":"Code Migration","category":"coding","domain":null,"eligible":true,"ranked":true,"score":62.552,"unit":"percent","family":"vals-ai:code-migration:overall","index":78.41926778584535,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/code-migration","measuredAt":"2026-10-29","stderr":4.354,"costPerTest":69.535349,"latency":10267.754},{"benchmarkId":"healthBenchProfessional","benchmarkName":"HealthBench Professional","category":"knowledge","score":60.5,"unit":"percent","family":"healthBenchProfessional","index":72.87931594586017,"aboveFit":false,"run":null,"source":"Rebecca Soskin Hicks, Mikhail Trofimov, Dominick Lim, Rahul K. Arora, Foivos Tsimpourlas, Preston Bowman, Michael Sharman, Chi Tong, Kavin Karthik, Arnav Dugar, Akshay Jagadeesh, Khaled Saab, Johannes Heidecke, Ashley Alexander, Nate Gross, Karan Singhal","sourceUrl":"https://arxiv.org/abs/2604.27470","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaSciCode","benchmarkName":"Artificial Analysis SciCode","category":"coding","score":60.3,"unit":"percent","family":"scicode-family","index":74.45237457131967,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/scicode","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:fabv2:overall","benchmarkName":"Finance Agent (v2)","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":56.637,"unit":"percent","family":"vals-ai:fabv2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/fabv2","measuredAt":"2026-10-29","stderr":0.382,"costPerTest":2.668231,"latency":368.853},{"benchmarkId":"epoch-ai:epoch-furniture-assembly:overall","benchmarkName":"Furniture Assembly","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":54.166666666666664,"unit":"percent","family":"epoch-ai:epoch-furniture-assembly:overall","index":71.91474680693977,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":6.432138878020231,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-long-horizon:overall","benchmarkName":"LiveBench Long Agentic","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":53.2,"unit":"percent","family":"livebench-2-agentic","index":57.49628583213851,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Long%20Agentic","measuredAt":"2026-10-14","stderr":null,"costPerTest":14.379889655172413,"latency":null},{"benchmarkId":"sweMarathon","benchmarkName":"SWE-Marathon","category":"coding","score":53.1,"unit":"percent","family":"sweMarathon","index":null,"aboveFit":false,"run":null,"source":"Abundant AI and BenchFlow","sourceUrl":"https://www.swe-marathon.org/","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaTerminalBench4","benchmarkName":"Artificial Analysis Terminal-Bench v4.0","category":"agentic","score":52,"unit":"percent","family":"aaTerminalBench4","index":75.86071958648239,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/terminalbench-v4-0","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-mystery-game-puzzles:overall","benchmarkName":"Mystery Game Puzzles","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":51,"unit":"percent","family":"epoch-ai:epoch-mystery-game-puzzles:overall","index":78.17615793403395,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":5.024183937956913,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:legal_research:overall","benchmarkName":"Legal Research Bench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":50.481,"unit":"percent","family":"vals-ai:legal_research:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_research","measuredAt":"2026-11-09","stderr":3.475,"costPerTest":3.980242,"latency":927.111},{"benchmarkId":"cursorBench40","benchmarkName":"cursorBench40","category":"coding","score":50.4,"unit":"percent","family":"cursorBench40","index":75.34115680455142,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":50.1,"unit":"percent","family":"hle","index":74.65161083930617,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"programBench","benchmarkName":"ProgramBench: Can Language Models Rebuild Programs From Scratch?","category":"coding","score":50,"unit":"percent","family":"programBench","index":71.77498718144835,"aboveFit":false,"run":null,"source":"John Yang, Kilian Lieret, Jeffrey Ma, Parth Thakkar, Dmitrii Pedchenko, Sten Sootla, Emily McMilin, Pengcheng Yin, Rui Hou, Gabriel Synnaeve, Diyi Yang, Ofir Press","sourceUrl":"https://programbench.com/static/paper.pdf","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:terminal-bench-4:overall","benchmarkName":"Terminal-Bench 4.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":49.495,"unit":"percent","family":"terminal-bench-4","index":78.01815152793199,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-4","measuredAt":"2026-11-09","stderr":2.201,"costPerTest":14.657694,"latency":3280.687},{"benchmarkId":"mka","benchmarkName":"Multimodal Knowledge Agent Benchmark","category":"agentic","score":47.5,"unit":"percent","family":"mka","index":75.77711522158506,"aboveFit":false,"run":null,"source":"Google DeepMind, in partnership with Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mka","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-mirrorcode:overall","benchmarkName":"MirrorCode","category":"coding","domain":null,"eligible":true,"ranked":true,"score":45.55555555555556,"unit":"percent","family":"epoch-ai:epoch-mirrorcode:overall","index":69.57233181169202,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":10.599324460188285,"costPerTest":null,"latency":null},{"benchmarkId":"aaAutomationBench","benchmarkName":"Artificial Analysis AutomationBench","category":"agentic","score":39.1,"unit":"percent","family":"aaAutomationBench","index":51.29773668266443,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/automationbench-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"omniscienceAccuracy","benchmarkName":"Artificial Analysis Omniscience Accuracy","category":"knowledge","score":38,"unit":"percent","family":"omniscienceAccuracy","index":58.57967786088696,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:tax_agent_bench:overall","benchmarkName":"Tax Agent Bench","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":36.269,"unit":"percent","family":"vals-ai:tax_agent_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench","measuredAt":"2026-11-09","stderr":3.47,"costPerTest":1.884439,"latency":492.843},{"benchmarkId":"bug-hunt-bench","benchmarkName":"Bug Hunt Bench","category":"coding","score":31.7,"unit":"percent","family":"bug-hunt-bench","index":71.14301839162096,"aboveFit":false,"run":null,"source":"Pawel Huryn","sourceUrl":"https://bughunt.productcompass.pm/method","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-ebr-bench:overall","benchmarkName":"EBR-bench","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":31.428571428571427,"unit":"percent","family":"epoch-ai:epoch-ebr-bench:overall","index":66.85021070503963,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":4.151332327181594,"costPerTest":null,"latency":null},{"benchmarkId":"aaGdpPdf","benchmarkName":"Artificial Analysis GDP.pdf","category":"agentic","score":30.6,"unit":"percent","family":"aaGdpPdf","index":78.55054741389284,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdp-pdf","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":28.3,"unit":"percent","family":"critpt","index":91.63461387535142,"aboveFit":true,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:mysterymechanism:overall","benchmarkName":"MysteryMechanism","category":null,"domain":"Science","eligible":false,"ranked":true,"score":27.477,"unit":"percent","family":"vals-ai:mysterymechanism:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mysterymechanism","measuredAt":"2026-11-11","stderr":3.003,"costPerTest":3.389451,"latency":1096.935},{"benchmarkId":"vals-ai:vcb-1-100:overall","benchmarkName":"Vibe Code Bench 1-100","category":"coding","domain":null,"eligible":true,"ranked":true,"score":26.55,"unit":"percent","family":"vals-ai:vcb-1-100:overall","index":74.37113212283847,"aboveFit":false,"run":"OpenHands","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vcb-1-100","measuredAt":"2026-10-29","stderr":4.333,"costPerTest":13.721565,"latency":1868.026},{"benchmarkId":"frontierCode2","benchmarkName":"FrontierCode 2.0","category":"coding","score":24.3,"unit":"percent","family":"frontierCode2","index":70.04265286813593,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:hlab:overall","benchmarkName":"HLAB-Vals-1.1","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":23.75,"unit":"percent","family":"vals-ai:hlab:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/hlab","measuredAt":"2026-11-09","stderr":3.646,"costPerTest":7.081987,"latency":906.14},{"benchmarkId":"vals-ai:terminal-bench-science:overall","benchmarkName":"Terminal-Bench-Science","category":null,"domain":"Science","eligible":false,"ranked":true,"score":22.857,"unit":"percent","family":"vals-ai:terminal-bench-science:overall","index":null,"aboveFit":false,"run":"Mini-SWE-agent","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-science","measuredAt":"2026-11-09","stderr":5.055,"costPerTest":19.609644,"latency":5584.198},{"benchmarkId":"aaMlcr","benchmarkName":"Medical Long Context Reasoning (MLCR-AA)","category":"reasoning","score":22.2,"unit":"percent","family":"aaMlcr","index":67.05288266773698,"aboveFit":false,"run":null,"source":"Wisedocs and Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mlcr-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaHarveyLabV11","benchmarkName":"Harvey LAB-AA v1.1","category":"agentic","score":10,"unit":"percent","family":"aaHarveyLabV11","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:cua_bench:overall","benchmarkName":"CUA-bench","category":null,"domain":"Other","eligible":false,"ranked":true,"score":7.166667,"unit":"percent","family":"vals-ai:cua_bench:overall","index":null,"aboveFit":false,"run":"muse-code","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/cua_bench","measuredAt":"2026-10-13","stderr":0,"costPerTest":597.211635,"latency":30.56},{"benchmarkId":"lmarena:agent-task-outcome:overall","benchmarkName":"Agent Arena task outcome","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":5.378923217176334,"unit":"index","family":"agent-arena","index":70.95311945609032,"aboveFit":false,"run":"max effort","source":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset","measuredAt":"2026-11-11","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"stemBench120","benchmarkName":"STEMBench120: six-domain scientific research progress","category":"reasoning","score":2.683,"unit":"percent","family":"stemBench120","index":null,"aboveFit":false,"run":null,"source":"Ulam","sourceUrl":"https://www.ulam.ai/stembench","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lmarena:agent-steerability:overall","benchmarkName":"Agent Arena steerability","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":2.5504756996694633,"unit":"index","family":"agent-arena","index":68.627692177227,"aboveFit":false,"run":"max effort","source":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset","measuredAt":"2026-11-11","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mazeBench","benchmarkName":"MazeBench: visual spatial reasoning in a 3D open world","category":"agentic","score":0,"unit":"percent","family":"mazeBench","index":61.97866734834783,"aboveFit":false,"run":null,"source":"Jonathan Pappas and David Pappas","sourceUrl":"https://mazebench.com/blog?post=maze-bench-results","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":4,"outputPerMillion":20,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":80,"timeToFirstToken":2.705,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":{"elo":1503.0484713809174,"eloUpper":1509.3772338127997,"eloLower":1496.719708949035,"votes":12032,"rank":6,"rankUpper":null,"rankLower":null,"arenaName":"Muse Wonder-max","organization":"meta","updatedAt":"2026-11-09"},"quality":{"coveragePercent":85,"benchmarkCount":43,"categoryCount":6,"confidence":"medium","state":"available"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:livebench","name":"LiveBench","url":"https://livebench.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:lmarena","name":"LMArena","url":"https://huggingface.co","license":"CC BY 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gpt-5-6-sol","providerId":"openai","slug":"gpt-5-6-sol","name":"GPT-5.6 Sol","provider":"OpenAI","description":"GPT-5.6 Sol is a reasoning model from OpenAI in the GPT-5.6 family. 70 benchmarks count toward its index, in 8 categories.","logoUrl":null,"releaseDate":"2026-07-09","contextWindowTokens":1050000,"modalities":["file","image","text"],"openWeights":false,"details":{"apiId":"openai/gpt-5.6-sol","maxOutputTokens":128000,"knowledgeCutoff":"2026-02-16","retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.44,"weightsUrl":null},"reasoning":true,"overallScore":72.97297497881291,"overallScoreRange":{"low":71.0402903175192,"high":74.90565964010663},"badges":[],"capabilityScores":{"agentic":71.94018735632024,"coding":70.41659893702888,"reasoning":76.26628354588013,"multimodal":72.57942949329549,"knowledge":72.71746923024673,"multilingual":66.42011052971586,"instruction":64.46261432165223,"math":75.12573230564459},"capabilityCounts":{"math":5,"reasoning":10,"coding":20,"knowledge":8,"multimodal":3,"agentic":21,"multilingual":1,"instruction":2},"benchmarks":[{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":100,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":67.09849544987334,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0,"costPerTest":null,"latency":null},{"benchmarkId":"arc-prize:v1-public-eval:overall","benchmarkName":"ARC-AGI-1 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":99,"unit":"percent","family":"arc-prize:v1-public-eval:overall","index":null,"aboveFit":false,"run":"max effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.37,"latency":null},{"benchmarkId":"arc-prize:v1-semi-private:overall","benchmarkName":"ARC-AGI-1 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":96.5,"unit":"percent","family":"arc-prize:v1-semi-private:overall","index":71.35189279722005,"aboveFit":false,"run":"max effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.54,"latency":null},{"benchmarkId":"vals-ai:swebench:overall","benchmarkName":"SWE-bench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":96.2,"unit":"percent","family":"swe-bench-verified","index":72.18096651226281,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/swebench","measuredAt":"2026-09-01","stderr":0.856,"costPerTest":1.1513,"latency":182.365},{"benchmarkId":"livebench:livebench-math:overall","benchmarkName":"LiveBench Mathematics","category":"math","domain":null,"eligible":true,"ranked":true,"score":96.19775,"unit":"percent","family":"livebench:livebench-math:overall","index":73.84748441838263,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Mathematics","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.1381245382585752,"latency":null},{"benchmarkId":"vals-ai:gpqa:overall","benchmarkName":"GPQA Diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":95.202,"unit":"percent","family":"gpqa-diamond","index":64.52577373556291,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","measuredAt":"2026-09-01","stderr":1.074,"costPerTest":0.10996,"latency":58.304},{"benchmarkId":"gpqa","benchmarkName":"Graduate-Level Google-Proof Q&A","category":"knowledge","score":94.6,"unit":"percent","family":"gpqa-diamond","index":63.97363578526119,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gpqaDiamond","benchmarkName":"GPQA Diamond","category":"knowledge","score":94.6,"unit":"percent","family":"gpqa-diamond","index":63.97363578526119,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":94.1,"unit":"percent","family":"gpqa-diamond","index":63.515049447801296,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"extractBenchLlamaIndex","benchmarkName":"ExtractBench: schema-guided enterprise document extraction (LlamaIndex leaderboard)","category":"multimodal","score":93.77,"unit":"percent","family":"extractBenchLlamaIndex","index":76.52135938095438,"aboveFit":false,"run":null,"source":"Boyang Zhang, Adrian Lyjak, Eli Stewart, Zhaoqi Li and Simon Suo (LlamaIndex)","sourceUrl":"https://arxiv.org/abs/2607.29677","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"arc-prize:v2-public-eval:overall","benchmarkName":"ARC-AGI-2 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":93.75,"unit":"percent","family":"arc-prize:v2-public-eval:overall","index":null,"aboveFit":false,"run":"max effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":1.52,"latency":null},{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":93.49747474747475,"unit":"percent","family":"gpqa-diamond","index":62.962429750236005,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.570109070867593,"costPerTest":null,"latency":null},{"benchmarkId":"arc-prize:v2-semi-private:overall","benchmarkName":"ARC-AGI-2 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":92.5,"unit":"percent","family":"arc-prize:v2-semi-private:overall","index":81.65847524253698,"aboveFit":false,"run":"max effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":1.44,"latency":null},{"benchmarkId":"browseComp","benchmarkName":"BrowseComp","category":"agentic","score":92.2,"unit":"percent","family":"browseComp","index":72.74882016373493,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"terminalBench21","benchmarkName":"Terminal-Bench 2.1","category":"agentic","score":91.9,"unit":"percent","family":"terminal-bench-2-1","index":74.88932920242166,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-reasoning:overall","benchmarkName":"LiveBench Reasoning","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":91.65375,"unit":"percent","family":"livebench","index":80.35110495973686,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Reasoning","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.2853633663366337,"latency":null},{"benchmarkId":"vals-ai:ioi:overall","benchmarkName":"IOI","category":"coding","domain":null,"eligible":true,"ranked":true,"score":91.167,"unit":"percent","family":"vals-ai:ioi:overall","index":80.34276818691261,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/ioi","measuredAt":"2026-10-29","stderr":4.51,"costPerTest":7.982135,"latency":3711.631},{"benchmarkId":"livebench:livebench-coding-2:overall","benchmarkName":"LiveBench Coding 2.0","category":"coding","domain":null,"eligible":true,"ranked":true,"score":89.7,"unit":"percent","family":"livebench:livebench-coding-2:overall","index":73.14387819634575,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Coding","measuredAt":"2026-10-14","stderr":null,"costPerTest":3.7688310344827585,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tiers-1-3-v2-private:overall","benchmarkName":"FrontierMath-Tiers-1-3-v2-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":89.12280701754386,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tiers-1-3-v2-private:overall","index":77.80463829475251,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.8475392571140503,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:mmlu_pro:overall","benchmarkName":"MMLU Pro","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":89.1,"unit":"percent","family":"mmlu-pro","index":56.73606343109718,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","measuredAt":"2026-09-01","stderr":0.308,"costPerTest":0.032087,"latency":15.654},{"benchmarkId":"vals-ai:mmmu:overall","benchmarkName":"MMMU Pro","category":"multimodal","domain":null,"eligible":true,"ranked":true,"score":88.844,"unit":"percent","family":"mmmu-pro","index":70.77548631008278,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmmu","measuredAt":"2026-09-01","stderr":0.757,"costPerTest":0.055795,"latency":26.198},{"benchmarkId":"livebench:livebench-browser:overall","benchmarkName":"LiveBench Browser Use","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":88,"unit":"percent","family":"livebench-2-agentic","index":82.94660793537042,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Browser","measuredAt":"2026-10-14","stderr":null,"costPerTest":1.3381720000000001,"latency":null},{"benchmarkId":"livebench:livebench-language:overall","benchmarkName":"LiveBench Language","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":87.68366666666667,"unit":"percent","family":"livebench:livebench-language:overall","index":73.7122754368893,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Language","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.17876533333333333,"latency":null},{"benchmarkId":"vulcanBench","benchmarkName":"VulcanBench v3","category":"coding","score":87,"unit":"percent","family":"vulcanBench","index":68.4175418120868,"aboveFit":false,"run":null,"source":"VulcanBench contributors","sourceUrl":"https://github.com/morganlinton/VulcanBench/tree/main","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:legal_bench:overall","benchmarkName":"LegalBench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":86.965,"unit":"percent","family":"vals-ai:legal_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","measuredAt":"2026-10-13","stderr":0.412,"costPerTest":0.012678,"latency":6.198},{"benchmarkId":"vals-ai:ioi-v1:overall","benchmarkName":"IOI v1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":86.667,"unit":"percent","family":"vals-ai:ioi-v1:overall","index":83.26216904215454,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/ioi-v1","measuredAt":"2026-08-09","stderr":1.667,"costPerTest":2.98545,"latency":1310.679},{"benchmarkId":"vulcanCiiV1","benchmarkName":"VulcanBench Coding Intelligence Index v1","category":"coding","score":86.5,"unit":"percent","family":"vulcanCiiV1","index":null,"aboveFit":false,"run":null,"source":"VulcanBench contributors","sourceUrl":"https://github.com/morganlinton/VulcanBench/blob/main/docs/results/cii-v1-2026-08/README.md","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:terminal-bench-2-1:overall","benchmarkName":"Terminal-Bench 2.1","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":85.768,"unit":"percent","family":"terminal-bench-2-1","index":71.30299610747335,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2-1","measuredAt":"2026-10-01","stderr":1.35,"costPerTest":1.023319,"latency":372.819},{"benchmarkId":"vals-ai:medscribe:overall","benchmarkName":"MedScribe","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":85.233,"unit":"percent","family":"vals-ai:medscribe:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medscribe","measuredAt":"2026-10-29","stderr":1.973,"costPerTest":0.276691,"latency":94.404},{"benchmarkId":"tau2Bench","benchmarkName":"τ²-Bench Tool-Agent-User Evaluation","category":"agentic","score":85.1,"unit":"percent","family":"tau2-bench","index":59.25951196986753,"aboveFit":false,"run":null,"source":"Victor Barres, Honghua Dong, Soham Ray, Xujie Si, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2506.07982","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmmuProPython","benchmarkName":"MMMU-Pro with Python","category":"multimodal","score":84.6,"unit":"percent","family":"mmmuProPython","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"cyberGym","benchmarkName":"CyberGym","category":"agentic","score":84.5,"unit":"percent","family":"cyberGym","index":69.48829616452488,"aboveFit":false,"run":null,"source":"Zhun Wang, Tianneng Shi, Jingxuan He, Matthew Cai, Jialin Zhang, Dawn Song","sourceUrl":"https://www.cybergym.io/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":84,"unit":"percent","family":"lcr","index":64.92881360836616,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-coding:overall","benchmarkName":"LiveBench Coding","category":"coding","domain":null,"eligible":true,"ranked":true,"score":83.941,"unit":"percent","family":"livebench:livebench-coding:overall","index":72.87945871538086,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Coding","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.10672136752136752,"latency":null},{"benchmarkId":"aaMmmuPro","benchmarkName":"Artificial Analysis MMMU-Pro","category":"multimodal","score":83.4,"unit":"percent","family":"mmmu-pro","index":63.69059793877406,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmmu-pro","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmmuPro","benchmarkName":"Massive Multi-discipline Multimodal Understanding Pro","category":"multimodal","score":83,"unit":"percent","family":"mmmu-pro","index":63.17003303272638,"aboveFit":false,"run":null,"source":"MMMU-Pro authors","sourceUrl":"https://arxiv.org/abs/2409.02813","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:proof_bench:overall","benchmarkName":"ProofBench v1.1","category":"math","domain":null,"eligible":true,"ranked":true,"score":83,"unit":"percent","family":"vals-ai:proof_bench:overall","index":75.8119307729774,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/proof_bench","measuredAt":"2026-11-09","stderr":3.775,"costPerTest":1.438313,"latency":520.405},{"benchmarkId":"epoch-ai:epoch-frontiermath-tier-4-v2-private:overall","benchmarkName":"FrontierMath-Tier-4-v2-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":82.92682926829268,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tier-4-v2-private:overall","index":81.06611259223705,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":5.949419959829497,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:lcb:overall","benchmarkName":"LiveCodeBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":82.604,"unit":"percent","family":"livecodebench","index":57.883710984041045,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","measuredAt":"2026-09-01","stderr":1.088,"costPerTest":0.089831,"latency":56.513},{"benchmarkId":"labBench2","benchmarkName":"LABBench2: An Improved Benchmark for AI Systems Performing Biology Research","category":"knowledge","score":82.1,"unit":"percent","family":"labBench2","index":null,"aboveFit":false,"run":null,"source":"Jon M. Laurent, Albert Bou, Michael Pieler, Conor Igoe, Alex Andonian, Siddharth Narayanan, James Braza, Alexandros Sanchez Vassopoulos, Jacob L. Steenwyk, Blake Lash, Andrew D. White, Samuel G. Rodriques","sourceUrl":"https://arxiv.org/abs/2604.09554","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-design:overall","benchmarkName":"LiveBench Design","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":81.6,"unit":"percent","family":"livebench-2-agentic","index":78.26608892787948,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Design","measuredAt":"2026-10-14","stderr":null,"costPerTest":1.706208,"latency":null},{"benchmarkId":"vals-ai:vibe-code:overall","benchmarkName":"Vibe Code Bench v1.1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":80.495,"unit":"percent","family":"vals-ai:vibe-code:overall","index":71.29338326226114,"aboveFit":false,"run":"OpenHands","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","measuredAt":"2026-11-09","stderr":3.716,"costPerTest":33.403634,"latency":2033.047},{"benchmarkId":"lifeVal","benchmarkName":"LifeVal: everyday life tasks","category":"agentic","score":80,"unit":"percent","family":"lifeVal","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-data-analysis:overall","benchmarkName":"LiveBench Data Analysis","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":79.84066666666666,"unit":"percent","family":"livebench","index":65.04803521042457,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Data%20Analysis","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.35311000000000003,"latency":null},{"benchmarkId":"vals-ai:cyber:overall","benchmarkName":"CyberBench v1.1","category":null,"domain":"Other","eligible":false,"ranked":true,"score":76.31,"unit":"percent","family":"vals-ai:cyber:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/cyber","measuredAt":"2026-11-05","stderr":5.177,"costPerTest":3.048302,"latency":646.763},{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":74.89375,"unit":"percent","family":"euroeval","index":73.83382164758409,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":0.4781213235152768,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:tax_eval_v2:overall","benchmarkName":"TaxEval v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":74.775,"unit":"percent","family":"vals-ai:tax_eval_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","measuredAt":"2026-09-01","stderr":0.856,"costPerTest":0.138418,"latency":62.034},{"benchmarkId":"parseBenchLlamaIndex","benchmarkName":"ParseBench: document parsing for AI agents (LlamaIndex leaderboard)","category":"multimodal","score":74.4,"unit":"percent","family":"parseBenchLlamaIndex","index":77.52633116015802,"aboveFit":false,"run":null,"source":"Boyang Zhang, Sebastián G. Acosta, Preston Carlson, Sacha Bron, Pierre-Loïc Doulcet, Daniel B. Ospina and Simon Suo (LlamaIndex)","sourceUrl":"https://arxiv.org/abs/2604.08538","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"deepSwe","benchmarkName":"DeepSWE","category":"agentic","score":72.7,"unit":"percent","family":"deepSwe","index":73.92759113922668,"aboveFit":false,"run":null,"source":"Datacurve AI","sourceUrl":"https://deepswe.datacurve.ai/blog","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaIfBench","benchmarkName":"Artificial Analysis IFBench","category":"instruction","score":72.7,"unit":"percent","family":"ifbench-family","index":56.962551090169775,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:emb:overall","benchmarkName":"EMB","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":72.339,"unit":"percent","family":"vals-ai:emb:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/emb","measuredAt":"2026-10-20","stderr":2.373,"costPerTest":6.010773,"latency":590.258},{"benchmarkId":"euroeval:euroeval-french:overall","benchmarkName":"EuroEval French","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":71.98125,"unit":"percent","family":"euroeval","index":69.88222929474182,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/french","measuredAt":null,"stderr":0.41864737548442843,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-instruction-following:overall","benchmarkName":"LiveBench Instruction Following","category":"instruction","domain":null,"eligible":true,"ranked":true,"score":71.846,"unit":"percent","family":"livebench:livebench-instruction-following:overall","index":71.96267755313465,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=IF","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.41762699999999997,"latency":null},{"benchmarkId":"vals-ai:biomysterybench:overall","benchmarkName":"BioMysteryBench","category":null,"domain":"Science","eligible":false,"ranked":true,"score":71.111,"unit":"percent","family":"vals-ai:biomysterybench:overall","index":null,"aboveFit":false,"run":"Terminus 2","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/biomysterybench","measuredAt":"2026-11-09","stderr":0.642,"costPerTest":1.925036,"latency":1098.261},{"benchmarkId":"euroeval:euroeval-portuguese:overall","benchmarkName":"EuroEval Portuguese","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":70.45375000000001,"unit":"percent","family":"euroeval","index":67.80976326247864,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/portuguese","measuredAt":null,"stderr":0.43801576741026116,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-italian:overall","benchmarkName":"EuroEval Italian","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":70.14125,"unit":"percent","family":"euroeval","index":67.3857726666801,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/italian","measuredAt":null,"stderr":0.3725461380822515,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-simpleqa-verified:overall","benchmarkName":"SimpleQA Verified","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":69.69999999999999,"unit":"percent","family":"epoch-ai:epoch-simpleqa-verified:overall","index":81.98530530625422,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.4539683710535192,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-dutch:overall","benchmarkName":"EuroEval Dutch","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":68.71777777777778,"unit":"percent","family":"euroeval","index":65.45444839275162,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/dutch","measuredAt":null,"stderr":0.43964632250126795,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:mortgage_tax:overall","benchmarkName":"MortgageTax","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":67.289,"unit":"percent","family":"vals-ai:mortgage_tax:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mortgage_tax","measuredAt":"2026-09-01","stderr":0.922,"costPerTest":0.071198,"latency":12.618},{"benchmarkId":"cursorBench32","benchmarkName":"cursorBench32","category":"coding","score":67.2,"unit":"percent","family":"cursorBench32","index":71.05322671134604,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:public-benefits-bench:overall","benchmarkName":"Public Benefits Bench v1.1","category":null,"domain":"Public benefits","eligible":false,"ranked":true,"score":66.509,"unit":"percent","family":"vals-ai:public-benefits-bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/public-benefits-bench","measuredAt":"2026-10-12","stderr":1.228,"costPerTest":8.847882,"latency":3677.634},{"benchmarkId":"euroeval:euroeval-spanish:overall","benchmarkName":"EuroEval Spanish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":66.25875,"unit":"percent","family":"euroeval","index":62.118113504479204,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/spanish","measuredAt":null,"stderr":0.3649743141647094,"costPerTest":null,"latency":null},{"benchmarkId":"terminalBenchHard","benchmarkName":"Terminal-Bench Hard","category":"agentic","score":65.9,"unit":"percent","family":"terminalBenchHard","index":73.18649660230983,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/terminalbench-hard","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"swePro","benchmarkName":"SWE-bench Pro","category":"coding","score":64.6,"unit":"percent","family":"swePro","index":63.853388458657214,"aboveFit":false,"run":null,"source":"Xiang Deng, Jeff Da, Edwin Pan, Yannis Yiming He, Charles Ide, Kanak Garg, Niklas Lauffer, Andrew Park, Nitin Pasari, Chetan Rane, Karmini Sampath, Maya Krishnan, Srivatsa Kundurthy, Sean Hendryx, Zifan Wang, Chen Bo Calvin Zhang, Noah Jacobson, Bing Liu, Brad Kenstler","sourceUrl":"https://arxiv.org/abs/2509.16941","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:corp_fin_v2:overall","benchmarkName":"CorpFin v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":64.375,"unit":"percent","family":"vals-ai:corp_fin_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","measuredAt":"2026-08-12","stderr":0.943,"costPerTest":0.484982,"latency":15.941},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":63.6075,"unit":"percent","family":"euroeval","index":58.5209772897245,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.47082706485502723,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":62.83571428571429,"unit":"percent","family":"euroeval","index":57.47384165827236,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.4049212571473569,"costPerTest":null,"latency":null},{"benchmarkId":"osWorld2","benchmarkName":"OSWorld 2.0","category":"agentic","score":62.6,"unit":"percent","family":"osWorld2","index":79.382302505942,"aboveFit":false,"run":null,"source":"Mengqi Yuan, Zilong Zhou, Xinzhuang Xiong, Weiming Wu, Jiayang Sun, Jiamin Song, Kaiqian Cui, Bowen Wang, Haoyuan Wu, Yitong Li, Dunjie Lu, Haikong Lu, Qi Zhen, Xinyuan Wang, Jiaqi Deng, Yuhao Yang, Cheng Chen, Boyuan Zheng, Alex Su, Xiao Yu, Hao Zou, Saaket Agashe, Xing Han Lu, Manpreet Kaur, Zhengyang Qi, Vincent Sunn Chen, Frederic Sala, Dayiheng Liu, Junyang Lin, Zhou Yu, Yu Su, Siva Reddy, Xin Eric Wang, Peng Qi, Tianbao Xie, Tao Yu","sourceUrl":"https://arxiv.org/abs/2606.29537","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"frontierCode11Extended","benchmarkName":"FrontierCode 1.1 Extended","category":"coding","score":60.6,"unit":"percent","family":"frontierCode11Extended","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"healthBenchProfessional","benchmarkName":"HealthBench Professional","category":"knowledge","score":60.5,"unit":"percent","family":"healthBenchProfessional","index":72.87931594586017,"aboveFit":false,"run":null,"source":"Rebecca Soskin Hicks, Mikhail Trofimov, Dominick Lim, Rahul K. Arora, Foivos Tsimpourlas, Preston Bowman, Michael Sharman, Chi Tong, Kavin Karthik, Arnav Dugar, Akshay Jagadeesh, Khaled Saab, Johannes Heidecke, Ashley Alexander, Nate Gross, Karan Singhal","sourceUrl":"https://arxiv.org/abs/2604.27470","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"omniscienceAccuracy","benchmarkName":"Artificial Analysis Omniscience Accuracy","category":"knowledge","score":59.4,"unit":"percent","family":"omniscienceAccuracy","index":85.54915591976103,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-chess-puzzles:overall","benchmarkName":"Chess Puzzles","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":59,"unit":"percent","family":"epoch-ai:epoch-chess-puzzles:overall","index":78.76657223899463,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":4.943110704237102,"costPerTest":null,"latency":null},{"benchmarkId":"toolathlon","benchmarkName":"Toolathlon","category":"agentic","score":58,"unit":"percent","family":"toolathlon","index":69.107275227178,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-mystery-game-puzzles:overall","benchmarkName":"Mystery Game Puzzles","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":57.99999999999999,"unit":"percent","family":"epoch-ai:epoch-mystery-game-puzzles:overall","index":83.59706221097015,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":4.9604496374885825,"costPerTest":null,"latency":null},{"benchmarkId":"aaSciCode","benchmarkName":"Artificial Analysis SciCode","category":"coding","score":57.1,"unit":"percent","family":"scicode-family","index":70.28056290279252,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/scicode","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-furniture-assembly:overall","benchmarkName":"Furniture Assembly","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":56.666666666666664,"unit":"percent","family":"epoch-ai:epoch-furniture-assembly:overall","index":73.08018724391738,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":6.340911315327727,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-agentic-coding:overall","benchmarkName":"LiveBench Agentic Coding","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":56.211999999999996,"unit":"percent","family":"livebench:livebench-agentic-coding:overall","index":67.13316857779935,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Agentic%20Coding","measuredAt":"2026-06-25","stderr":null,"costPerTest":3.400068055555556,"latency":null},{"benchmarkId":"aaItbench","benchmarkName":"Artificial Analysis ITBench-AA","category":"agentic","score":56.2,"unit":"percent","family":"aaItbench","index":80.31528990195588,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/itbench-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gdpvalAaNormalized","benchmarkName":"GDPval-AA normalized","category":"agentic","score":55.4,"unit":"percent","family":"gdpvalAaNormalized","index":73.64348751492028,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hleVerified","benchmarkName":"HLE-Verified","category":"knowledge","score":54.5,"unit":"percent","family":"hleVerified","index":null,"aboveFit":false,"run":null,"source":"Weiqi Zhai, Zhihai Wang, Jinghang Wang, Boyu Yang, Xiaogang Li, Xiang Xu, Bohan Wang, Peng Wang, Xingzhe Wu, Anfeng Li, Qiyuan Feng, Yuhao Zhou, Shoulin Han, Wenjie Luo, Yiyuan Li, Yaxuan Wang, Ruixian Luo, Guojie Lin, Peiyao Xiao, Chengliang Xu, Ben Wang, Zeyu Wang, Zichao Chen, Jianan Ye, Yijie Hu, Jialong Chen, Zongwen Shen, Yuliang Xu, An Yang, Bowen Yu, Dayiheng Liu, Junyang Lin, Hu Wei, Que Shen, Bing Zhao","sourceUrl":"https://arxiv.org/abs/2602.13964","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-research:overall","benchmarkName":"LiveBench Research","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":54.5,"unit":"percent","family":"livebench-2-agentic","index":58.447016255535104,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Research","measuredAt":"2026-10-14","stderr":null,"costPerTest":14.425968181818183,"latency":null},{"benchmarkId":"vals-ai:skillsbench:overall","benchmarkName":"SkillsBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":54.1,"unit":"percent","family":"vals-ai:skillsbench:overall","index":65.41022151768505,"aboveFit":false,"run":"OpenHands","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/skillsbench","measuredAt":"2026-10-29","stderr":4.73,"costPerTest":4.13999,"latency":787.486},{"benchmarkId":"vals-ai:fabv2:overall","benchmarkName":"Finance Agent (v2)","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":53.756,"unit":"percent","family":"vals-ai:fabv2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/fabv2","measuredAt":"2026-10-29","stderr":0.846,"costPerTest":1.253419,"latency":1165.258},{"benchmarkId":"vals-ai:code-migration:overall","benchmarkName":"Code Migration","category":"coding","domain":null,"eligible":true,"ranked":true,"score":52.916,"unit":"percent","family":"vals-ai:code-migration:overall","index":73.30023824293858,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/code-migration","measuredAt":"2026-10-29","stderr":4.353,"costPerTest":24.541448,"latency":3551.876},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":49.5,"unit":"percent","family":"hle","index":74.04963519228829,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-long-horizon:overall","benchmarkName":"LiveBench Long Agentic","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":48.5,"unit":"percent","family":"livebench-2-agentic","index":54.059029686012366,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Long%20Agentic","measuredAt":"2026-10-14","stderr":null,"costPerTest":15.961511111111113,"latency":null},{"benchmarkId":"vals-ai:legal_research:overall","benchmarkName":"Legal Research Bench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":48.077,"unit":"percent","family":"vals-ai:legal_research:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_research","measuredAt":"2026-11-09","stderr":3.473,"costPerTest":21.605566,"latency":4631.383},{"benchmarkId":"aaAnalystAgent","benchmarkName":"Artificial Analysis AnalystAgent","category":"agentic","score":47.5,"unit":"percent","family":"aaAnalystAgent","index":69.18583814263772,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/aa-analyst-agent","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"tau2-bench:banking-knowledge:overall","benchmarkName":"τ²-bench Banking","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":46.90721649484536,"unit":"percent","family":"tau2-bench","index":33.91580776640389,"aboveFit":false,"run":"xhigh effort · Sierra","source":"Sierra Research","sourceUrl":"https://github.com/sierra-research/tau2-bench","measuredAt":"2026-10-20","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"bioBasicsBench","benchmarkName":"BioBasicsBench: everyday wet-lab science","category":"reasoning","score":45.56,"unit":"percent","family":"bioBasicsBench","index":null,"aboveFit":false,"run":null,"source":"OpenAI, Ligo","sourceUrl":"https://openai.com/index/measuring-how-ai-agents-handle-wet-lab-science/","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-ebr-bench:overall","benchmarkName":"EBR-bench","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":44.76190476190475,"unit":"percent","family":"epoch-ai:epoch-ebr-bench:overall","index":72.96924250881659,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":4.151332327181594,"costPerTest":null,"latency":null},{"benchmarkId":"aaTau3Banking","benchmarkName":"Artificial Analysis Tau3-Banking","category":"agentic","score":44.3,"unit":"percent","family":"aaTau3Banking","index":69.32650872517132,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/tau3-banking","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:medcode:overall","benchmarkName":"MedCode","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":43.974,"unit":"percent","family":"vals-ai:medcode:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medcode","measuredAt":"2026-10-29","stderr":2.258,"costPerTest":0.280517,"latency":96.589},{"benchmarkId":"vals-ai:web_search:overall","benchmarkName":"Vals Web Search Index","category":null,"domain":"Other","eligible":false,"ranked":true,"score":43.58411538461539,"unit":"percent","family":"vals-ai:web_search:overall","index":null,"aboveFit":false,"run":"Native","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/web_search","measuredAt":"2026-11-04","stderr":1.9821316327108887,"costPerTest":0.953765,"latency":328.83156009615385},{"benchmarkId":"epoch-ai:epoch-oeis-open-lite:overall","benchmarkName":"OEIS Open Lite","category":"math","domain":null,"eligible":true,"ranked":true,"score":43,"unit":"percent","family":"epoch-ai:epoch-oeis-open-lite:overall","index":null,"aboveFit":false,"run":"medium effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":4.950757517794625,"costPerTest":null,"latency":null},{"benchmarkId":"aaEnterpriseOpsGym","benchmarkName":"Artificial Analysis EnterpriseOps-Gym","category":"agentic","score":42.9,"unit":"percent","family":"aaEnterpriseOpsGym","index":null,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/enterprise-ops-gym-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"bug-hunt-bench","benchmarkName":"Bug Hunt Bench","category":"coding","score":42,"unit":"percent","family":"bug-hunt-bench","index":77.78250706844686,"aboveFit":false,"run":null,"source":"Pawel Huryn","sourceUrl":"https://bughunt.productcompass.pm/method","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"cursorBench40","benchmarkName":"cursorBench40","category":"coding","score":41.7,"unit":"percent","family":"cursorBench40","index":68.33971228730672,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"terminal-bench:terminal-bench-4-1-0:overall","benchmarkName":"Terminal-Bench 4.1.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":38.48,"unit":"percent","family":"terminal-bench:terminal-bench-4-1-0:overall","index":70.91530885954892,"aboveFit":false,"run":"max effort · Codex","source":"Terminal-Bench","sourceUrl":"https://www.tbench.ai/leaderboard","measuredAt":"2026-09-29","stderr":1.9540816326530612,"costPerTest":7.702121212121211,"latency":2388.4},{"benchmarkId":"vals-ai:terminal-bench-4:overall","benchmarkName":"Terminal-Bench 4.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":37.879,"unit":"percent","family":"terminal-bench-4","index":72.50242598537056,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-4","measuredAt":"2026-11-09","stderr":0,"costPerTest":7.983597,"latency":2026.072},{"benchmarkId":"terminal-bench:terminal-bench-4-0-0:overall","benchmarkName":"Terminal-Bench 4.0.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":37.27,"unit":"percent","family":"terminal-bench-4","index":72.21324925883648,"aboveFit":false,"run":"max effort · Codex","source":"Terminal-Bench","sourceUrl":"https://www.tbench.ai/leaderboard","measuredAt":"2026-09-29","stderr":1.9285714285714286,"costPerTest":7.702121212121211,"latency":2388.4},{"benchmarkId":"postTrainBench11","benchmarkName":"PostTrainBench v1.1","category":"coding","score":36.23,"unit":"percent","family":"postTrainBench11","index":71.02843870437351,"aboveFit":false,"run":null,"source":"PostTrainBench team","sourceUrl":"https://posttrainbench.com/?version=v1.1","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"frontierBench","benchmarkName":"Terminal-Bench 3.0","category":"agentic","score":34.59,"unit":"percent","family":"frontierBench","index":77.10439824294379,"aboveFit":false,"run":null,"source":"Ryan Marten, Alex Shaw, Andy Konwinski, Harbor, and the Laude Institute","sourceUrl":"https://www.frontierbench.ai/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"exploitGym","benchmarkName":"ExploitGym","category":"agentic","score":33.7,"unit":"percent","family":"exploitGym","index":79.43549240592606,"aboveFit":false,"run":null,"source":"Zhun Wang, Nico Schiller, Hongwei Li, Srijiith Sesha Narayana, Milad Nasr, Nicholas Carlini, Xiangyu Qi, Eric Wallace, Elie Bursztein, Luca Invernizzi, Kurt Thomas, Yan Shoshitaishvili, Wenbo Guo, Jingxuan He, Thorsten Holz, Dawn Song","sourceUrl":"https://arxiv.org/abs/2605.11086","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:mysterymechanism:overall","benchmarkName":"MysteryMechanism","category":null,"domain":"Science","eligible":false,"ranked":true,"score":33.333,"unit":"percent","family":"vals-ai:mysterymechanism:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mysterymechanism","measuredAt":"2026-11-11","stderr":3.171,"costPerTest":0.96335,"latency":522.744},{"benchmarkId":"healthBenchHard","benchmarkName":"HealthBench Hard","category":"knowledge","score":33.1,"unit":"percent","family":"healthBenchHard","index":72.85436682456253,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":32.3,"unit":"percent","family":"critpt","index":95,"aboveFit":true,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"frontierSweV2","benchmarkName":"FrontierSWE v2","category":"coding","score":32.2,"unit":"percent","family":"frontierSweV2","index":68.74124851648727,"aboveFit":false,"run":null,"source":"Proximal","sourceUrl":"https://www.frontierswe.com/blog/v2","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:tax_agent_bench:overall","benchmarkName":"Tax Agent Bench","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":31.088,"unit":"percent","family":"vals-ai:tax_agent_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench","measuredAt":"2026-11-09","stderr":3.34,"costPerTest":7.513965,"latency":2681.175},{"benchmarkId":"vals-ai:srebench:overall","benchmarkName":"SRE Bench","category":null,"domain":"Other","eligible":false,"ranked":true,"score":30.534,"unit":"percent","family":"vals-ai:srebench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/srebench","measuredAt":"2026-11-05","stderr":2.851,"costPerTest":42.521152,"latency":5285.426},{"benchmarkId":"geneBenchPro","benchmarkName":"GeneBench-Pro","category":"reasoning","score":28.7,"unit":"percent","family":"geneBenchPro","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"frontierCode2","benchmarkName":"FrontierCode 2.0","category":"coding","score":27.6,"unit":"percent","family":"frontierCode2","index":72.48914714159156,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"apprenticeBench","benchmarkName":"ApprenticeBench: end-to-end computer use, continual learning, and long-horizon agency on a real accounts-payable job","category":"agentic","score":26,"unit":"percent","family":"apprenticeBench","index":71.38924539610532,"aboveFit":false,"run":null,"source":"NeoCognition","sourceUrl":"https://neocognition.io/blog/apprentice-bench/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:time_horizon_index:overall","benchmarkName":"Time Horizon Index: KSP","category":null,"domain":"Other","eligible":false,"ranked":true,"score":23.833,"unit":"percent","family":"vals-ai:time_horizon_index:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/time_horizon_index","measuredAt":"2026-10-07","stderr":0,"costPerTest":2027.203388,"latency":0},{"benchmarkId":"vals-ai:vcb-1-100:overall","benchmarkName":"Vibe Code Bench 1-100","category":"coding","domain":null,"eligible":true,"ranked":true,"score":20.011,"unit":"percent","family":"vals-ai:vcb-1-100:overall","index":68.30034174098907,"aboveFit":false,"run":"OpenHands","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vcb-1-100","measuredAt":"2026-10-29","stderr":3.629,"costPerTest":47.425154,"latency":9421.289},{"benchmarkId":"epoch-ai:epoch-mirrorcode:overall","benchmarkName":"MirrorCode","category":"coding","domain":null,"eligible":true,"ranked":true,"score":20,"unit":"percent","family":"epoch-ai:epoch-mirrorcode:overall","index":58.349068736517495,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":9.229582069908972,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:terminal-bench-science:overall","benchmarkName":"Terminal-Bench-Science","category":null,"domain":"Science","eligible":false,"ranked":true,"score":20,"unit":"percent","family":"vals-ai:terminal-bench-science:overall","index":null,"aboveFit":false,"run":"Mini-SWE-agent","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-science","measuredAt":"2026-11-09","stderr":4.815,"costPerTest":7.322071,"latency":6216.027},{"benchmarkId":"mazeBench","benchmarkName":"MazeBench: visual spatial reasoning in a 3D open world","category":"agentic","score":13,"unit":"percent","family":"mazeBench","index":85.63845875136961,"aboveFit":false,"run":null,"source":"Jonathan Pappas and David Pappas","sourceUrl":"https://mazebench.com/blog?post=maze-bench-results","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:cua_bench:overall","benchmarkName":"CUA-bench","category":null,"domain":"Other","eligible":false,"ranked":true,"score":8.333333,"unit":"percent","family":"vals-ai:cua_bench:overall","index":null,"aboveFit":false,"run":"codex","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/cua_bench","measuredAt":"2026-10-13","stderr":0,"costPerTest":206.991579,"latency":18.8},{"benchmarkId":"arc-prize:v3-semi-private:overall","benchmarkName":"ARC-AGI-3 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":7.779999999999999,"unit":"percent","family":"arc-prize:v3-semi-private:overall","index":68.61101952289864,"aboveFit":false,"run":"max effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lmarena:agent-steerability:overall","benchmarkName":"Agent Arena steerability","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":5.53851302229843,"unit":"index","family":"agent-arena","index":71.08432729500632,"aboveFit":false,"run":"xhigh effort","source":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset","measuredAt":"2026-11-11","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lmarena:agent-task-outcome:overall","benchmarkName":"Agent Arena task outcome","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":2.295112294522883,"unit":"index","family":"agent-arena","index":68.41774342453645,"aboveFit":false,"run":"xhigh effort","source":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset","measuredAt":"2026-11-11","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:programbench:overall","benchmarkName":"ProgramBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":1.5,"unit":"percent","family":"vals-ai:programbench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/programbench","measuredAt":"2026-11-09","stderr":0.862,"costPerTest":15.289359,"latency":1764.488},{"benchmarkId":"lmarena:agent-bash-recovery:overall","benchmarkName":"Agent Arena command recovery","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":0.7262773119837584,"unit":"index","family":"agent-arena","index":67.12791512054613,"aboveFit":false,"run":"xhigh effort","source":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset","measuredAt":"2026-10-08","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-erdos:overall","benchmarkName":"FrontierMath-Erdos","category":"math","domain":null,"eligible":true,"ranked":true,"score":0,"unit":"percent","family":"epoch-ai:epoch-frontiermath-erdos:overall","index":null,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":4.4,"outputPerMillion":22,"batch":{"inputPerMillion":2,"outputPerMillion":10},"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":64,"timeToFirstToken":2.7735,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":{"elo":1457.120492040675,"eloUpper":1461.3635286646531,"eloLower":1452.8774554166966,"votes":43994,"rank":45,"rankUpper":null,"rankLower":null,"arenaName":"GPT-5.6 Sol-xhigh","organization":"openai","updatedAt":"2026-11-09"},"quality":{"coveragePercent":100,"benchmarkCount":70,"categoryCount":8,"confidence":"high","state":"available"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:arc-prize-foundation","name":"ARC Prize Foundation","url":"https://arcprize.org","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:livebench","name":"LiveBench","url":"https://livebench.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:sierra-research","name":"Sierra Research","url":"https://github.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:terminal-bench","name":"Terminal-Bench","url":"https://www.tbench.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:lmarena","name":"LMArena","url":"https://huggingface.co","license":"CC BY 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gpt-6-sol","providerId":"openai","slug":"gpt-6-sol","name":"GPT-6 Sol","provider":"OpenAI","description":"GPT-6 Sol is a reasoning model from OpenAI in the GPT-6 family. 56 benchmarks count toward its index, in 8 categories.","logoUrl":null,"releaseDate":"2026-09-22","contextWindowTokens":1050000,"modalities":["file","image","text"],"openWeights":false,"details":{"apiId":"openai/gpt-6-sol","maxOutputTokens":128000,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.19999999999999998,"weightsUrl":null},"reasoning":true,"overallScore":72.77564065742183,"overallScoreRange":{"low":70.61483351655988,"high":74.93644779828378},"badges":[],"capabilityScores":{"agentic":68.62658484111422,"coding":73.93691773976028,"reasoning":76.11060912086711,"multimodal":70.13675885269778,"knowledge":71.92857553738432,"multilingual":65.47046581525957,"instruction":65.3434344503222,"math":75.79945652494388},"capabilityCounts":{"math":5,"reasoning":11,"multimodal":3,"knowledge":7,"coding":8,"agentic":20,"multilingual":1,"instruction":1},"benchmarks":[{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":100,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":67.09849544987334,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0,"costPerTest":null,"latency":null},{"benchmarkId":"arc-prize:v1-public-eval:overall","benchmarkName":"ARC-AGI-1 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":97.75,"unit":"percent","family":"arc-prize:v1-public-eval:overall","index":null,"aboveFit":false,"run":"max effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.11090294000000003,"latency":null},{"benchmarkId":"livebench:livebench-math:overall","benchmarkName":"LiveBench Mathematics","category":"math","domain":null,"eligible":true,"ranked":true,"score":96.356,"unit":"percent","family":"livebench:livebench-math:overall","index":74.06804049657799,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Mathematics","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.0853868073878628,"latency":null},{"benchmarkId":"arc-prize:v1-semi-private:overall","benchmarkName":"ARC-AGI-1 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":95.5,"unit":"percent","family":"arc-prize:v1-semi-private:overall","index":70.89215669424564,"aboveFit":false,"run":"max effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.14080454,"latency":null},{"benchmarkId":"extractBenchLlamaIndex","benchmarkName":"ExtractBench: schema-guided enterprise document extraction (LlamaIndex leaderboard)","category":"multimodal","score":94.57,"unit":"percent","family":"extractBenchLlamaIndex","index":77.58824265861591,"aboveFit":false,"run":null,"source":"Boyang Zhang, Adrian Lyjak, Eli Stewart, Zhaoqi Li and Simon Suo (LlamaIndex)","sourceUrl":"https://arxiv.org/abs/2607.29677","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":94.31818181818183,"unit":"percent","family":"gpqa-diamond","index":63.71515984960199,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.4938512008159717,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-coding-2:overall","benchmarkName":"LiveBench Coding 2.0","category":"coding","domain":null,"eligible":true,"ranked":true,"score":93.1,"unit":"percent","family":"livebench:livebench-coding-2:overall","index":75.60658345479347,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Coding","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.9768793103448276,"latency":null},{"benchmarkId":"livebench:livebench-browser:overall","benchmarkName":"LiveBench Browser Use","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":92,"unit":"percent","family":"livebench-2-agentic","index":85.87193231505225,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Browser","measuredAt":"2026-10-14","stderr":null,"costPerTest":3.2837639999999997,"latency":null},{"benchmarkId":"arc-prize:v2-public-eval:overall","benchmarkName":"ARC-AGI-2 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":90,"unit":"percent","family":"arc-prize:v2-public-eval:overall","index":null,"aboveFit":false,"run":"max effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.413703533333333,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tier-4-v2-private:overall","benchmarkName":"FrontierMath-Tier-4-v2-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":90,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tier-4-v2-private:overall","index":83.84480798857413,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":5.173952057462543,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tiers-1-3-v2-private:overall","benchmarkName":"FrontierMath-Tiers-1-3-v2-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":89.82456140350877,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tiers-1-3-v2-private:overall","index":78.17400791671652,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.7939690721286368,"costPerTest":null,"latency":null},{"benchmarkId":"arc-prize:v2-semi-private:overall","benchmarkName":"ARC-AGI-2 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":89.58333333333334,"unit":"percent","family":"arc-prize:v2-semi-private:overall","index":80.31933236468987,"aboveFit":false,"run":"max effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.43889828333333336,"latency":null},{"benchmarkId":"livebench:livebench-reasoning:overall","benchmarkName":"LiveBench Reasoning","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":88.65375,"unit":"percent","family":"livebench","index":76.46480297129719,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Reasoning","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.12091435643564355,"latency":null},{"benchmarkId":"vals-ai:vibe-code:overall","benchmarkName":"Vibe Code Bench v1.1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":87.824,"unit":"percent","family":"vals-ai:vibe-code:overall","index":73.93780160948712,"aboveFit":false,"run":"OpenHands","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","measuredAt":"2026-11-09","stderr":2.526,"costPerTest":26.356012,"latency":2336.393},{"benchmarkId":"livebench:livebench-language:overall","benchmarkName":"LiveBench Language","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":85.29866666666668,"unit":"percent","family":"livebench:livebench-language:overall","index":71.01529445133912,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Language","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.086128,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":83.7,"unit":"percent","family":"lcr","index":64.72298945069046,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:terminal-bench-2-1:overall","benchmarkName":"Terminal-Bench 2.1","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":83.146,"unit":"percent","family":"terminal-bench-2-1","index":69.76950534182521,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2-1","measuredAt":"2026-10-01","stderr":1.297,"costPerTest":0.373024,"latency":298.876},{"benchmarkId":"vals-ai:proof_bench:overall","benchmarkName":"ProofBench v1.1","category":"math","domain":null,"eligible":true,"ranked":true,"score":83,"unit":"percent","family":"vals-ai:proof_bench:overall","index":75.8119307729774,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/proof_bench","measuredAt":"2026-11-09","stderr":3.775,"costPerTest":0.476321,"latency":304.509},{"benchmarkId":"aaMmmuPro","benchmarkName":"Artificial Analysis MMMU-Pro","category":"multimodal","score":82.9,"unit":"percent","family":"mmmu-pro","index":63.03989180621448,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmmu-pro","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:ioi:overall","benchmarkName":"IOI","category":"coding","domain":null,"eligible":true,"ranked":true,"score":82.611,"unit":"percent","family":"vals-ai:ioi:overall","index":77.13065412567688,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/ioi","measuredAt":"2026-10-29","stderr":9.24,"costPerTest":2.703348,"latency":1837.553},{"benchmarkId":"livebench:livebench-design:overall","benchmarkName":"LiveBench Design","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":82.2,"unit":"percent","family":"livebench-2-agentic","index":78.70488758483177,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Design","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.42538800000000004,"latency":null},{"benchmarkId":"vals-ai:medscribe:overall","benchmarkName":"MedScribe","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":82.034,"unit":"percent","family":"vals-ai:medscribe:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medscribe","measuredAt":"2026-10-29","stderr":1.942,"costPerTest":0.083138,"latency":81.135},{"benchmarkId":"livebench:livebench-coding:overall","benchmarkName":"LiveBench Coding","category":"coding","domain":null,"eligible":true,"ranked":true,"score":81.767,"unit":"percent","family":"livebench:livebench-coding:overall","index":69.82197207856775,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Coding","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.0894923076923077,"latency":null},{"benchmarkId":"livebench:livebench-data-analysis:overall","benchmarkName":"LiveBench Data Analysis","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":81.18933333333334,"unit":"percent","family":"livebench","index":66.79514385989424,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Data%20Analysis","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.19892866666666667,"latency":null},{"benchmarkId":"vals-ai:cyber:overall","benchmarkName":"CyberBench v1.1","category":null,"domain":"Other","eligible":false,"ranked":true,"score":77.976,"unit":"percent","family":"vals-ai:cyber:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/cyber","measuredAt":"2026-11-05","stderr":5.11,"costPerTest":1.338572,"latency":734.454},{"benchmarkId":"vals-ai:biomysterybench:overall","benchmarkName":"BioMysteryBench","category":null,"domain":"Science","eligible":false,"ranked":true,"score":74.815,"unit":"percent","family":"vals-ai:biomysterybench:overall","index":null,"aboveFit":false,"run":"Terminus 2","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/biomysterybench","measuredAt":"2026-11-09","stderr":2.429,"costPerTest":0.662358,"latency":324.465},{"benchmarkId":"hermesBench","benchmarkName":"Hermes Bench: 150 agent tasks graded on the files and state the agent leaves behind","category":"agentic","score":73.5,"unit":"percent","family":"hermesBench","index":null,"aboveFit":false,"run":null,"source":"Nous Research","sourceUrl":"https://portal.nousresearch.com/bench","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:emb:overall","benchmarkName":"EMB","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":71.53,"unit":"percent","family":"vals-ai:emb:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/emb","measuredAt":"2026-10-20","stderr":2.351,"costPerTest":1.276882,"latency":621.736},{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":71.32125,"unit":"percent","family":"euroeval","index":68.98676115641534,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":0.5334483925367102,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-french:overall","benchmarkName":"EuroEval French","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":70.3775,"unit":"percent","family":"euroeval","index":67.70630955710378,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/french","measuredAt":null,"stderr":0.3993373417300215,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-portuguese:overall","benchmarkName":"EuroEval Portuguese","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":70.075,"unit":"percent","family":"euroeval","index":67.29588666037081,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/portuguese","measuredAt":null,"stderr":0.4016275482334348,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-italian:overall","benchmarkName":"EuroEval Italian","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":69.92250000000001,"unit":"percent","family":"euroeval","index":67.08897924962115,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/italian","measuredAt":null,"stderr":0.4054877464239826,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-research:overall","benchmarkName":"LiveBench Research","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":69.3,"unit":"percent","family":"livebench-2-agentic","index":69.27071646035787,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Research","measuredAt":"2026-10-14","stderr":null,"costPerTest":3.5050772727272728,"latency":null},{"benchmarkId":"parseBenchLlamaIndex","benchmarkName":"ParseBench: document parsing for AI agents (LlamaIndex leaderboard)","category":"multimodal","score":68.85,"unit":"percent","family":"parseBenchLlamaIndex","index":69.78214209326296,"aboveFit":false,"run":null,"source":"Boyang Zhang, Sebastián G. Acosta, Preston Carlson, Sacha Bron, Pierre-Loïc Doulcet, Daniel B. Ospina and Simon Suo (LlamaIndex)","sourceUrl":"https://arxiv.org/abs/2604.08538","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"deepSwe","benchmarkName":"DeepSWE","category":"agentic","score":68.8,"unit":"percent","family":"deepSwe","index":71.09240001165024,"aboveFit":false,"run":null,"source":"Datacurve AI","sourceUrl":"https://deepswe.datacurve.ai/blog","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-instruction-following:overall","benchmarkName":"LiveBench Instruction Following","category":"instruction","domain":null,"eligible":true,"ranked":true,"score":68.57475,"unit":"percent","family":"livebench:livebench-instruction-following:overall","index":65.3434344503222,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=IF","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.4188425,"latency":null},{"benchmarkId":"euroeval:euroeval-dutch:overall","benchmarkName":"EuroEval Dutch","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":67.53666666666668,"unit":"percent","family":"euroeval","index":63.851952380897984,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/dutch","measuredAt":null,"stderr":0.432780273848172,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-long-horizon:overall","benchmarkName":"LiveBench Long Agentic","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":66.9,"unit":"percent","family":"livebench-2-agentic","index":67.51552183254879,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Long%20Agentic","measuredAt":"2026-10-14","stderr":null,"costPerTest":8.62146111111111,"latency":null},{"benchmarkId":"euroeval:euroeval-spanish:overall","benchmarkName":"EuroEval Spanish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":65.55000000000001,"unit":"percent","family":"euroeval","index":61.15650283320815,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/spanish","measuredAt":null,"stderr":0.43168094410108027,"costPerTest":null,"latency":null},{"benchmarkId":"aaCweBench","benchmarkName":"Artificial Analysis CWE-Bench-AA","category":"agentic","score":64.2,"unit":"percent","family":"aaCweBench","index":75.44320176967074,"aboveFit":false,"run":null,"source":"Artificial Analysis; dataset by Collinear AI","sourceUrl":"https://artificialanalysis.ai/evaluations/cwe-bench-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":63.084999999999994,"unit":"percent","family":"euroeval","index":57.81206501354934,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.4806587276852466,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":62.41714285714285,"unit":"percent","family":"euroeval","index":56.90593654024278,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.3626771206976683,"costPerTest":null,"latency":null},{"benchmarkId":"healthBenchProfessional","benchmarkName":"HealthBench Professional","category":"knowledge","score":60.8,"unit":"percent","family":"healthBenchProfessional","index":73.18265203378209,"aboveFit":false,"run":null,"source":"Rebecca Soskin Hicks, Mikhail Trofimov, Dominick Lim, Rahul K. Arora, Foivos Tsimpourlas, Preston Bowman, Michael Sharman, Chi Tong, Kavin Karthik, Arnav Dugar, Akshay Jagadeesh, Khaled Saab, Johannes Heidecke, Ashley Alexander, Nate Gross, Karan Singhal","sourceUrl":"https://arxiv.org/abs/2604.27470","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-simpleqa-verified:overall","benchmarkName":"SimpleQA Verified","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":60.699999999999996,"unit":"percent","family":"epoch-ai:epoch-simpleqa-verified:overall","index":74.48751080134451,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.5452824654081525,"costPerTest":null,"latency":null},{"benchmarkId":"osWorld2","benchmarkName":"OSWorld 2.0","category":"agentic","score":60.5,"unit":"percent","family":"osWorld2","index":78.5332060603912,"aboveFit":false,"run":null,"source":"Mengqi Yuan, Zilong Zhou, Xinzhuang Xiong, Weiming Wu, Jiayang Sun, Jiamin Song, Kaiqian Cui, Bowen Wang, Haoyuan Wu, Yitong Li, Dunjie Lu, Haikong Lu, Qi Zhen, Xinyuan Wang, Jiaqi Deng, Yuhao Yang, Cheng Chen, Boyuan Zheng, Alex Su, Xiao Yu, Hao Zou, Saaket Agashe, Xing Han Lu, Manpreet Kaur, Zhengyang Qi, Vincent Sunn Chen, Frederic Sala, Dayiheng Liu, Junyang Lin, Zhou Yu, Yu Su, Siva Reddy, Xin Eric Wang, Peng Qi, Tianbao Xie, Tao Yu","sourceUrl":"https://arxiv.org/abs/2606.29537","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"healthBenchProfessionalRaw","benchmarkName":"HealthBench Professional raw score","category":"knowledge","score":59.5,"unit":"percent","family":"healthBenchProfessionalRaw","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-furniture-assembly:overall","benchmarkName":"Furniture Assembly","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":58.333333333333336,"unit":"percent","family":"epoch-ai:epoch-furniture-assembly:overall","index":73.85714753523577,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":6.418399868645064,"costPerTest":null,"latency":null},{"benchmarkId":"aaSciCode","benchmarkName":"Artificial Analysis SciCode","category":"coding","score":57.6,"unit":"percent","family":"scicode-family","index":70.93240847599989,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/scicode","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:code-migration:overall","benchmarkName":"Code Migration","category":"coding","domain":null,"eligible":true,"ranked":true,"score":57.195,"unit":"percent","family":"vals-ai:code-migration:overall","index":75.5734146038869,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/code-migration","measuredAt":"2026-10-29","stderr":4.213,"costPerTest":15.676945,"latency":5136.155},{"benchmarkId":"vals-ai:public-benefits-bench:overall","benchmarkName":"Public Benefits Bench v1.1","category":null,"domain":"Public benefits","eligible":false,"ranked":true,"score":56.631,"unit":"percent","family":"vals-ai:public-benefits-bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/public-benefits-bench","measuredAt":"2026-10-12","stderr":1.289,"costPerTest":2.491613,"latency":1862.494},{"benchmarkId":"agentsLastExam","benchmarkName":"Agents' Last Exam","category":"agentic","score":56.4,"unit":"percent","family":"agentsLastExam","index":87.87966748270352,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-mystery-game-puzzles:overall","benchmarkName":"Mystery Game Puzzles","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":56.00000000000001,"unit":"percent","family":"epoch-ai:epoch-mystery-game-puzzles:overall","index":82.04823241755982,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":4.988876515698588,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-chess-puzzles:overall","benchmarkName":"Chess Puzzles","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":56.00000000000001,"unit":"percent","family":"epoch-ai:epoch-chess-puzzles:overall","index":76.48952665354433,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":4.988876515698588,"costPerTest":null,"latency":null},{"benchmarkId":"omniscienceAccuracy","benchmarkName":"Artificial Analysis Omniscience Accuracy","category":"knowledge","score":54.5,"unit":"percent","family":"omniscienceAccuracy","index":79.37390159786932,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-ebr-bench:overall","benchmarkName":"EBR-bench","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":53.333333333333336,"unit":"percent","family":"epoch-ai:epoch-ebr-bench:overall","index":76.90290581124464,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":4.616837959444122,"costPerTest":null,"latency":null},{"benchmarkId":"healthBenchLengthAdjusted","benchmarkName":"HealthBench length-adjusted score","category":"knowledge","score":53.2,"unit":"percent","family":"healthBenchLengthAdjusted","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-agentic-coding:overall","benchmarkName":"LiveBench Agentic Coding","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":52.87866666666667,"unit":"percent","family":"livebench:livebench-agentic-coding:overall","index":63.695558683401245,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Agentic%20Coding","measuredAt":"2026-06-25","stderr":null,"costPerTest":1.0611208333333333,"latency":null},{"benchmarkId":"cweBenchV1","benchmarkName":"CWE-bench v1","category":"agentic","score":52,"unit":"percent","family":"cweBenchV1","index":66.03358074751323,"aboveFit":false,"run":null,"source":"Collinear AI","sourceUrl":"https://cwe-bench.com/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gdpvalAaNormalized","benchmarkName":"GDPval-AA normalized","category":"agentic","score":50.4,"unit":"percent","family":"gdpvalAaNormalized","index":70.06860370646112,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"terminal-bench:terminal-bench-4-1-0:overall","benchmarkName":"Terminal-Bench 4.1.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":50.3,"unit":"percent","family":"terminal-bench:terminal-bench-4-1-0:overall","index":76.42236879264222,"aboveFit":false,"run":"max effort · Codex","source":"Terminal-Bench","sourceUrl":"https://www.tbench.ai/leaderboard","measuredAt":"2026-09-29","stderr":1.6479591836734695,"costPerTest":4.397939393939394,"latency":2662.2},{"benchmarkId":"aaItbench","benchmarkName":"Artificial Analysis ITBench-AA","category":"agentic","score":49.4,"unit":"percent","family":"aaItbench","index":71.0004004604389,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/itbench-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"terminal-bench:terminal-bench-4-0-0:overall","benchmarkName":"Terminal-Bench 4.0.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":49.39,"unit":"percent","family":"terminal-bench-4","index":77.96829347163302,"aboveFit":false,"run":"max effort · Codex","source":"Terminal-Bench","sourceUrl":"https://www.tbench.ai/leaderboard","measuredAt":"2026-09-29","stderr":1.6479591836734695,"costPerTest":4.397939393939394,"latency":2662.2},{"benchmarkId":"vals-ai:fabv2:overall","benchmarkName":"Finance Agent (v2)","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":49.05,"unit":"percent","family":"vals-ai:fabv2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/fabv2","measuredAt":"2026-10-29","stderr":0.58,"costPerTest":2.122121,"latency":539.907},{"benchmarkId":"epoch-ai:epoch-mirrorcode:overall","benchmarkName":"MirrorCode","category":"coding","domain":null,"eligible":true,"ranked":true,"score":48.888888888888886,"unit":"percent","family":"epoch-ai:epoch-mirrorcode:overall","index":71.03623569106261,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":11.205944504585897,"costPerTest":null,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":47.9,"unit":"percent","family":"hle","index":72.44436680024063,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"healthBench","benchmarkName":"HealthBench raw score","category":"knowledge","score":47.1,"unit":"percent","family":"healthBench","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:medcode:overall","benchmarkName":"MedCode","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":47.072,"unit":"percent","family":"vals-ai:medcode:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medcode","measuredAt":"2026-10-29","stderr":2.119,"costPerTest":0.085827,"latency":62.714},{"benchmarkId":"aaDeepsecBench","benchmarkName":"Artificial Analysis DeepsecBench-AA","category":"agentic","score":46.3,"unit":"percent","family":"aaDeepsecBench","index":85.90024521489744,"aboveFit":false,"run":null,"source":"Artificial Analysis; dataset by Vercel","sourceUrl":"https://artificialanalysis.ai/evaluations/deepsecbench-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:terminal-bench-4:overall","benchmarkName":"Terminal-Bench 4.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":44.444,"unit":"percent","family":"terminal-bench-4","index":75.61974160063535,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-4","measuredAt":"2026-11-09","stderr":3.535,"costPerTest":5.79322,"latency":2096.128},{"benchmarkId":"hermesIndex","benchmarkName":"Hermes Index: agentic capability and cost per task inside Hermes Agent","category":"agentic","score":44.1,"unit":"percent","family":"hermesIndex","index":71.22686525146985,"aboveFit":false,"run":null,"source":"Nous Research","sourceUrl":"https://portal.nousresearch.com/bench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaCyberIndex","benchmarkName":"Artificial Analysis Cyber Index","category":"agentic","score":36.8,"unit":"percent","family":"aaCyberIndex","index":65.01857061102612,"aboveFit":false,"run":null,"source":"Artificial Analysis with the Cyber Index Alliance (Collinear AI, IBM, NVIDIA, Pasta Lab, Vercel)","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-cyber-index","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"frontierCode2","benchmarkName":"FrontierCode 2.0","category":"coding","score":34.3,"unit":"percent","family":"frontierCode2","index":77.4562718786075,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaMlcr","benchmarkName":"Medical Long Context Reasoning (MLCR-AA)","category":"reasoning","score":33.9,"unit":"percent","family":"aaMlcr","index":78.08446674358348,"aboveFit":false,"run":null,"source":"Wisedocs and Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mlcr-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"automationBench","benchmarkName":"AutomationBench","category":"agentic","score":33.2,"unit":"percent","family":"automationBench","index":45.240013618560646,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":30.9,"unit":"percent","family":"critpt","index":95,"aboveFit":true,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:srebench:overall","benchmarkName":"SRE Bench","category":null,"domain":"Other","eligible":false,"ranked":true,"score":30.534,"unit":"percent","family":"vals-ai:srebench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/srebench","measuredAt":"2026-11-05","stderr":2.851,"costPerTest":20.651048,"latency":3807.782},{"benchmarkId":"vals-ai:mysterymechanism:overall","benchmarkName":"MysteryMechanism","category":null,"domain":"Science","eligible":false,"ranked":true,"score":30.18,"unit":"percent","family":"vals-ai:mysterymechanism:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mysterymechanism","measuredAt":"2026-11-11","stderr":3.088,"costPerTest":0.460931,"latency":373.426},{"benchmarkId":"healthBenchHard","benchmarkName":"HealthBench Hard","category":"knowledge","score":30.1,"unit":"percent","family":"healthBenchHard","index":69.28114322751256,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:terminal-bench-science:overall","benchmarkName":"Terminal-Bench-Science","category":null,"domain":"Science","eligible":false,"ranked":true,"score":30,"unit":"percent","family":"vals-ai:terminal-bench-science:overall","index":null,"aboveFit":false,"run":"Mini-SWE-agent","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-science","measuredAt":"2026-11-09","stderr":5.517,"costPerTest":5.821477,"latency":4930.428},{"benchmarkId":"vals-ai:legal_research:overall","benchmarkName":"Legal Research Bench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":28.846,"unit":"percent","family":"vals-ai:legal_research:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_research","measuredAt":"2026-11-09","stderr":3.149,"costPerTest":4.897717,"latency":1449.451},{"benchmarkId":"exploitGym","benchmarkName":"ExploitGym","category":"agentic","score":22.1,"unit":"percent","family":"exploitGym","index":72.39641419723657,"aboveFit":false,"run":null,"source":"Zhun Wang, Nico Schiller, Hongwei Li, Srijiith Sesha Narayana, Milad Nasr, Nicholas Carlini, Xiangyu Qi, Eric Wallace, Elie Bursztein, Luca Invernizzi, Kurt Thomas, Yan Shoshitaishvili, Wenbo Guo, Jingxuan He, Thorsten Holz, Dawn Song","sourceUrl":"https://arxiv.org/abs/2605.11086","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:tax_agent_bench:overall","benchmarkName":"Tax Agent Bench","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":15.002,"unit":"percent","family":"vals-ai:tax_agent_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench","measuredAt":"2026-11-09","stderr":2.575,"costPerTest":2.277613,"latency":1052.036},{"benchmarkId":"epoch-ai:epoch-spireclimb-slay-the-spire-2-ascension-10:overall","benchmarkName":"SpireClimb (Slay the Spire 2, Ascension 10)","category":null,"domain":"Other","eligible":false,"ranked":true,"score":13.333333333333334,"unit":"percent","family":"epoch-ai:epoch-spireclimb-slay-the-spire-2-ascension-10:overall","index":null,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":3.9516598557587943,"costPerTest":null,"latency":null},{"benchmarkId":"lmarena:agent-steerability:overall","benchmarkName":"Agent Arena steerability","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":10.144882834437619,"unit":"index","family":"agent-arena","index":74.87148542729608,"aboveFit":false,"run":"max effort","source":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset","measuredAt":"2026-11-11","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-spireclimb:overall","benchmarkName":"SpireClimb","category":null,"domain":"Other","eligible":false,"ranked":true,"score":9.166666666666668,"unit":"percent","family":"epoch-ai:epoch-spireclimb:overall","index":null,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.432409457371575,"costPerTest":null,"latency":null},{"benchmarkId":"lmarena:agent-bash-recovery:overall","benchmarkName":"Agent Arena command recovery","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":7.030190807524606,"unit":"index","family":"agent-arena","index":72.31072028676914,"aboveFit":false,"run":"max effort","source":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset","measuredAt":"2026-10-08","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mazeBench","benchmarkName":"MazeBench: visual spatial reasoning in a 3D open world","category":"agentic","score":7,"unit":"percent","family":"mazeBench","index":74.71855502689802,"aboveFit":false,"run":null,"source":"Jonathan Pappas and David Pappas","sourceUrl":"https://mazebench.com/blog?post=maze-bench-results","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-spireclimb-slay-the-spire-1-ascension-20-heart:overall","benchmarkName":"SpireClimb (Slay the Spire 1, Ascension 20 Heart)","category":null,"domain":"Other","eligible":false,"ranked":true,"score":5,"unit":"percent","family":"epoch-ai:epoch-spireclimb-slay-the-spire-1-ascension-20-heart:overall","index":null,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.8374015326751194,"costPerTest":null,"latency":null},{"benchmarkId":"arc-prize:v3-semi-private:overall","benchmarkName":"ARC-AGI-3 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":4.623568186131146,"unit":"percent","family":"arc-prize:v3-semi-private:overall","index":67.26996924314844,"aboveFit":false,"run":"max effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lmarena:agent-task-outcome:overall","benchmarkName":"Agent Arena task outcome","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":4.533370193412278,"unit":"index","family":"agent-arena","index":70.25794231208609,"aboveFit":false,"run":"max effort","source":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset","measuredAt":"2026-11-11","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaHarveyLabV11","benchmarkName":"Harvey LAB-AA v1.1","category":"agentic","score":3.6,"unit":"percent","family":"aaHarveyLabV11","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:programbench:overall","benchmarkName":"ProgramBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":2,"unit":"percent","family":"vals-ai:programbench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/programbench","measuredAt":"2026-11-09","stderr":0.992,"costPerTest":11.672651,"latency":2771.488},{"benchmarkId":"epoch-ai:epoch-frontiermath-erdos:overall","benchmarkName":"FrontierMath-Erdos","category":"math","domain":null,"eligible":true,"ranked":true,"score":1.4705882352941175,"unit":"percent","family":"epoch-ai:epoch-frontiermath-erdos:overall","index":null,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.4705882352941175,"costPerTest":null,"latency":null},{"benchmarkId":"aaCyberGymE2e","benchmarkName":"Artificial Analysis CyberGym-E2E-AA","category":"agentic","score":0,"unit":"percent","family":"aaCyberGymE2e","index":5,"aboveFit":false,"run":null,"source":"Artificial Analysis; dataset by Berkeley RDI","sourceUrl":"https://artificialanalysis.ai/evaluations/cybergym-e2e-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":2,"outputPerMillion":10,"batch":{"inputPerMillion":1,"outputPerMillion":5},"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":61,"timeToFirstToken":2.2985,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":{"elo":1396.535649786022,"eloUpper":1402.0351904063896,"eloLower":1391.0361091656546,"votes":19890,"rank":163,"rankUpper":null,"rankLower":null,"arenaName":"GPT-6 Sol-max","organization":"openai","updatedAt":"2026-11-09"},"quality":{"coveragePercent":100,"benchmarkCount":56,"categoryCount":8,"confidence":"high","state":"available"},"hosts":[{"host":"amazon-bedrock","name":"Amazon Bedrock","kind":"cloud","url":"https://docs.aws.amazon.com/bedrock/latest/userguide/models-supported.html","inputPerMillion":2,"outputPerMillion":10,"cacheReadPerMillion":0.2,"contextTokens":1050000,"maxOutputTokens":128000,"deprecated":false},{"host":"digitalocean","name":"DigitalOcean","kind":"cloud","url":"https://docs.digitalocean.com/products/gradient-ai-platform/details/models/","inputPerMillion":2,"outputPerMillion":10,"cacheReadPerMillion":0.2,"contextTokens":1050000,"maxOutputTokens":128000,"deprecated":false},{"host":"azure","name":"Microsoft Azure","kind":"cloud","url":"https://learn.microsoft.com/en-us/azure/ai-services/openai/concepts/models","inputPerMillion":2,"outputPerMillion":10,"cacheReadPerMillion":0.2,"contextTokens":1050000,"maxOutputTokens":128000,"deprecated":false},{"host":"openai","name":"OpenAI","kind":"maker","url":"https://platform.openai.com/docs/models","inputPerMillion":2,"outputPerMillion":10,"cacheReadPerMillion":0.2,"contextTokens":1050000,"maxOutputTokens":128000,"deprecated":false},{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":2,"outputPerMillion":10,"cacheReadPerMillion":0.2,"contextTokens":1050000,"maxOutputTokens":128000,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:livebench","name":"LiveBench","url":"https://livebench.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:arc-prize-foundation","name":"ARC Prize Foundation","url":"https://arcprize.org","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:terminal-bench","name":"Terminal-Bench","url":"https://www.tbench.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:lmarena","name":"LMArena","url":"https://huggingface.co","license":"CC BY 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"hy4","providerId":"tencent","slug":"hy4","name":"Hy4","provider":"Tencent","description":"Hy4 is a reasoning model from Tencent. 14 benchmarks count toward its index, in 4 categories.","logoUrl":null,"releaseDate":"2026-11-04","contextWindowTokens":1048576,"modalities":["text","image"],"openWeights":null,"details":{"apiId":"tencent/hy4","maxOutputTokens":65536,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.020999999999999998,"weightsUrl":null},"reasoning":true,"overallScore":72.03866231653724,"overallScoreRange":{"low":64.21704803481333,"high":79.86027659826115},"badges":[],"capabilityScores":{"agentic":74.51218305455352,"coding":68.43046964951746,"reasoning":null,"multimodal":57.44381906620205,"knowledge":83.68124554457432,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"coding":3,"agentic":9,"multimodal":1,"knowledge":1},"benchmarks":[{"benchmarkId":"vals-ai:vibe-code:overall","benchmarkName":"Vibe Code Bench v1.1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":81.008,"unit":"percent","family":"vals-ai:vibe-code:overall","index":71.47848172209562,"aboveFit":false,"run":"OpenHands","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","measuredAt":"2026-11-09","stderr":3.548,"costPerTest":2.394585,"latency":2281.498},{"benchmarkId":"osWorld21","benchmarkName":"OSWorld 2.1 (all 108 tasks)","category":"agentic","score":78.9,"unit":"percent","family":"osWorld21","index":76.64864031446719,"aboveFit":false,"run":null,"source":"Mengqi Yuan, Zilong Zhou, Xinzhuang Xiong, Weiming Wu, Jiayang Sun, Jiamin Song, Kaiqian Cui, Bowen Wang, Haoyuan Wu, Yitong Li, Dunjie Lu, Haikong Lu, Qi Zhen, Xinyuan Wang, Jiaqi Deng, Yuhao Yang, Cheng Chen, Boyuan Zheng, Alex Su, Xiao Yu, Hao Zou, Saaket Agashe, Xing Han Lu, Manpreet Kaur, Zhengyang Qi, Vincent Sunn Chen, Frederic Sala, Dayiheng Liu, Junyang Lin, Zhou Yu, Yu Su, Siva Reddy, Xin Eric Wang, Peng Qi, Tianbao Xie, Tao Yu","sourceUrl":"https://github.com/xlang-ai/OSWorld-V2/releases","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmmuPro","benchmarkName":"Massive Multi-discipline Multimodal Understanding Pro","category":"multimodal","score":78.6,"unit":"percent","family":"mmmu-pro","index":57.44381906620205,"aboveFit":false,"run":null,"source":"MMMU-Pro authors","sourceUrl":"https://arxiv.org/abs/2409.02813","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"apexAgents","benchmarkName":"APEX-Agents","category":"agentic","score":75.2,"unit":"percent","family":"apexAgents","index":84.3024061950143,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"deepSwe","benchmarkName":"DeepSWE","category":"agentic","score":74.9,"unit":"percent","family":"deepSwe","index":75.52692972401337,"aboveFit":false,"run":null,"source":"Datacurve AI","sourceUrl":"https://deepswe.datacurve.ai/blog","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hleWithTools","benchmarkName":"Humanity's Last Exam with tools","category":"agentic","score":59.1,"unit":"percent","family":"hleWithTools","index":67.79547307382724,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hle","benchmarkName":"Humanity's Last Exam","category":"knowledge","score":59.1,"unit":"percent","family":"hle","index":83.68124554457432,"aboveFit":false,"run":null,"source":"Center for AI Safety, Scale AI, and thousands of expert contributors","sourceUrl":"https://lastexam.ai/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"automationBench","benchmarkName":"AutomationBench","category":"agentic","score":56.2,"unit":"percent","family":"automationBench","index":88.9879450303125,"aboveFit":true,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"zeroBenchPython","benchmarkName":"ZeroBench_main with Python","category":"multimodal","score":55.4,"unit":"percent","family":"zeroBenchPython","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"sweMarathon","benchmarkName":"SWE-Marathon","category":"coding","score":43.8,"unit":"percent","family":"sweMarathon","index":null,"aboveFit":false,"run":null,"source":"Abundant AI and BenchFlow","sourceUrl":"https://www.swe-marathon.org/","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:legal_research:overall","benchmarkName":"Legal Research Bench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":41.346,"unit":"percent","family":"vals-ai:legal_research:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_research","measuredAt":"2026-11-09","stderr":3.423,"costPerTest":2.222804,"latency":5455.029},{"benchmarkId":"agentsLastExam","benchmarkName":"Agents' Last Exam","category":"agentic","score":41,"unit":"percent","family":"agentsLastExam","index":75.09519946766898,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:terminal-bench-4:overall","benchmarkName":"Terminal-Bench 4.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":39.394,"unit":"percent","family":"terminal-bench-4","index":73.22180651197013,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-4","measuredAt":"2026-11-09","stderr":0.875,"costPerTest":1.280113,"latency":2600.367},{"benchmarkId":"postTrainBench12","benchmarkName":"PostTrainBench v1.2","category":"coding","score":35.2,"unit":"percent","family":"postTrainBench12","index":null,"aboveFit":false,"run":null,"source":"PostTrainBench authors","sourceUrl":"https://posttrainbench.com/","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:tax_agent_bench:overall","benchmarkName":"Tax Agent Bench","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":34.399,"unit":"percent","family":"vals-ai:tax_agent_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench","measuredAt":"2026-11-09","stderr":3.298,"costPerTest":1.687683,"latency":3807.674},{"benchmarkId":"frontierCode2","benchmarkName":"FrontierCode 2.0","category":"coding","score":31,"unit":"percent","family":"frontierCode2","index":75.00977760515188,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"programBench","benchmarkName":"ProgramBench: Can Language Models Rebuild Programs From Scratch?","category":"coding","score":30,"unit":"percent","family":"programBench","index":58.803149621304875,"aboveFit":false,"run":null,"source":"John Yang, Kilian Lieret, Jeffrey Ma, Parth Thakkar, Dmitrii Pedchenko, Sten Sootla, Emily McMilin, Pengcheng Yin, Rui Hou, Gabriel Synnaeve, Diyi Yang, Ofir Press","sourceUrl":"https://programbench.com/static/paper.pdf","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"terminalBenchScience","benchmarkName":"Terminal-Bench-Science 0.1","category":"agentic","score":19.8,"unit":"percent","family":"terminalBenchScience","index":56.15555107375799,"aboveFit":false,"run":null,"source":"Terminal-Bench-Science Team","sourceUrl":"https://www.terminal-bench-science.ai/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:hlab:overall","benchmarkName":"HLAB-Vals-1.1","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":12.5,"unit":"percent","family":"vals-ai:hlab:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/hlab","measuredAt":"2026-11-09","stderr":2.663,"costPerTest":1.036564,"latency":1167.168},{"benchmarkId":"lmarena:agent-steerability:overall","benchmarkName":"Agent Arena steerability","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":8.77651634217722,"unit":"index","family":"agent-arena","index":73.74647365002626,"aboveFit":false,"run":null,"source":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset","measuredAt":"2026-11-11","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lmarena:agent-task-outcome:overall","benchmarkName":"Agent Arena task outcome","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":6.658240167987028,"unit":"index","family":"agent-arena","index":72.00491854987365,"aboveFit":false,"run":null,"source":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset","measuredAt":"2026-11-11","stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":0.417,"outputPerMillion":1.2505,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":52,"timeToFirstToken":3.36,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":65,"benchmarkCount":14,"categoryCount":4,"confidence":"low","state":"available"},"hosts":[{"host":"novita-ai","name":"Novita AI","kind":"host","url":"https://novita.ai/docs/guides/introduction","inputPerMillion":0.417,"outputPerMillion":1.2505,"cacheReadPerMillion":0.021,"contextTokens":1000000,"maxOutputTokens":64000,"deprecated":false},{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":0.417,"outputPerMillion":1.2505,"cacheReadPerMillion":0.021,"contextTokens":1048576,"maxOutputTokens":65536,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:lmarena","name":"LMArena","url":"https://huggingface.co","license":"CC BY 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"claude-mythos-5","providerId":"anthropic","slug":"claude-mythos-5","name":"Claude Mythos 5","provider":"Anthropic","description":"Claude Mythos 5 is a reasoning model from Anthropic in the Claude Mythos family. 10 benchmarks count toward its index, in 4 categories.","logoUrl":null,"releaseDate":"2026-06-09","contextWindowTokens":1000000,"modalities":["text"],"openWeights":false,"details":null,"reasoning":true,"overallScore":70.8014261895963,"overallScoreRange":{"low":62.18802321310403,"high":79.41482916608857},"badges":[],"capabilityScores":{"agentic":70.04813368506794,"coding":74.44417068556139,"reasoning":null,"multimodal":70.83700145094049,"knowledge":74.92751038335228,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"coding":2,"knowledge":2,"multimodal":2,"agentic":4},"benchmarks":[{"benchmarkId":"usamo2026","benchmarkName":"United States of America Mathematical Olympiad 2026","category":"math","score":97.6,"unit":"percent","family":"usamo2026","index":null,"aboveFit":false,"run":null,"source":"Mathematical Association of America","sourceUrl":"https://www.maa.org/math-competitions/usamo","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"sweVerified","benchmarkName":"Software Engineering Benchmark Verified","category":"coding","score":95.5,"unit":"percent","family":"swe-bench-verified","index":71.66467620781422,"aboveFit":false,"run":null,"source":"Carlos E. Jimenez, John Yang, Alexander Wettig, Shunyu Yao, Kexin Pei, Ofir Press, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2310.06770","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gpqa","benchmarkName":"Graduate-Level Google-Proof Q&A","category":"knowledge","score":94.1,"unit":"percent","family":"gpqa-diamond","index":63.515049447801296,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"charxiv","benchmarkName":"CharXiv Reasoning","category":"multimodal","score":93.5,"unit":"percent","family":"charxiv","index":70.63380382188436,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"charxivNoTools","benchmarkName":"CharXiv Reasoning without tools","category":"multimodal","score":88.9,"unit":"percent","family":"charxivNoTools","index":71.04019907999663,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"terminalBench21","benchmarkName":"Terminal-Bench 2.1","category":"agentic","score":88,"unit":"percent","family":"terminal-bench-2-1","index":72.60839328097704,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"browseComp","benchmarkName":"BrowseComp","category":"agentic","score":88,"unit":"percent","family":"browseComp","index":69.43073697853778,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"osWorldVerified","benchmarkName":"OSWorld-Verified","category":"agentic","score":85,"unit":"percent","family":"osWorldVerified","index":69.09786078177461,"aboveFit":false,"run":null,"source":"Tianbao Xie, Danyang Zhang, Jixuan Chen, Xiaochuan Li, Siheng Zhao, Ruisheng Cao, Toh Jing Hua, Zhoujun Cheng, Dongchan Shin, Fangyu Lei, Yitao Liu, Yiheng Xu, Shuyan Zhou, Silvio Savarese, Caiming Xiong, Victor Zhong, Tao Yu","sourceUrl":"https://os-world.github.io/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"cyberGym","benchmarkName":"CyberGym","category":"agentic","score":83.8,"unit":"percent","family":"cyberGym","index":69.05554369898238,"aboveFit":false,"run":null,"source":"Zhun Wang, Tianneng Shi, Jingxuan He, Matthew Cai, Jialin Zhang, Dawn Song","sourceUrl":"https://www.cybergym.io/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"swePro","benchmarkName":"SWE-bench Pro","category":"coding","score":80.3,"unit":"percent","family":"swePro","index":77.22366516330854,"aboveFit":false,"run":null,"source":"Xiang Deng, Jeff Da, Edwin Pan, Yannis Yiming He, Charles Ide, Kanak Garg, Niklas Lauffer, Andrew Park, Nitin Pasari, Chetan Rane, Karmini Sampath, Maya Krishnan, Srivatsa Kundurthy, Sean Hendryx, Zifan Wang, Chen Bo Calvin Zhang, Noah Jacobson, Bing Liu, Brad Kenstler","sourceUrl":"https://arxiv.org/abs/2509.16941","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hle","benchmarkName":"Humanity's Last Exam","category":"knowledge","score":64.5,"unit":"percent","family":"hle","index":89.0990263677352,"aboveFit":true,"run":null,"source":"Center for AI Safety, Scale AI, and thousands of expert contributors","sourceUrl":"https://lastexam.ai/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hleNoTools","benchmarkName":"Humanity's Last Exam without tools","category":"knowledge","score":59,"unit":"percent","family":"hle","index":83.58091627007134,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":10,"outputPerMillion":50,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":65,"benchmarkCount":10,"categoryCount":4,"confidence":"low","state":"available"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"mimo-v2-6-pro","providerId":"xiaomi","slug":"mimo-v2-6-pro","name":"MiMo-V2.6-Pro","provider":"Xiaomi","description":"MiMo-V2.6-Pro is a reasoning model from Xiaomi in the MiMo-V2.6 family. 36 benchmarks count toward its index, in 5 categories.","logoUrl":null,"releaseDate":"2026-09-22","contextWindowTokens":1050000,"modalities":["text","image","video","audio"],"openWeights":true,"details":{"apiId":"xiaomi/mimo-v2.6-pro","maxOutputTokens":131072,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.0036,"weightsUrl":"https://huggingface.co/XiaomiMiMo/MiMo-V2.6-Pro-RL"},"reasoning":true,"overallScore":70.15290697170158,"overallScoreRange":{"low":65.45790697170159,"high":74.84790697170158},"badges":[],"capabilityScores":{"agentic":72.08669196839615,"coding":66.60505918791326,"reasoning":68.26206124665833,"multimodal":null,"knowledge":64.24808231484087,"multilingual":null,"instruction":null,"math":71.32374966008697},"capabilityCounts":{"agentic":22,"reasoning":4,"coding":7,"math":1,"knowledge":2},"benchmarks":[{"benchmarkId":"cyberGym","benchmarkName":"CyberGym","category":"agentic","score":94,"unit":"percent","family":"cyberGym","index":75.36136533974471,"aboveFit":false,"run":null,"source":"Zhun Wang, Tianneng Shi, Jingxuan He, Matthew Cai, Jialin Zhang, Dawn Song","sourceUrl":"https://www.cybergym.io/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"terminalBench21","benchmarkName":"Terminal-Bench 2.1","category":"agentic","score":89.9,"unit":"percent","family":"terminal-bench-2-1","index":73.71961847347569,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:medscribe:overall","benchmarkName":"MedScribe","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":88.307,"unit":"percent","family":"vals-ai:medscribe:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medscribe","measuredAt":"2026-10-29","stderr":1.938,"costPerTest":0.00988,"latency":173.345},{"benchmarkId":"livebench:livebench-browser:overall","benchmarkName":"LiveBench Browser Use","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":88,"unit":"percent","family":"livebench-2-agentic","index":82.94660793537042,"aboveFit":false,"run":"high effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Browser","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.456672,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":86.3,"unit":"percent","family":"lcr","index":66.50679881721318,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:vibe-code:overall","benchmarkName":"Vibe Code Bench v1.1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":85.223,"unit":"percent","family":"vals-ai:vibe-code:overall","index":72.99931994471228,"aboveFit":false,"run":"OpenHands","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","measuredAt":"2026-11-09","stderr":3.392,"costPerTest":1.043846,"latency":3646.574},{"benchmarkId":"vals-ai:sage:overall","benchmarkName":"SAGE","category":null,"domain":"Education","eligible":false,"ranked":true,"score":84.448,"unit":"percent","family":"vals-ai:sage:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/sage","measuredAt":"2026-11-05","stderr":1.871,"costPerTest":0.021366,"latency":141.384},{"benchmarkId":"osWorldVerified","benchmarkName":"OSWorld-Verified","category":"agentic","score":82,"unit":"percent","family":"osWorldVerified","index":66.67608960274501,"aboveFit":false,"run":null,"source":"Tianbao Xie, Danyang Zhang, Jixuan Chen, Xiaochuan Li, Siheng Zhao, Ruisheng Cao, Toh Jing Hua, Zhoujun Cheng, Dongchan Shin, Fangyu Lei, Yitao Liu, Yiheng Xu, Shuyan Zhou, Silvio Savarese, Caiming Xiong, Victor Zhong, Tao Yu","sourceUrl":"https://os-world.github.io/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-coding-2:overall","benchmarkName":"LiveBench Coding 2.0","category":"coding","domain":null,"eligible":true,"ranked":true,"score":81.5,"unit":"percent","family":"livebench:livebench-coding-2:overall","index":67.20441257303065,"aboveFit":false,"run":"high effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Coding","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.22390689655172413,"latency":null},{"benchmarkId":"livebench:livebench-reasoning:overall","benchmarkName":"LiveBench Reasoning","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":79.6,"unit":"percent","family":"livebench","index":64.7362674286853,"aboveFit":false,"run":"high effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Reasoning","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.023731544554455444,"latency":null},{"benchmarkId":"aaCyberGymE2e","benchmarkName":"Artificial Analysis CyberGym-E2E-AA","category":"agentic","score":78.6,"unit":"percent","family":"aaCyberGymE2e","index":95,"aboveFit":true,"run":null,"source":"Artificial Analysis; dataset by Berkeley RDI","sourceUrl":"https://artificialanalysis.ai/evaluations/cybergym-e2e-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-design:overall","benchmarkName":"LiveBench Design","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":78.5,"unit":"percent","family":"livebench-2-agentic","index":75.99896253362607,"aboveFit":false,"run":"high effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Design","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.093224,"latency":null},{"benchmarkId":"toolathlonVerified","benchmarkName":"Toolathlon-Verified","category":"agentic","score":76.9,"unit":"percent","family":"toolathlonVerified","index":70.91362631272116,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-research:overall","benchmarkName":"LiveBench Research","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":76.7,"unit":"percent","family":"livebench-2-agentic","index":74.68256656276925,"aboveFit":false,"run":"high effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Research","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.14229090909090908,"latency":null},{"benchmarkId":"vals-ai:cyber:overall","benchmarkName":"CyberBench v1.1","category":null,"domain":"Other","eligible":false,"ranked":true,"score":72.857,"unit":"percent","family":"vals-ai:cyber:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/cyber","measuredAt":"2026-11-05","stderr":5.5,"costPerTest":0.094131,"latency":1661.322},{"benchmarkId":"livebench:livebench-data-analysis:overall","benchmarkName":"LiveBench Data Analysis","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":72.3,"unit":"percent","family":"livebench","index":55.27959925681544,"aboveFit":false,"run":"high effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Data%20Analysis","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.03856399333333333,"latency":null},{"benchmarkId":"deepSwe","benchmarkName":"DeepSWE","category":"agentic","score":71.9,"unit":"percent","family":"deepSwe","index":73.34601347203152,"aboveFit":false,"run":null,"source":"Datacurve AI","sourceUrl":"https://deepswe.datacurve.ai/blog","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:proof_bench:overall","benchmarkName":"ProofBench v1.1","category":"math","domain":null,"eligible":true,"ranked":true,"score":70,"unit":"percent","family":"vals-ai:proof_bench:overall","index":71.32374966008697,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/proof_bench","measuredAt":"2026-11-09","stderr":4.606,"costPerTest":0.24535,"latency":3693.469},{"benchmarkId":"vals-ai:public-benefits-bench:overall","benchmarkName":"Public Benefits Bench v1.1","category":null,"domain":"Public benefits","eligible":false,"ranked":true,"score":68.945,"unit":"percent","family":"vals-ai:public-benefits-bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/public-benefits-bench","measuredAt":"2026-10-12","stderr":1.204,"costPerTest":0.081936,"latency":2234.344},{"benchmarkId":"vals-ai:terminal-bench-2-1:overall","benchmarkName":"Terminal-Bench 2.1","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":67.79,"unit":"percent","family":"terminal-bench-2-1","index":60.78846636497817,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2-1","measuredAt":"2026-10-01","stderr":1.873,"costPerTest":0.072656,"latency":939.123},{"benchmarkId":"aaCweBench","benchmarkName":"Artificial Analysis CWE-Bench-AA","category":"agentic","score":63.3,"unit":"percent","family":"aaCweBench","index":74.67974800325587,"aboveFit":false,"run":null,"source":"Artificial Analysis; dataset by Collinear AI","sourceUrl":"https://artificialanalysis.ai/evaluations/cwe-bench-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:emb:overall","benchmarkName":"EMB","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":62.858,"unit":"percent","family":"vals-ai:emb:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/emb","measuredAt":"2026-10-20","stderr":3.141,"costPerTest":0.328999,"latency":3313.924},{"benchmarkId":"jobBench","benchmarkName":"JobBench","category":"agentic","score":62,"unit":"percent","family":"jobBench","index":70.78574160627774,"aboveFit":false,"run":null,"source":"Yuetai Li, Yichen Feng, Zhangchen Xu, Zixian Ma, Kaiyuan Zheng, Fengqing Jiang, Xinghua Sun, Rulin Shao, Zichen Chen, Yue Huang, Xinyang Han, Brian Lee, Kayla Xu, Shenglai Zeng, Hang Hua, Xiangliang Zhang, Basel Alomair, Ranjay Krishna, Luke Zettlemoyer, Pang Wei Koh, Bhaskar Ramasubramanian, Luyao Niu, Xiang Yue, Radha Poovendran","sourceUrl":"https://arxiv.org/abs/2605.26329","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaSciCode","benchmarkName":"Artificial Analysis SciCode","category":"coding","score":60.9,"unit":"percent","family":"scicode-family","index":75.2345892591685,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/scicode","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gdpvalAaNormalized","benchmarkName":"GDPval-AA normalized","category":"agentic","score":59.2,"unit":"percent","family":"gdpvalAaNormalized","index":76.36039920934924,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaAutomationBench","benchmarkName":"Artificial Analysis AutomationBench","category":"agentic","score":58.6,"unit":"percent","family":"aaAutomationBench","index":69.09452760514533,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/automationbench-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:fabv2:overall","benchmarkName":"Finance Agent (v2)","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":57.339,"unit":"percent","family":"vals-ai:fabv2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/fabv2","measuredAt":"2026-10-29","stderr":0.574,"costPerTest":0.198909,"latency":648.075},{"benchmarkId":"aaCyberIndex","benchmarkName":"Artificial Analysis Cyber Index","category":"agentic","score":56.1,"unit":"percent","family":"aaCyberIndex","index":84.81912731829973,"aboveFit":false,"run":null,"source":"Artificial Analysis with the Cyber Index Alliance (Collinear AI, IBM, NVIDIA, Pasta Lab, Vercel)","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-cyber-index","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"automationBench","benchmarkName":"AutomationBench","category":"agentic","score":53.1,"unit":"percent","family":"automationBench","index":83.09148470959812,"aboveFit":true,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":49.4,"unit":"percent","family":"hle","index":73.94930591778532,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-long-horizon:overall","benchmarkName":"LiveBench Long Agentic","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":47.5,"unit":"percent","family":"livebench-2-agentic","index":53.32769859109191,"aboveFit":false,"run":"high effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Long%20Agentic","measuredAt":"2026-10-14","stderr":null,"costPerTest":1.2277034482758622,"latency":null},{"benchmarkId":"vals-ai:legal_research:overall","benchmarkName":"Legal Research Bench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":47.115,"unit":"percent","family":"vals-ai:legal_research:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_research","measuredAt":"2026-11-09","stderr":3.469,"costPerTest":0.181201,"latency":1816.322},{"benchmarkId":"vals-ai:medcode:overall","benchmarkName":"MedCode","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":44.969,"unit":"percent","family":"vals-ai:medcode:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medcode","measuredAt":"2026-10-29","stderr":2.097,"costPerTest":0.009528,"latency":176.915},{"benchmarkId":"vals-ai:code-migration:overall","benchmarkName":"Code Migration","category":"coding","domain":null,"eligible":true,"ranked":true,"score":43.013,"unit":"percent","family":"vals-ai:code-migration:overall","index":68.03936759501354,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/code-migration","measuredAt":"2026-10-29","stderr":4.32,"costPerTest":0.691641,"latency":10136.401},{"benchmarkId":"vals-ai:ioi:overall","benchmarkName":"IOI","category":"coding","domain":null,"eligible":true,"ranked":true,"score":39.333,"unit":"percent","family":"vals-ai:ioi:overall","index":60.883123463900404,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/ioi","measuredAt":"2026-10-29","stderr":2.406,"costPerTest":0.835497,"latency":8782.516},{"benchmarkId":"aaAnalystAgent","benchmarkName":"Artificial Analysis AnalystAgent","category":"agentic","score":37.5,"unit":"percent","family":"aaAnalystAgent","index":57.88486710885368,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/aa-analyst-agent","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mka","benchmarkName":"Multimodal Knowledge Agent Benchmark","category":"agentic","score":37.3,"unit":"percent","family":"mka","index":66.99057978317238,"aboveFit":false,"run":null,"source":"Google DeepMind, in partnership with Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mka","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaTerminalBench4","benchmarkName":"Artificial Analysis Terminal-Bench v4.0","category":"agentic","score":34.8,"unit":"percent","family":"aaTerminalBench4","index":67.55168403887811,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/terminalbench-v4-0","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"omniscienceAccuracy","benchmarkName":"Artificial Analysis Omniscience Accuracy","category":"knowledge","score":34.8,"unit":"percent","family":"omniscienceAccuracy","index":54.54685871189644,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:tax_agent_bench:overall","benchmarkName":"Tax Agent Bench","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":32.388,"unit":"percent","family":"vals-ai:tax_agent_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench","measuredAt":"2026-11-09","stderr":3.361,"costPerTest":0.134306,"latency":1547.695},{"benchmarkId":"agentsLastExam","benchmarkName":"Agents' Last Exam","category":"agentic","score":31.6,"unit":"percent","family":"agentsLastExam","index":67.2916930169336,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:terminal-bench-4:overall","benchmarkName":"Terminal-Bench 4.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":31.313,"unit":"percent","family":"terminal-bench-4","index":69.38463553147436,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-4","measuredAt":"2026-11-09","stderr":3.072,"costPerTest":0.456609,"latency":9720.048},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":26.6,"unit":"percent","family":"critpt","index":88.34361223696331,"aboveFit":true,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"programBench","benchmarkName":"ProgramBench: Can Language Models Rebuild Programs From Scratch?","category":"coding","score":26.5,"unit":"percent","family":"programBench","index":56.53307804827977,"aboveFit":false,"run":null,"source":"John Yang, Kilian Lieret, Jeffrey Ma, Parth Thakkar, Dmitrii Pedchenko, Sten Sootla, Emily McMilin, Pengcheng Yin, Rui Hou, Gabriel Synnaeve, Diyi Yang, Ofir Press","sourceUrl":"https://programbench.com/static/paper.pdf","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaDeepsecBench","benchmarkName":"Artificial Analysis DeepsecBench-AA","category":"agentic","score":26.2,"unit":"percent","family":"aaDeepsecBench","index":70.43828868273062,"aboveFit":false,"run":null,"source":"Artificial Analysis; dataset by Vercel","sourceUrl":"https://artificialanalysis.ai/evaluations/deepsecbench-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"bug-hunt-bench","benchmarkName":"Bug Hunt Bench","category":"coding","score":22.7,"unit":"percent","family":"bug-hunt-bench","index":65.34152343128764,"aboveFit":false,"run":null,"source":"Pawel Huryn","sourceUrl":"https://bughunt.productcompass.pm/method","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaGdpPdf","benchmarkName":"Artificial Analysis GDP.pdf","category":"agentic","score":19.2,"unit":"percent","family":"aaGdpPdf","index":65.39865974656786,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdp-pdf","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"exploitGym","benchmarkName":"ExploitGym","category":"agentic","score":17.8,"unit":"percent","family":"exploitGym","index":69.7871007233258,"aboveFit":false,"run":null,"source":"Zhun Wang, Nico Schiller, Hongwei Li, Srijiith Sesha Narayana, Milad Nasr, Nicholas Carlini, Xiangyu Qi, Eric Wallace, Elie Bursztein, Luca Invernizzi, Kurt Thomas, Yan Shoshitaishvili, Wenbo Guo, Jingxuan He, Thorsten Holz, Dawn Song","sourceUrl":"https://arxiv.org/abs/2605.11086","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:mysterymechanism:overall","benchmarkName":"MysteryMechanism","category":null,"domain":"Science","eligible":false,"ranked":true,"score":15.315,"unit":"percent","family":"vals-ai:mysterymechanism:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mysterymechanism","measuredAt":"2026-11-11","stderr":2.423,"costPerTest":0.153721,"latency":2639.688},{"benchmarkId":"vals-ai:hlab:overall","benchmarkName":"HLAB-Vals-1.1","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":13.75,"unit":"percent","family":"vals-ai:hlab:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/hlab","measuredAt":"2026-11-09","stderr":2.836,"costPerTest":0.216614,"latency":1861.848},{"benchmarkId":"aaMlcr","benchmarkName":"Medical Long Context Reasoning (MLCR-AA)","category":"reasoning","score":12.8,"unit":"percent","family":"aaMlcr","index":58.189900589706475,"aboveFit":false,"run":null,"source":"Wisedocs and Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mlcr-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lmarena:agent-task-outcome:overall","benchmarkName":"Agent Arena task outcome","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":5.010024727016605,"unit":"index","family":"agent-arena","index":70.64982706447293,"aboveFit":false,"run":null,"source":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset","measuredAt":"2026-11-11","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:srebench:overall","benchmarkName":"SRE Bench","category":null,"domain":"Other","eligible":false,"ranked":true,"score":3.053,"unit":"percent","family":"vals-ai:srebench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/srebench","measuredAt":"2026-11-05","stderr":1.065,"costPerTest":0.53277,"latency":8562.864},{"benchmarkId":"vals-ai:terminal-bench-science:overall","benchmarkName":"Terminal-Bench-Science","category":null,"domain":"Science","eligible":false,"ranked":true,"score":2.857,"unit":"percent","family":"vals-ai:terminal-bench-science:overall","index":null,"aboveFit":false,"run":"Mini-SWE-agent","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-science","measuredAt":"2026-11-09","stderr":2.006,"costPerTest":0.62088,"latency":11504.04},{"benchmarkId":"lmarena:agent-bash-recovery:overall","benchmarkName":"Agent Arena command recovery","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":2.3425984954652757,"unit":"index","family":"agent-arena","index":68.45678452618587,"aboveFit":false,"run":null,"source":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset","measuredAt":"2026-10-08","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"stemBench120","benchmarkName":"STEMBench120: six-domain scientific research progress","category":"reasoning","score":2.172,"unit":"percent","family":"stemBench120","index":null,"aboveFit":false,"run":null,"source":"Ulam","sourceUrl":"https://www.ulam.ai/stembench","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lmarena:agent-steerability:overall","benchmarkName":"Agent Arena steerability","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":1.502427798604139,"unit":"index","family":"agent-arena","index":67.76603249869038,"aboveFit":false,"run":null,"source":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset","measuredAt":"2026-11-11","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:programbench:overall","benchmarkName":"ProgramBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":0.5,"unit":"percent","family":"vals-ai:programbench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/programbench","measuredAt":"2026-11-09","stderr":0.5,"costPerTest":0.726582,"latency":10825.24}],"price":{"inputPerMillion":0.435,"outputPerMillion":0.87,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":31,"timeToFirstToken":3.8995,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":{"elo":1491.7070850560972,"eloUpper":1501.1318987400714,"eloLower":1482.2822713721225,"votes":4066,"rank":11,"rankUpper":null,"rankLower":null,"arenaName":"MiMo-V2.6-Pro","organization":"xiaomi","updatedAt":"2026-11-09"},"quality":{"coveragePercent":80,"benchmarkCount":36,"categoryCount":5,"confidence":"medium","state":"available"},"hosts":[{"host":"vultr","name":"Vultr","kind":"cloud","url":"https://api.vultrinference.com/","inputPerMillion":0.4,"outputPerMillion":0.8,"cacheReadPerMillion":null,"contextTokens":1048576,"maxOutputTokens":131072,"deprecated":false},{"host":"deepinfra","name":"Deep Infra","kind":"host","url":"https://deepinfra.com/models","inputPerMillion":0.43,"outputPerMillion":0.87,"cacheReadPerMillion":0.0036,"contextTokens":1048576,"maxOutputTokens":131072,"deprecated":false},{"host":"novita-ai","name":"Novita AI","kind":"host","url":"https://novita.ai/docs/guides/introduction","inputPerMillion":0.435,"outputPerMillion":0.87,"cacheReadPerMillion":0.0036,"contextTokens":1048576,"maxOutputTokens":131072,"deprecated":false},{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":0.435,"outputPerMillion":0.87,"cacheReadPerMillion":0.0036,"contextTokens":1050000,"maxOutputTokens":131072,"deprecated":false},{"host":"xiaomi","name":"Xiaomi","kind":"maker","url":"https://platform.xiaomimimo.com/#/docs","inputPerMillion":0.435,"outputPerMillion":0.87,"cacheReadPerMillion":0.0036,"contextTokens":1048576,"maxOutputTokens":131072,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:livebench","name":"LiveBench","url":"https://livebench.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:lmarena","name":"LMArena","url":"https://huggingface.co","license":"CC BY 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"muse-spark-1-3","providerId":"meta","slug":"muse-spark-1-3","name":"Muse Spark 1.3","provider":"Meta","description":"Muse Spark 1.3 is a reasoning model from Meta in the Muse Spark family. 41 benchmarks count toward its index, in 6 categories.","logoUrl":null,"releaseDate":"2026-09-02","contextWindowTokens":1048576,"modalities":["text","image","video","file"],"openWeights":false,"details":{"apiId":"meta/muse-spark-1.3","maxOutputTokens":943718,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.15,"weightsUrl":null},"reasoning":true,"overallScore":70.11249009489217,"overallScoreRange":{"low":66.08716036515341,"high":74.13781982463092},"badges":[],"capabilityScores":{"agentic":70.44204294138558,"coding":68.82907722897986,"reasoning":68.12615005110914,"multimodal":null,"knowledge":67.50786863000437,"multilingual":null,"instruction":84.42364930335084,"math":68.56353566139568},"capabilityCounts":{"math":5,"knowledge":4,"coding":10,"reasoning":5,"agentic":16,"instruction":1},"benchmarks":[{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":99.16666666666667,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":66.6136670784291,"aboveFit":false,"run":"xhigh effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0.6154574548966636,"costPerTest":null,"latency":null},{"benchmarkId":"mrcrv2_256_512","benchmarkName":"OpenAI MRCR v2 8-needle 256K-512K","category":"reasoning","score":98.5,"unit":"percent","family":"mrcrv2_256_512","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mrcrv2_512_1m","benchmarkName":"OpenAI MRCR v2 8-needle 512K-1M","category":"reasoning","score":98.1,"unit":"percent","family":"mrcrv2_512_1m","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-math:overall","benchmarkName":"LiveBench Mathematics","category":"math","domain":null,"eligible":true,"ranked":true,"score":95.94999999999999,"unit":"percent","family":"livebench:livebench-math:overall","index":73.50219046341958,"aboveFit":false,"run":"xhigh effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Mathematics","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.10447757255936675,"latency":null},{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":93.5,"unit":"percent","family":"gpqa-diamond","index":62.96474584284943,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-coding-2:overall","benchmarkName":"LiveBench Coding 2.0","category":"coding","domain":null,"eligible":true,"ranked":true,"score":90.2,"unit":"percent","family":"livebench:livebench-coding-2:overall","index":73.50604073435277,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Coding","measuredAt":"2026-10-14","stderr":null,"costPerTest":1.3716172413793102,"latency":null},{"benchmarkId":"livebench:livebench-reasoning:overall","benchmarkName":"LiveBench Reasoning","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":89.65375,"unit":"percent","family":"livebench","index":77.76023696744375,"aboveFit":false,"run":"xhigh effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Reasoning","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.1357222772277228,"latency":null},{"benchmarkId":"deepSearchQa","benchmarkName":"DeepSearchQA","category":"agentic","score":89.4,"unit":"percent","family":"deepSearchQa","index":65.51566911832825,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"terminalBench21","benchmarkName":"Terminal-Bench 2.1","category":"agentic","score":88.8,"unit":"percent","family":"terminal-bench-2-1","index":73.07627757255543,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":83,"unit":"percent","family":"lcr","index":64.2427330827805,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:vibe-code:overall","benchmarkName":"Vibe Code Bench v1.1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":82.859,"unit":"percent","family":"vals-ai:vibe-code:overall","index":72.14635160348672,"aboveFit":false,"run":"OpenHands","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","measuredAt":"2026-11-09","stderr":2.901,"costPerTest":2.101762,"latency":773.797},{"benchmarkId":"livebench:livebench-language:overall","benchmarkName":"LiveBench Language","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":82.79366666666667,"unit":"percent","family":"livebench:livebench-language:overall","index":68.18261630928322,"aboveFit":false,"run":"xhigh effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Language","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.111828,"latency":null},{"benchmarkId":"livebench:livebench-coding:overall","benchmarkName":"LiveBench Coding","category":"coding","domain":null,"eligible":true,"ranked":true,"score":81.0625,"unit":"percent","family":"livebench:livebench-coding:overall","index":68.83117201617824,"aboveFit":false,"run":"xhigh effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Coding","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.05948547008547008,"latency":null},{"benchmarkId":"livebench:livebench-browser:overall","benchmarkName":"LiveBench Browser Use","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":80,"unit":"percent","family":"livebench-2-agentic","index":77.09595917600676,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Browser","measuredAt":"2026-10-14","stderr":null,"costPerTest":1.689796,"latency":null},{"benchmarkId":"livebench:livebench-data-analysis:overall","benchmarkName":"LiveBench Data Analysis","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":79.56533333333333,"unit":"percent","family":"livebench","index":64.69135905015222,"aboveFit":false,"run":"xhigh effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Data%20Analysis","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.16942866666666664,"latency":null},{"benchmarkId":"livebench:livebench-instruction-following:overall","benchmarkName":"LiveBench Instruction Following","category":"instruction","domain":null,"eligible":true,"ranked":true,"score":78.00425000000001,"unit":"percent","family":"livebench:livebench-instruction-following:overall","index":84.42364930335084,"aboveFit":false,"run":"xhigh effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=IF","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.1329045,"latency":null},{"benchmarkId":"deepSwe","benchmarkName":"DeepSWE","category":"agentic","score":75.4,"unit":"percent","family":"deepSwe","index":75.89041576601036,"aboveFit":false,"run":null,"source":"Datacurve AI","sourceUrl":"https://deepswe.datacurve.ai/blog","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-design:overall","benchmarkName":"LiveBench Design","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":74.8,"unit":"percent","family":"livebench-2-agentic","index":73.29303748242039,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Design","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.8286239999999999,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tiers-1-3-v2-private:overall","benchmarkName":"FrontierMath-Tiers-1-3-v2-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":74.03508771929825,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tiers-1-3-v2-private:overall","index":69.86319142252621,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.6016750822385526,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:terminal-bench-2-1:overall","benchmarkName":"Terminal-Bench 2.1","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":72.285,"unit":"percent","family":"terminal-bench-2-1","index":63.4173912282842,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2-1","measuredAt":"2026-10-01","stderr":0.375,"costPerTest":0.663249,"latency":857.636},{"benchmarkId":"vals-ai:cyber:overall","benchmarkName":"CyberBench v1.1","category":null,"domain":"Other","eligible":false,"ranked":true,"score":69.405,"unit":"percent","family":"vals-ai:cyber:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/cyber","measuredAt":"2026-11-05","stderr":5.758,"costPerTest":4.258407,"latency":1492.536},{"benchmarkId":"livebench:livebench-research:overall","benchmarkName":"LiveBench Research","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":68.6,"unit":"percent","family":"livebench-2-agentic","index":68.75878469391355,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Research","measuredAt":"2026-10-14","stderr":null,"costPerTest":2.694118181818182,"latency":null},{"benchmarkId":"osWorld2","benchmarkName":"OSWorld 2.0","category":"agentic","score":66.9,"unit":"percent","family":"osWorld2","index":81.12092856111744,"aboveFit":false,"run":null,"source":"Mengqi Yuan, Zilong Zhou, Xinzhuang Xiong, Weiming Wu, Jiayang Sun, Jiamin Song, Kaiqian Cui, Bowen Wang, Haoyuan Wu, Yitong Li, Dunjie Lu, Haikong Lu, Qi Zhen, Xinyuan Wang, Jiaqi Deng, Yuhao Yang, Cheng Chen, Boyuan Zheng, Alex Su, Xiao Yu, Hao Zou, Saaket Agashe, Xing Han Lu, Manpreet Kaur, Zhengyang Qi, Vincent Sunn Chen, Frederic Sala, Dayiheng Liu, Junyang Lin, Zhou Yu, Yu Su, Siva Reddy, Xin Eric Wang, Peng Qi, Tianbao Xie, Tao Yu","sourceUrl":"https://arxiv.org/abs/2606.29537","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"jobBench","benchmarkName":"JobBench","category":"agentic","score":64.9,"unit":"percent","family":"jobBench","index":72.38972949048556,"aboveFit":false,"run":null,"source":"Yuetai Li, Yichen Feng, Zhangchen Xu, Zixian Ma, Kaiyuan Zheng, Fengqing Jiang, Xinghua Sun, Rulin Shao, Zichen Chen, Yue Huang, Xinyang Han, Brian Lee, Kayla Xu, Shenglai Zeng, Hang Hua, Xiangliang Zhang, Basel Alomair, Ranjay Krishna, Luke Zettlemoyer, Pang Wei Koh, Bhaskar Ramasubramanian, Luyao Niu, Xiang Yue, Radha Poovendran","sourceUrl":"https://arxiv.org/abs/2605.26329","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-agentic-coding:overall","benchmarkName":"LiveBench Agentic Coding","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":64.091,"unit":"percent","family":"livebench:livebench-agentic-coding:overall","index":75.25864708518819,"aboveFit":false,"run":"xhigh effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Agentic%20Coding","measuredAt":"2026-06-25","stderr":null,"costPerTest":1.1724875,"latency":null},{"benchmarkId":"vals-ai:emb:overall","benchmarkName":"EMB","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":62.712,"unit":"percent","family":"vals-ai:emb:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/emb","measuredAt":"2026-10-20","stderr":2.956,"costPerTest":3.552349,"latency":1374.999},{"benchmarkId":"sweAtlasCodebaseQna","benchmarkName":"SWE-Atlas Codebase QnA","category":"coding","score":59.4,"unit":"percent","family":"sweAtlasCodebaseQna","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gdpvalAaNormalized","benchmarkName":"GDPval-AA normalized","category":"agentic","score":59,"unit":"percent","family":"gdpvalAaNormalized","index":76.21740385701088,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:fabv2:overall","benchmarkName":"Finance Agent (v2)","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":58.901,"unit":"percent","family":"vals-ai:fabv2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/fabv2","measuredAt":"2026-10-29","stderr":0.444,"costPerTest":0.741132,"latency":341.936},{"benchmarkId":"aaSciCode","benchmarkName":"Artificial Analysis SciCode","category":"coding","score":58.8,"unit":"percent","family":"scicode-family","index":72.49683785169756,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/scicode","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"cweBenchV1","benchmarkName":"CWE-bench v1","category":"agentic","score":55,"unit":"percent","family":"cweBenchV1","index":68.55293574761009,"aboveFit":false,"run":null,"source":"Collinear AI","sourceUrl":"https://cwe-bench.com/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:proof_bench:overall","benchmarkName":"ProofBench v1.1","category":"math","domain":null,"eligible":true,"ranked":true,"score":55,"unit":"percent","family":"vals-ai:proof_bench:overall","index":66.14507914521342,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/proof_bench","measuredAt":"2026-11-09","stderr":5,"costPerTest":0.433712,"latency":534.06},{"benchmarkId":"epoch-ai:epoch-mirrorcode:overall","benchmarkName":"MirrorCode","category":"coding","domain":null,"eligible":true,"ranked":true,"score":54.44444444444444,"unit":"percent","family":"epoch-ai:epoch-mirrorcode:overall","index":73.4760754900136,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":10.846044638639055,"costPerTest":null,"latency":null},{"benchmarkId":"aaTau3Banking","benchmarkName":"Artificial Analysis Tau3-Banking","category":"agentic","score":50.5,"unit":"percent","family":"aaTau3Banking","index":76.99819198914871,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/tau3-banking","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"automationBench","benchmarkName":"AutomationBench","category":"agentic","score":49.4,"unit":"percent","family":"automationBench","index":76.05377400422934,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":48.7,"unit":"percent","family":"hle","index":73.24700099626446,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tier-4-v2-private:overall","benchmarkName":"FrontierMath-Tier-4-v2-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":46.34146341463415,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tier-4-v2-private:overall","index":66.69355019739008,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":7.884502378168105,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:ioi:overall","benchmarkName":"IOI","category":"coding","domain":null,"eligible":true,"ranked":true,"score":43.944,"unit":"percent","family":"vals-ai:ioi:overall","index":62.61419615398432,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/ioi","measuredAt":"2026-10-29","stderr":1.448,"costPerTest":2.360741,"latency":1791.133},{"benchmarkId":"omniscienceAccuracy","benchmarkName":"Artificial Analysis Omniscience Accuracy","category":"knowledge","score":43.6,"unit":"percent","family":"omniscienceAccuracy","index":65.63711137162036,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"cursorBench40","benchmarkName":"cursorBench40","category":"coding","score":41.6,"unit":"percent","family":"cursorBench40","index":68.2592359135453,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:legal_research:overall","benchmarkName":"Legal Research Bench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":40.865,"unit":"percent","family":"vals-ai:legal_research:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_research","measuredAt":"2026-11-09","stderr":3.417,"costPerTest":0.650063,"latency":530.813},{"benchmarkId":"livebench:livebench-long-horizon:overall","benchmarkName":"LiveBench Long Agentic","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":40.4,"unit":"percent","family":"livebench-2-agentic","index":48.13524781715666,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Long%20Agentic","measuredAt":"2026-10-14","stderr":null,"costPerTest":8.404320689655172,"latency":null},{"benchmarkId":"vals-ai:tax_agent_bench:overall","benchmarkName":"Tax Agent Bench","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":37.892,"unit":"percent","family":"vals-ai:tax_agent_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench","measuredAt":"2026-11-09","stderr":3.45,"costPerTest":0.319558,"latency":505.365},{"benchmarkId":"epoch-ai:epoch-chess-puzzles:overall","benchmarkName":"Chess Puzzles","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":37,"unit":"percent","family":"epoch-ai:epoch-chess-puzzles:overall","index":62.068237945692495,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":4.8523658709390975,"costPerTest":null,"latency":null},{"benchmarkId":"bug-hunt-bench","benchmarkName":"Bug Hunt Bench","category":"coding","score":32.2,"unit":"percent","family":"bug-hunt-bench","index":71.46532366719504,"aboveFit":false,"run":null,"source":"Pawel Huryn","sourceUrl":"https://bughunt.productcompass.pm/method","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:code-migration:overall","benchmarkName":"Code Migration","category":"coding","domain":null,"eligible":true,"ranked":true,"score":27.58,"unit":"percent","family":"vals-ai:code-migration:overall","index":59.840739228919716,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/code-migration","measuredAt":"2026-10-29","stderr":4.067,"costPerTest":4.616854,"latency":4061.108},{"benchmarkId":"frontierSweV2","benchmarkName":"FrontierSWE v2","category":"coding","score":25.8,"unit":"percent","family":"frontierSweV2","index":65.65479963042533,"aboveFit":false,"run":null,"source":"Proximal","sourceUrl":"https://www.frontierswe.com/blog/v2","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-mystery-game-puzzles:overall","benchmarkName":"Mystery Game Puzzles","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":25,"unit":"percent","family":"epoch-ai:epoch-mystery-game-puzzles:overall","index":58.041370619699485,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":4.464048473115664,"costPerTest":null,"latency":null},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":24.9,"unit":"percent","family":"critpt","index":85.0526105985752,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"apprenticeBench","benchmarkName":"ApprenticeBench: end-to-end computer use, continual learning, and long-horizon agency on a real accounts-payable job","category":"agentic","score":19,"unit":"percent","family":"apprenticeBench","index":67.8406050788133,"aboveFit":false,"run":null,"source":"NeoCognition","sourceUrl":"https://neocognition.io/blog/apprentice-bench/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"terminal-bench:terminal-bench-4-1-0:overall","benchmarkName":"Terminal-Bench 4.1.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":14.85,"unit":"percent","family":"terminal-bench:terminal-bench-4-1-0:overall","index":59.90584809652061,"aboveFit":false,"run":"xhigh effort · Muse Code","source":"Terminal-Bench","sourceUrl":"https://www.tbench.ai/leaderboard","measuredAt":"2026-09-29","stderr":1.530612244897959,"costPerTest":0,"latency":4341.7},{"benchmarkId":"terminal-bench:terminal-bench-4-0-0:overall","benchmarkName":"Terminal-Bench 4.0.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":14.55,"unit":"percent","family":"terminal-bench-4","index":61.42491555300008,"aboveFit":false,"run":"xhigh effort · Muse Code","source":"Terminal-Bench","sourceUrl":"https://www.tbench.ai/leaderboard","measuredAt":"2026-09-29","stderr":1.5,"costPerTest":0,"latency":4341.7},{"benchmarkId":"vals-ai:terminal-bench-4:overall","benchmarkName":"Terminal-Bench 4.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":10.606,"unit":"percent","family":"terminal-bench-4","index":59.552151990684116,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-4","measuredAt":"2026-11-09","stderr":1.75,"costPerTest":7.036467,"latency":7112.43},{"benchmarkId":"lmarena:agent-bash-recovery:overall","benchmarkName":"Agent Arena command recovery","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":6.077311225492221,"unit":"index","family":"agent-arena","index":71.5273038861413,"aboveFit":false,"run":"max effort","source":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset","measuredAt":"2026-10-08","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:terminal-bench-science:overall","benchmarkName":"Terminal-Bench-Science","category":null,"domain":"Science","eligible":false,"ranked":true,"score":4.286,"unit":"percent","family":"vals-ai:terminal-bench-science:overall","index":null,"aboveFit":false,"run":"Mini-SWE-agent","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-science","measuredAt":"2026-11-09","stderr":2.438,"costPerTest":4.407744,"latency":4050.414},{"benchmarkId":"lmarena:agent-task-outcome:overall","benchmarkName":"Agent Arena task outcome","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":3.71923026718487,"unit":"index","family":"agent-arena","index":69.58859165892999,"aboveFit":false,"run":"max effort","source":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset","measuredAt":"2026-11-11","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lmarena:agent-steerability:overall","benchmarkName":"Agent Arena steerability","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":2.6333067879621144,"unit":"index","family":"agent-arena","index":68.69579231734326,"aboveFit":false,"run":"max effort","source":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset","measuredAt":"2026-11-11","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:programbench:overall","benchmarkName":"ProgramBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":0.5,"unit":"percent","family":"vals-ai:programbench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/programbench","measuredAt":"2026-11-09","stderr":0.5,"costPerTest":11.555338,"latency":4112.719},{"benchmarkId":"mazeBench","benchmarkName":"MazeBench: visual spatial reasoning in a 3D open world","category":"agentic","score":0,"unit":"percent","family":"mazeBench","index":61.97866734834783,"aboveFit":false,"run":null,"source":"Jonathan Pappas and David Pappas","sourceUrl":"https://mazebench.com/blog?post=maze-bench-results","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":1.25,"outputPerMillion":4.25,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":76,"timeToFirstToken":3.3995,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":{"elo":1491.468287476569,"eloUpper":1496.5208114214065,"eloLower":1486.415763531731,"votes":26111,"rank":13,"rankUpper":null,"rankLower":null,"arenaName":"Muse Spark 1.3-max","organization":"meta","updatedAt":"2026-11-09"},"quality":{"coveragePercent":85,"benchmarkCount":41,"categoryCount":6,"confidence":"medium","state":"available"},"hosts":[{"host":"meta","name":"Meta","kind":"maker","url":"https://dev.meta.ai/docs","inputPerMillion":1.25,"outputPerMillion":4.25,"cacheReadPerMillion":0.15,"contextTokens":1048576,"maxOutputTokens":131072,"deprecated":false},{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":1.25,"outputPerMillion":4.25,"cacheReadPerMillion":0.15,"contextTokens":1048576,"maxOutputTokens":943718,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:livebench","name":"LiveBench","url":"https://livebench.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:terminal-bench","name":"Terminal-Bench","url":"https://www.tbench.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:lmarena","name":"LMArena","url":"https://huggingface.co","license":"CC BY 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"minimax-m3-1-flash","providerId":"minimax","slug":"minimax-m3-1-flash","name":"MiniMax M3.1 Flash","provider":"MiniMax","description":"MiniMax M3.1 Flash is a reasoning model from MiniMax. 22 benchmarks count toward its index, in 5 categories.","logoUrl":null,"releaseDate":"2026-10-19","contextWindowTokens":1048576,"modalities":["text","image","video"],"openWeights":true,"details":{"apiId":"minimax/minimax-m3.1-flash","maxOutputTokens":524288,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.015,"weightsUrl":"https://huggingface.co/MiniMaxAI/MiniMax-M3.1-Flash"},"reasoning":true,"overallScore":69.27835804080617,"overallScoreRange":{"low":63.830902457335945,"high":74.72581362427638},"badges":[],"capabilityScores":{"agentic":71.83088911477856,"coding":69.35649232050545,"reasoning":67.85347848721072,"multimodal":58.10242316970552,"knowledge":68.88230905444237,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"agentic":10,"coding":5,"reasoning":3,"multimodal":2,"knowledge":2},"benchmarks":[{"benchmarkId":"livebench:livebench-browser:overall","benchmarkName":"LiveBench Browser Use","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":92,"unit":"percent","family":"livebench-2-agentic","index":85.87193231505225,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Browser","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.22678399999999999,"latency":null},{"benchmarkId":"babyVisionPython","benchmarkName":"BabyVision with Python","category":"multimodal","score":91,"unit":"percent","family":"babyVisionPython","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:sage:overall","benchmarkName":"SAGE","category":null,"domain":"Education","eligible":false,"ranked":true,"score":85.258,"unit":"percent","family":"vals-ai:sage:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/sage","measuredAt":"2026-11-05","stderr":1.996,"costPerTest":0.007619,"latency":73.758},{"benchmarkId":"livebench:livebench-coding-2:overall","benchmarkName":"LiveBench Coding 2.0","category":"coding","domain":null,"eligible":true,"ranked":true,"score":83.9,"unit":"percent","family":"livebench:livebench-coding-2:overall","index":68.94279275546435,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Coding","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.2464310344827586,"latency":null},{"benchmarkId":"livebench:livebench-reasoning:overall","benchmarkName":"LiveBench Reasoning","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":82.9,"unit":"percent","family":"livebench","index":69.01119961596895,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Reasoning","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.01107289108910891,"latency":null},{"benchmarkId":"charxiv","benchmarkName":"CharXiv Reasoning","category":"multimodal","score":82.5,"unit":"percent","family":"charxiv","index":58.110321140649404,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"cyberGym","benchmarkName":"CyberGym","category":"agentic","score":82.2,"unit":"percent","family":"cyberGym","index":68.06639520631379,"aboveFit":false,"run":null,"source":"Zhun Wang, Tianneng Shi, Jingxuan He, Matthew Cai, Jialin Zhang, Dawn Song","sourceUrl":"https://www.cybergym.io/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":79.7,"unit":"percent","family":"lcr","index":61.97866734834783,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaMmmuPro","benchmarkName":"Artificial Analysis MMMU-Pro","category":"multimodal","score":79.1,"unit":"percent","family":"mmmu-pro","index":58.09452519876163,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmmu-pro","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"bankerToolBench","benchmarkName":"BankerToolBench","category":"agentic","score":76.2,"unit":"percent","family":"bankerToolBench","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-design:overall","benchmarkName":"LiveBench Design","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":74,"unit":"percent","family":"livebench-2-agentic","index":72.70797260648402,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Design","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.15822,"latency":null},{"benchmarkId":"vals-ai:vibe-code:overall","benchmarkName":"Vibe Code Bench v1.1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":73.944,"unit":"percent","family":"vals-ai:vibe-code:overall","index":68.92967953833191,"aboveFit":false,"run":"OpenHands","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","measuredAt":"2026-11-09","stderr":4.249,"costPerTest":1.146649,"latency":1263.916},{"benchmarkId":"deepSwe","benchmarkName":"DeepSWE","category":"agentic","score":73.7,"unit":"percent","family":"deepSwe","index":74.65456322322063,"aboveFit":false,"run":null,"source":"Datacurve AI","sourceUrl":"https://deepswe.datacurve.ai/blog","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-research:overall","benchmarkName":"LiveBench Research","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":72.7,"unit":"percent","family":"livebench-2-agentic","index":71.75724218308743,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Research","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.2662136363636364,"latency":null},{"benchmarkId":"livebench:livebench-data-analysis:overall","benchmarkName":"LiveBench Data Analysis","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":71.4,"unit":"percent","family":"livebench","index":54.11370866028355,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Data%20Analysis","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.01233386,"latency":null},{"benchmarkId":"automationBench","benchmarkName":"AutomationBench","category":"agentic","score":62.2,"unit":"percent","family":"automationBench","index":95,"aboveFit":true,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"osWorld2","benchmarkName":"OSWorld 2.0","category":"agentic","score":61.7,"unit":"percent","family":"osWorld2","index":79.01840402927736,"aboveFit":false,"run":null,"source":"Mengqi Yuan, Zilong Zhou, Xinzhuang Xiong, Weiming Wu, Jiayang Sun, Jiamin Song, Kaiqian Cui, Bowen Wang, Haoyuan Wu, Yitong Li, Dunjie Lu, Haikong Lu, Qi Zhen, Xinyuan Wang, Jiaqi Deng, Yuhao Yang, Cheng Chen, Boyuan Zheng, Alex Su, Xiao Yu, Hao Zou, Saaket Agashe, Xing Han Lu, Manpreet Kaur, Zhengyang Qi, Vincent Sunn Chen, Frederic Sala, Dayiheng Liu, Junyang Lin, Zhou Yu, Yu Su, Siva Reddy, Xin Eric Wang, Peng Qi, Tianbao Xie, Tao Yu","sourceUrl":"https://arxiv.org/abs/2606.29537","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"nl2Repo","benchmarkName":"NL2Repo","category":"coding","score":61.2,"unit":"percent","family":"nl2Repo","index":69.9311899993813,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"locaBench256k","benchmarkName":"LOCA-Bench 256K","category":"agentic","score":60.3,"unit":"percent","family":"locaBench256k","index":null,"aboveFit":false,"run":null,"source":"Weihao Zeng, Yuzhen Huang, Junxian He","sourceUrl":"https://arxiv.org/abs/2602.07962","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaSciCode","benchmarkName":"Artificial Analysis SciCode","category":"coding","score":53.4,"unit":"percent","family":"scicode-family","index":65.45690566105802,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/scicode","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":48,"unit":"percent","family":"hle","index":72.5446960747436,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gdpvalAaNormalized","benchmarkName":"GDPval-AA normalized","category":"agentic","score":47.9,"unit":"percent","family":"gdpvalAaNormalized","index":68.28116180223154,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"omniscienceAccuracy","benchmarkName":"Artificial Analysis Omniscience Accuracy","category":"knowledge","score":45.1,"unit":"percent","family":"omniscienceAccuracy","index":67.52749534770967,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hleNoTools","benchmarkName":"Humanity's Last Exam without tools","category":"knowledge","score":43.4,"unit":"percent","family":"hle","index":67.92954944760655,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"postTrainBench11","benchmarkName":"PostTrainBench v1.1","category":"coding","score":39.3,"unit":"percent","family":"postTrainBench11","index":73.5218936482917,"aboveFit":false,"run":null,"source":"PostTrainBench team","sourceUrl":"https://posttrainbench.com/?version=v1.1","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-long-horizon:overall","benchmarkName":"LiveBench Long Agentic","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":34.7,"unit":"percent","family":"livebench-2-agentic","index":43.96666057611006,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Long%20Agentic","measuredAt":"2026-10-14","stderr":null,"costPerTest":1.2589347826086956,"latency":null},{"benchmarkId":"vals-ai:legal_research:overall","benchmarkName":"Legal Research Bench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":34.135,"unit":"percent","family":"vals-ai:legal_research:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_research","measuredAt":"2026-11-09","stderr":3.296,"costPerTest":0.444777,"latency":2911.983},{"benchmarkId":"mka","benchmarkName":"Multimodal Knowledge Agent Benchmark","category":"agentic","score":33.5,"unit":"percent","family":"mka","index":63.717164619842166,"aboveFit":false,"run":null,"source":"Google DeepMind, in partnership with Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mka","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:tax_agent_bench:overall","benchmarkName":"Tax Agent Bench","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":32.136,"unit":"percent","family":"vals-ai:tax_agent_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench","measuredAt":"2026-11-09","stderr":3.342,"costPerTest":0.320948,"latency":1962.771},{"benchmarkId":"agentsLastExam","benchmarkName":"Agents' Last Exam","category":"agentic","score":29.6,"unit":"percent","family":"agentsLastExam","index":65.63137249550056,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:terminal-bench-4:overall","benchmarkName":"Terminal-Bench 4.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":25.253,"unit":"percent","family":"terminal-bench-4","index":66.5071134250761,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-4","measuredAt":"2026-11-09","stderr":1.01,"costPerTest":0.734722,"latency":3030.551},{"benchmarkId":"sweRefactorBench","benchmarkName":"SWE Refactor Bench: Can Coding Agents Complete a Long-Horizon, Whole-Repository Stack Migration?","category":"agentic","score":23,"unit":"percent","family":"sweRefactorBench","index":null,"aboveFit":false,"run":null,"source":"Deyao Hong, Yizhe Chi, Wenyi Li, Xiaoqiu Wang, Mingju Gao, Kaisen Yang, Bingxiang He, Youjie Zheng, Calvin Xiao, Qinhuai Na","sourceUrl":"https://arxiv.org/abs/2608.23564","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":22.3,"unit":"percent","family":"critpt","index":80.0193139751581,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:hlab:overall","benchmarkName":"HLAB-Vals-1.1","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":7.5,"unit":"percent","family":"vals-ai:hlab:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/hlab","measuredAt":"2026-11-09","stderr":2.265,"costPerTest":0.472848,"latency":1497.601},{"benchmarkId":"lmarena:agent-task-outcome:overall","benchmarkName":"Agent Arena task outcome","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":-0.6705674694213015,"unit":"index","family":"agent-arena","index":65.97948972502019,"aboveFit":false,"run":null,"source":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset","measuredAt":"2026-11-11","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lmarena:agent-steerability:overall","benchmarkName":"Agent Arena steerability","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":-2.566509461661415,"unit":"index","family":"agent-arena","index":64.42072817805544,"aboveFit":false,"run":null,"source":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset","measuredAt":"2026-11-11","stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":0.15,"outputPerMillion":0.6,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":99.5,"timeToFirstToken":1.371,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":80,"benchmarkCount":22,"categoryCount":5,"confidence":"medium","state":"available"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:livebench","name":"LiveBench","url":"https://livebench.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:lmarena","name":"LMArena","url":"https://huggingface.co","license":"CC BY 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"grok-4-7","providerId":"xai","slug":"grok-4-7","name":"Grok 4.7","provider":"xAI","description":"Grok 4.7 is a reasoning model from xAI. 55 benchmarks count toward its index, in 6 categories.","logoUrl":null,"releaseDate":"2026-09-21","contextWindowTokens":500000,"modalities":["text","image","file"],"openWeights":false,"details":{"apiId":"x-ai/grok-4.7","maxOutputTokens":450000,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.5,"weightsUrl":null},"reasoning":true,"overallScore":69.13287977697333,"overallScoreRange":{"low":65.4525174217672,"high":72.81324213217945},"badges":[],"capabilityScores":{"agentic":71.87586561016397,"coding":69.91876378824453,"reasoning":64.51719862364511,"multimodal":null,"knowledge":66.5314026705455,"multilingual":null,"instruction":78.9016273155556,"math":61.84018278540046},"capabilityCounts":{"math":5,"knowledge":6,"reasoning":10,"agentic":22,"coding":11,"instruction":1},"benchmarks":[{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":98.05555555555556,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":65.96722924983679,"aboveFit":false,"run":"xhigh effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.4299486306926388,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-math:overall","benchmarkName":"LiveBench Mathematics","category":"math","domain":null,"eligible":true,"ranked":true,"score":95.678,"unit":"percent","family":"livebench:livebench-math:overall","index":73.12309881558934,"aboveFit":false,"run":"xhigh effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Mathematics","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.3351303430079156,"latency":null},{"benchmarkId":"arc-prize:v1-public-eval:overall","benchmarkName":"ARC-AGI-1 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":95.125,"unit":"percent","family":"arc-prize:v1-public-eval:overall","index":null,"aboveFit":false,"run":"xhigh effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.48661554499999965,"latency":null},{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":92.73989898989899,"unit":"percent","family":"gpqa-diamond","index":62.26760196620586,"aboveFit":false,"run":"xhigh effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.5505824295534454,"costPerTest":null,"latency":null},{"benchmarkId":"arc-prize:v1-semi-private:overall","benchmarkName":"ARC-AGI-1 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":89.5,"unit":"percent","family":"arc-prize:v1-semi-private:overall","index":68.13374007639926,"aboveFit":false,"run":"xhigh effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.7784497599999999,"latency":null},{"benchmarkId":"vals-ai:medscribe:overall","benchmarkName":"MedScribe","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":89.377,"unit":"percent","family":"vals-ai:medscribe:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medscribe","measuredAt":"2026-10-29","stderr":1.886,"costPerTest":0.074505,"latency":153.053},{"benchmarkId":"vals-ai:sage:overall","benchmarkName":"SAGE","category":null,"domain":"Education","eligible":false,"ranked":true,"score":88.705,"unit":"percent","family":"vals-ai:sage:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/sage","measuredAt":"2026-11-05","stderr":1.806,"costPerTest":0.110254,"latency":120.504},{"benchmarkId":"livebench:livebench-research:overall","benchmarkName":"LiveBench Research","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":86.4,"unit":"percent","family":"livebench-2-agentic","index":81.77647818349769,"aboveFit":false,"run":"xhigh effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Research","measuredAt":"2026-10-14","stderr":null,"costPerTest":4.577345454545455,"latency":null},{"benchmarkId":"vals-ai:vibe-code:overall","benchmarkName":"Vibe Code Bench v1.1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":86.175,"unit":"percent","family":"vals-ai:vibe-code:overall","index":73.34281650175406,"aboveFit":false,"run":"OpenHands","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","measuredAt":"2026-11-09","stderr":2.181,"costPerTest":15.82706,"latency":2147.065},{"benchmarkId":"livebench:livebench-coding-2:overall","benchmarkName":"LiveBench Coding 2.0","category":"coding","domain":null,"eligible":true,"ranked":true,"score":84.5,"unit":"percent","family":"livebench:livebench-coding-2:overall","index":69.37738780107277,"aboveFit":false,"run":"xhigh effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Coding","measuredAt":"2026-10-14","stderr":null,"costPerTest":4.87988275862069,"latency":null},{"benchmarkId":"vals-ai:legal_bench:overall","benchmarkName":"LegalBench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":84.389,"unit":"percent","family":"vals-ai:legal_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","measuredAt":"2026-10-13","stderr":0.456,"costPerTest":0.010658,"latency":26.019},{"benchmarkId":"livebench:livebench-reasoning:overall","benchmarkName":"LiveBench Reasoning","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":82.65375,"unit":"percent","family":"livebench","index":68.69219899441785,"aboveFit":false,"run":"xhigh effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Reasoning","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.38507227722772275,"latency":null},{"benchmarkId":"livebench:livebench-language:overall","benchmarkName":"LiveBench Language","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":80.13866666666667,"unit":"percent","family":"livebench:livebench-language:overall","index":65.18031672159526,"aboveFit":false,"run":"xhigh effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Language","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.305302,"latency":null},{"benchmarkId":"livebench:livebench-design:overall","benchmarkName":"LiveBench Design","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":78,"unit":"percent","family":"livebench-2-agentic","index":75.63329698616585,"aboveFit":false,"run":"xhigh effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Design","measuredAt":"2026-10-14","stderr":null,"costPerTest":3.3695679999999997,"latency":null},{"benchmarkId":"livebench:livebench-coding:overall","benchmarkName":"LiveBench Coding","category":"coding","domain":null,"eligible":true,"ranked":true,"score":77.1585,"unit":"percent","family":"livebench:livebench-coding:overall","index":63.340634835811045,"aboveFit":false,"run":"xhigh effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Coding","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.3017094017094017,"latency":null},{"benchmarkId":"livebench:livebench-data-analysis:overall","benchmarkName":"LiveBench Data Analysis","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":76.88333333333334,"unit":"percent","family":"livebench","index":61.21700507248717,"aboveFit":false,"run":"xhigh effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Data%20Analysis","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.487966,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":76.7,"unit":"percent","family":"lcr","index":59.92042577159084,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-instruction-following:overall","benchmarkName":"LiveBench Instruction Following","category":"instruction","domain":null,"eligible":true,"ranked":true,"score":75.27525,"unit":"percent","family":"livebench:livebench-instruction-following:overall","index":78.9016273155556,"aboveFit":false,"run":"xhigh effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=IF","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.2369285,"latency":null},{"benchmarkId":"hermesBench","benchmarkName":"Hermes Bench: 150 agent tasks graded on the files and state the agent leaves behind","category":"agentic","score":74.38,"unit":"percent","family":"hermesBench","index":null,"aboveFit":false,"run":null,"source":"Nous Research","sourceUrl":"https://portal.nousresearch.com/bench","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaCyberGymE2e","benchmarkName":"Artificial Analysis CyberGym-E2E-AA","category":"agentic","score":74,"unit":"percent","family":"aaCyberGymE2e","index":95,"aboveFit":true,"run":null,"source":"Artificial Analysis; dataset by Berkeley RDI","sourceUrl":"https://artificialanalysis.ai/evaluations/cybergym-e2e-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:terminal-bench-2-1:overall","benchmarkName":"Terminal-Bench 2.1","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":73.408,"unit":"percent","family":"terminal-bench-2-1","index":64.07418380258736,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2-1","measuredAt":"2026-10-01","stderr":1.498,"costPerTest":1.076601,"latency":797.952},{"benchmarkId":"deepSwe","benchmarkName":"DeepSWE","category":"agentic","score":71,"unit":"percent","family":"deepSwe","index":72.69173859643695,"aboveFit":false,"run":null,"source":"Datacurve AI","sourceUrl":"https://deepswe.datacurve.ai/blog","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:cyber:overall","benchmarkName":"CyberBench v1.1","category":null,"domain":"Other","eligible":false,"ranked":true,"score":69.464,"unit":"percent","family":"vals-ai:cyber:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/cyber","measuredAt":"2026-11-05","stderr":5.666,"costPerTest":8.539306,"latency":930.024},{"benchmarkId":"vals-ai:biomysterybench:overall","benchmarkName":"BioMysteryBench","category":null,"domain":"Science","eligible":false,"ranked":true,"score":69.259,"unit":"percent","family":"vals-ai:biomysterybench:overall","index":null,"aboveFit":false,"run":"Terminus 2","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/biomysterybench","measuredAt":"2026-11-09","stderr":1.335,"costPerTest":2.487742,"latency":696.818},{"benchmarkId":"aaCweBench","benchmarkName":"Artificial Analysis CWE-Bench-AA","category":"agentic","score":68.3,"unit":"percent","family":"aaCweBench","index":78.92115781667178,"aboveFit":false,"run":null,"source":"Artificial Analysis; dataset by Collinear AI","sourceUrl":"https://artificialanalysis.ai/evaluations/cwe-bench-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"cweBenchV1","benchmarkName":"CWE-bench v1","category":"agentic","score":68,"unit":"percent","family":"cweBenchV1","index":79.47014074802985,"aboveFit":false,"run":null,"source":"Collinear AI","sourceUrl":"https://cwe-bench.com/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:emb:overall","benchmarkName":"EMB","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":66.991,"unit":"percent","family":"vals-ai:emb:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/emb","measuredAt":"2026-10-20","stderr":3.045,"costPerTest":6.481284,"latency":2097.023},{"benchmarkId":"vals-ai:public-benefits-bench:overall","benchmarkName":"Public Benefits Bench v1.1","category":null,"domain":"Public benefits","eligible":false,"ranked":true,"score":65.629,"unit":"percent","family":"vals-ai:public-benefits-bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/public-benefits-bench","measuredAt":"2026-10-12","stderr":1.235,"costPerTest":2.246775,"latency":1016.967},{"benchmarkId":"aaAutomationBench","benchmarkName":"Artificial Analysis AutomationBench","category":"agentic","score":65.6,"unit":"percent","family":"aaAutomationBench","index":75.4831192183436,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/automationbench-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"eeBench","benchmarkName":"EEBench V1 core corpus","category":"coding","score":64,"unit":"percent","family":"eeBench","index":null,"aboveFit":false,"run":null,"source":"atopile","sourceUrl":"https://www.eebench.org/","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"arc-prize:v2-semi-private:overall","benchmarkName":"ARC-AGI-2 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":61.38888888888888,"unit":"percent","family":"arc-prize:v2-semi-private:overall","index":67.37428454550101,"aboveFit":false,"run":"xhigh effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":2.0149060666666663,"latency":null},{"benchmarkId":"gdpvalAaNormalized","benchmarkName":"GDPval-AA normalized","category":"agentic","score":60.7,"unit":"percent","family":"gdpvalAaNormalized","index":77.43286435188699,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"arc-prize:v2-public-eval:overall","benchmarkName":"ARC-AGI-2 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":60,"unit":"percent","family":"arc-prize:v2-public-eval:overall","index":null,"aboveFit":false,"run":"xhigh effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":2.0405754333333324,"latency":null},{"benchmarkId":"livebench:livebench-browser:overall","benchmarkName":"LiveBench Browser Use","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":60,"unit":"percent","family":"livebench-2-agentic","index":62.46933727759762,"aboveFit":false,"run":"xhigh effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Browser","measuredAt":"2026-10-14","stderr":null,"costPerTest":4.777796,"latency":null},{"benchmarkId":"vals-ai:ioi:overall","benchmarkName":"IOI","category":"coding","domain":null,"eligible":true,"ranked":true,"score":57.722,"unit":"percent","family":"vals-ai:ioi:overall","index":67.7867659921921,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/ioi","measuredAt":"2026-10-29","stderr":1.987,"costPerTest":12.707908,"latency":2808.481},{"benchmarkId":"aaSciCode","benchmarkName":"Artificial Analysis SciCode","category":"coding","score":57.4,"unit":"percent","family":"scicode-family","index":70.67167024671694,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/scicode","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"healthBenchProfessional","benchmarkName":"HealthBench Professional","category":"knowledge","score":56.7,"unit":"percent","family":"healthBenchProfessional","index":69.03705883218245,"aboveFit":false,"run":null,"source":"Rebecca Soskin Hicks, Mikhail Trofimov, Dominick Lim, Rahul K. Arora, Foivos Tsimpourlas, Preston Bowman, Michael Sharman, Chi Tong, Kavin Karthik, Arnav Dugar, Akshay Jagadeesh, Khaled Saab, Johannes Heidecke, Ashley Alexander, Nate Gross, Karan Singhal","sourceUrl":"https://arxiv.org/abs/2604.27470","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaCyberIndex","benchmarkName":"Artificial Analysis Cyber Index","category":"agentic","score":56.4,"unit":"percent","family":"aaCyberIndex","index":85.12690799250606,"aboveFit":false,"run":null,"source":"Artificial Analysis with the Cyber Index Alliance (Collinear AI, IBM, NVIDIA, Pasta Lab, Vercel)","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-cyber-index","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-simpleqa-verified:overall","benchmarkName":"SimpleQA Verified","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":56.00000000000001,"unit":"percent","family":"epoch-ai:epoch-simpleqa-verified:overall","index":70.57199589322501,"aboveFit":false,"run":"xhigh effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.5704987954361718,"costPerTest":null,"latency":null},{"benchmarkId":"cweBenchV2","benchmarkName":"CWE-bench v2","category":"agentic","score":54.7,"unit":"percent","family":"cweBenchV2","index":69.84610154546651,"aboveFit":false,"run":null,"source":"Collinear AI","sourceUrl":"https://cwe-bench.com/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-mirrorcode:overall","benchmarkName":"MirrorCode","category":"coding","domain":null,"eligible":true,"ranked":true,"score":54.44444444444444,"unit":"percent","family":"epoch-ai:epoch-mirrorcode:overall","index":73.4760754900136,"aboveFit":false,"run":"xhigh effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":11.439589045541112,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-agentic-coding:overall","benchmarkName":"LiveBench Agentic Coding","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":53.989666666666665,"unit":"percent","family":"livebench:livebench-agentic-coding:overall","index":64.84131406120413,"aboveFit":false,"run":"xhigh effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Agentic%20Coding","measuredAt":"2026-06-25","stderr":null,"costPerTest":4.154619444444445,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tiers-1-3-v2-private:overall","benchmarkName":"FrontierMath-Tiers-1-3-v2-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":52.98245614035088,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tiers-1-3-v2-private:overall","index":58.78210276360579,"aboveFit":false,"run":"xhigh effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.9616712041900364,"costPerTest":null,"latency":null},{"benchmarkId":"aaAnalystAgent","benchmarkName":"Artificial Analysis AnalystAgent","category":"agentic","score":52.5,"unit":"percent","family":"aaAnalystAgent","index":74.83632365952973,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/aa-analyst-agent","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:fabv2:overall","benchmarkName":"Finance Agent (v2)","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":52.251,"unit":"percent","family":"vals-ai:fabv2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/fabv2","measuredAt":"2026-10-29","stderr":0.355,"costPerTest":2.719136,"latency":932.724},{"benchmarkId":"livebench:livebench-long-horizon:overall","benchmarkName":"LiveBench Long Agentic","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":50.1,"unit":"percent","family":"livebench-2-agentic","index":55.229159437885095,"aboveFit":false,"run":"xhigh effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Long%20Agentic","measuredAt":"2026-10-14","stderr":null,"costPerTest":17.41556896551724,"latency":null},{"benchmarkId":"vals-ai:medcode:overall","benchmarkName":"MedCode","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":49.553,"unit":"percent","family":"vals-ai:medcode:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medcode","measuredAt":"2026-10-29","stderr":2.171,"costPerTest":0.105497,"latency":213.135},{"benchmarkId":"vals-ai:legal_research:overall","benchmarkName":"Legal Research Bench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":47.115,"unit":"percent","family":"vals-ai:legal_research:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_research","measuredAt":"2026-11-09","stderr":3.469,"costPerTest":4.920812,"latency":1274.337},{"benchmarkId":"cursorBench40","benchmarkName":"cursorBench40","category":"coding","score":46.3,"unit":"percent","family":"cursorBench40","index":72.04162548033266,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:code-migration:overall","benchmarkName":"Code Migration","category":"coding","domain":null,"eligible":true,"ranked":true,"score":44.816,"unit":"percent","family":"vals-ai:code-migration:overall","index":68.99719348395718,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/code-migration","measuredAt":"2026-10-29","stderr":4.215,"costPerTest":36.55237,"latency":4141.411},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":43.1,"unit":"percent","family":"hle","index":67.62856162409761,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"omniscienceAccuracy","benchmarkName":"Artificial Analysis Omniscience Accuracy","category":"knowledge","score":42.7,"unit":"percent","family":"omniscienceAccuracy","index":64.50288098596678,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaItbench","benchmarkName":"Artificial Analysis ITBench-AA","category":"agentic","score":42.1,"unit":"percent","family":"aaItbench","index":61.000592677633904,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/itbench-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-chess-puzzles:overall","benchmarkName":"Chess Puzzles","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":41,"unit":"percent","family":"epoch-ai:epoch-chess-puzzles:overall","index":65.10429872629288,"aboveFit":false,"run":"xhigh effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":4.943110704237103,"costPerTest":null,"latency":null},{"benchmarkId":"mka","benchmarkName":"Multimodal Knowledge Agent Benchmark","category":"agentic","score":40.1,"unit":"percent","family":"mka","index":69.40256990352097,"aboveFit":false,"run":null,"source":"Google DeepMind, in partnership with Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mka","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hermesIndex","benchmarkName":"Hermes Index: agentic capability and cost per task inside Hermes Agent","category":"agentic","score":39.32,"unit":"percent","family":"hermesIndex","index":67.75786035759566,"aboveFit":false,"run":null,"source":"Nous Research","sourceUrl":"https://portal.nousresearch.com/bench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"terminal-bench:terminal-bench-4-1-0:overall","benchmarkName":"Terminal-Bench 4.1.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":38.79,"unit":"percent","family":"terminal-bench:terminal-bench-4-1-0:overall","index":71.05974105745577,"aboveFit":false,"run":"xhigh effort · Grok Build","source":"Terminal-Bench","sourceUrl":"https://www.tbench.ai/leaderboard","measuredAt":"2026-09-29","stderr":1.8316326530612244,"costPerTest":11.161476200000001,"latency":5700.9},{"benchmarkId":"terminal-bench:terminal-bench-4-0-0:overall","benchmarkName":"Terminal-Bench 4.0.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":37.58,"unit":"percent","family":"terminal-bench-4","index":72.36044923457632,"aboveFit":false,"run":"xhigh effort · Grok Build","source":"Terminal-Bench","sourceUrl":"https://www.tbench.ai/leaderboard","measuredAt":"2026-09-29","stderr":1.806122448979592,"costPerTest":11.161476200000001,"latency":5700.9},{"benchmarkId":"vals-ai:tax_agent_bench:overall","benchmarkName":"Tax Agent Bench","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":35.751,"unit":"percent","family":"vals-ai:tax_agent_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench","measuredAt":"2026-11-09","stderr":3.459,"costPerTest":1.796539,"latency":996.469},{"benchmarkId":"frontierSweV2","benchmarkName":"FrontierSWE v2","category":"coding","score":29.5,"unit":"percent","family":"frontierSweV2","index":67.43915289267989,"aboveFit":false,"run":null,"source":"Proximal","sourceUrl":"https://www.frontierswe.com/blog/v2","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"bug-hunt-bench","benchmarkName":"Bug Hunt Bench","category":"coding","score":29,"unit":"percent","family":"bug-hunt-bench","index":69.40256990352097,"aboveFit":false,"run":null,"source":"Pawel Huryn","sourceUrl":"https://bughunt.productcompass.pm/method","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-mystery-game-puzzles:overall","benchmarkName":"Mystery Game Puzzles","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":28.999999999999996,"unit":"percent","family":"epoch-ai:epoch-mystery-game-puzzles:overall","index":61.139030206520175,"aboveFit":false,"run":"xhigh effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":4.560480215720687,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:terminal-bench-4:overall","benchmarkName":"Terminal-Bench 4.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":28.788,"unit":"percent","family":"terminal-bench-4","index":68.18566798714176,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-4","measuredAt":"2026-11-09","stderr":2.314,"costPerTest":18.011582,"latency":3007.523},{"benchmarkId":"frontierCode2","benchmarkName":"FrontierCode 2.0","category":"coding","score":28.6,"unit":"percent","family":"frontierCode2","index":73.23050904263872,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaDeepsecBench","benchmarkName":"Artificial Analysis DeepsecBench-AA","category":"agentic","score":26.9,"unit":"percent","family":"aaDeepsecBench","index":70.97676478086578,"aboveFit":false,"run":null,"source":"Artificial Analysis; dataset by Vercel","sourceUrl":"https://artificialanalysis.ai/evaluations/deepsecbench-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:proof_bench:overall","benchmarkName":"ProofBench v1.1","category":"math","domain":null,"eligible":true,"ranked":true,"score":26,"unit":"percent","family":"vals-ai:proof_bench:overall","index":56.13298281645785,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/proof_bench","measuredAt":"2026-11-09","stderr":4.408,"costPerTest":0.787951,"latency":767.177},{"benchmarkId":"aaTerminalBench4","benchmarkName":"Artificial Analysis Terminal-Bench v4.0","category":"agentic","score":25.8,"unit":"percent","family":"aaTerminalBench4","index":63.20393288024796,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/terminalbench-v4-0","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:mysterymechanism:overall","benchmarkName":"MysteryMechanism","category":null,"domain":"Science","eligible":false,"ranked":true,"score":25.225,"unit":"percent","family":"vals-ai:mysterymechanism:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mysterymechanism","measuredAt":"2026-11-11","stderr":2.921,"costPerTest":1.841835,"latency":1052.564},{"benchmarkId":"epoch-ai:epoch-furniture-assembly:overall","benchmarkName":"Furniture Assembly","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":20.833333333333336,"unit":"percent","family":"epoch-ai:epoch-furniture-assembly:overall","index":56.37554098057187,"aboveFit":false,"run":"xhigh effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":5.219972869740667,"costPerTest":null,"latency":null},{"benchmarkId":"aaGdpPdf","benchmarkName":"Artificial Analysis GDP.pdf","category":"agentic","score":20,"unit":"percent","family":"aaGdpPdf","index":66.32159923199417,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdp-pdf","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaMlcr","benchmarkName":"Medical Long Context Reasoning (MLCR-AA)","category":"reasoning","score":20,"unit":"percent","family":"aaMlcr","index":64.97856771330432,"aboveFit":false,"run":null,"source":"Wisedocs and Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mlcr-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaHarveyLab","benchmarkName":"Artificial Analysis Harvey LAB-AA v1.0","category":"agentic","score":19.6,"unit":"percent","family":"aaHarveyLab","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:hlab:overall","benchmarkName":"HLAB-Vals-1.1","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":18.333,"unit":"percent","family":"vals-ai:hlab:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/hlab","measuredAt":"2026-11-09","stderr":3.183,"costPerTest":10.746259,"latency":2148.428},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":17.7,"unit":"percent","family":"critpt","index":71.11425071834321,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tier-4-v2-private:overall","benchmarkName":"FrontierMath-Tier-4-v2-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":17.073170731707318,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tier-4-v2-private:overall","index":55.1955002815125,"aboveFit":false,"run":"xhigh effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":5.949419959829496,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:terminal-bench-science:overall","benchmarkName":"Terminal-Bench-Science","category":null,"domain":"Science","eligible":false,"ranked":true,"score":10,"unit":"percent","family":"vals-ai:terminal-bench-science:overall","index":null,"aboveFit":false,"run":"Mini-SWE-agent","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-science","measuredAt":"2026-11-09","stderr":3.612,"costPerTest":17.040638,"latency":5553.191},{"benchmarkId":"lmarena:agent-bash-recovery:overall","benchmarkName":"Agent Arena command recovery","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":7.203833245634916,"unit":"index","family":"agent-arena","index":72.45348159305138,"aboveFit":false,"run":"xhigh effort","source":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset","measuredAt":"2026-10-08","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lmarena:agent-task-outcome:overall","benchmarkName":"Agent Arena task outcome","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":5.069960536959939,"unit":"index","family":"agent-arena","index":70.69910369646286,"aboveFit":false,"run":"xhigh effort","source":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset","measuredAt":"2026-11-11","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:srebench:overall","benchmarkName":"SRE Bench","category":null,"domain":"Other","eligible":false,"ranked":true,"score":3.817,"unit":"percent","family":"vals-ai:srebench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/srebench","measuredAt":"2026-11-05","stderr":1.186,"costPerTest":64.767258,"latency":6042.744},{"benchmarkId":"arc-prize:v3-semi-private:overall","benchmarkName":"ARC-AGI-3 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":1.8162526536637038,"unit":"percent","family":"arc-prize:v3-semi-private:overall","index":66.07724546447504,"aboveFit":false,"run":"xhigh effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mazeBench","benchmarkName":"MazeBench: visual spatial reasoning in a 3D open world","category":"agentic","score":1,"unit":"percent","family":"mazeBench","index":63.79865130242642,"aboveFit":false,"run":null,"source":"Jonathan Pappas and David Pappas","sourceUrl":"https://mazebench.com/blog?post=maze-bench-results","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:programbench:overall","benchmarkName":"ProgramBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":0.5,"unit":"percent","family":"vals-ai:programbench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/programbench","measuredAt":"2026-11-09","stderr":0.5,"costPerTest":46.490946,"latency":13078.924},{"benchmarkId":"epoch-ai:epoch-frontiermath-erdos:overall","benchmarkName":"FrontierMath-Erdos","category":"math","domain":null,"eligible":true,"ranked":true,"score":0,"unit":"percent","family":"epoch-ai:epoch-frontiermath-erdos:overall","index":null,"aboveFit":false,"run":"xhigh effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0,"costPerTest":null,"latency":null},{"benchmarkId":"lmarena:agent-steerability:overall","benchmarkName":"Agent Arena steerability","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":-2.2652272474674717,"unit":"index","family":"agent-arena","index":64.66842939053561,"aboveFit":false,"run":"xhigh effort","source":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset","measuredAt":"2026-11-11","stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":2,"outputPerMillion":6,"batch":{"inputPerMillion":1.5999999999999999,"outputPerMillion":4.8},"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":51,"timeToFirstToken":5.098,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":{"elo":1398.469600223421,"eloUpper":1403.804586487242,"eloLower":1393.1346139595998,"votes":21586,"rank":160,"rankUpper":null,"rankLower":null,"arenaName":"Grok 4.7-xhigh","organization":"xai","updatedAt":"2026-11-09"},"quality":{"coveragePercent":85,"benchmarkCount":55,"categoryCount":6,"confidence":"medium","state":"available"},"hosts":[{"host":"amazon-bedrock","name":"Amazon Bedrock","kind":"cloud","url":"https://docs.aws.amazon.com/bedrock/latest/userguide/models-supported.html","inputPerMillion":2,"outputPerMillion":6,"cacheReadPerMillion":0.5,"contextTokens":500000,"maxOutputTokens":500000,"deprecated":false},{"host":"google-vertex","name":"Google Vertex AI","kind":"cloud","url":"https://cloud.google.com/vertex-ai/generative-ai/docs/models","inputPerMillion":2,"outputPerMillion":6,"cacheReadPerMillion":0.5,"contextTokens":524288,"maxOutputTokens":500000,"deprecated":false},{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":2,"outputPerMillion":6,"cacheReadPerMillion":0.5,"contextTokens":500000,"maxOutputTokens":450000,"deprecated":false},{"host":"xai","name":"xAI","kind":"maker","url":"https://docs.x.ai/docs/models","inputPerMillion":2,"outputPerMillion":6,"cacheReadPerMillion":0.5,"contextTokens":500000,"maxOutputTokens":500000,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:livebench","name":"LiveBench","url":"https://livebench.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:arc-prize-foundation","name":"ARC Prize Foundation","url":"https://arcprize.org","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:terminal-bench","name":"Terminal-Bench","url":"https://www.tbench.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:lmarena","name":"LMArena","url":"https://huggingface.co","license":"CC BY 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gpt-5-5","providerId":"openai","slug":"gpt-5-5","name":"GPT-5.5","provider":"OpenAI","description":"GPT-5.5 is a reasoning model from OpenAI. 68 benchmarks count toward its index, in 7 categories.","logoUrl":null,"releaseDate":"2026-04-23","contextWindowTokens":1050000,"modalities":["file","image","text"],"openWeights":false,"details":{"apiId":"openai/gpt-5.5","maxOutputTokens":128000,"knowledgeCutoff":"2025-12-01","retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.5,"weightsUrl":null},"reasoning":true,"overallScore":69.10754301502682,"overallScoreRange":{"low":66.64664248660807,"high":71.56844354344557},"badges":[],"capabilityScores":{"agentic":65.77969712109214,"coding":65.5386874738395,"reasoning":74.06049310309672,"multimodal":66.82220456644293,"knowledge":70.41060114512058,"multilingual":null,"instruction":64.82798189994331,"math":74.65429289663987},"capabilityCounts":{"math":6,"agentic":25,"reasoning":10,"knowledge":6,"multimodal":4,"coding":15,"instruction":2},"benchmarks":[{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":100,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":67.09849544987334,"aboveFit":false,"run":"xhigh effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0,"costPerTest":null,"latency":null},{"benchmarkId":"tau2Bench","benchmarkName":"τ²-Bench Tool-Agent-User Evaluation","category":"agentic","score":98,"unit":"percent","family":"tau2-bench","index":67.81960511365625,"aboveFit":false,"run":null,"source":"Victor Barres, Honghua Dong, Soham Ray, Xujie Si, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2506.07982","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"arc-prize:v1-public-eval:overall","benchmarkName":"ARC-AGI-1 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":97.5,"unit":"percent","family":"arc-prize:v1-public-eval:overall","index":null,"aboveFit":false,"run":"xhigh effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.52,"latency":null},{"benchmarkId":"livebench:livebench-math:overall","benchmarkName":"LiveBench Mathematics","category":"math","domain":null,"eligible":true,"ranked":true,"score":95.85725,"unit":"percent","family":"livebench:livebench-math:overall","index":73.37292299894806,"aboveFit":false,"run":"xhigh effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Mathematics","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.1962633245382586,"latency":null},{"benchmarkId":"arc-prize:v1-semi-private:overall","benchmarkName":"ARC-AGI-1 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":95,"unit":"percent","family":"arc-prize:v1-semi-private:overall","index":70.66228864275845,"aboveFit":false,"run":"xhigh effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.73,"latency":null},{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":94.00252525252525,"unit":"percent","family":"gpqa-diamond","index":63.42564827292275,"aboveFit":false,"run":"xhigh effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.546977117727751,"costPerTest":null,"latency":null},{"benchmarkId":"gpqa","benchmarkName":"Graduate-Level Google-Proof Q&A","category":"knowledge","score":93.6,"unit":"percent","family":"gpqa-diamond","index":63.056463110341404,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gpqaDiamond","benchmarkName":"GPQA Diamond","category":"knowledge","score":93.6,"unit":"percent","family":"gpqa-diamond","index":63.056463110341404,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":93.5,"unit":"percent","family":"gpqa-diamond","index":62.96474584284943,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"extractBenchLlamaIndex","benchmarkName":"ExtractBench: schema-guided enterprise document extraction (LlamaIndex leaderboard)","category":"multimodal","score":93.35,"unit":"percent","family":"extractBenchLlamaIndex","index":75.96124566018207,"aboveFit":false,"run":null,"source":"Boyang Zhang, Adrian Lyjak, Eli Stewart, Zhaoqi Li and Simon Suo (LlamaIndex)","sourceUrl":"https://arxiv.org/abs/2607.29677","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:gpqa:overall","benchmarkName":"GPQA Diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":93.182,"unit":"percent","family":"gpqa-diamond","index":62.67308493222494,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","measuredAt":"2026-09-01","stderr":1.288,"costPerTest":0.082767,"latency":112.326},{"benchmarkId":"arc-prize:v2-public-eval:overall","benchmarkName":"ARC-AGI-2 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":90.56,"unit":"percent","family":"arc-prize:v2-public-eval:overall","index":null,"aboveFit":false,"run":"xhigh effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":1.91,"latency":null},{"benchmarkId":"livebench:livebench-reasoning:overall","benchmarkName":"LiveBench Reasoning","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":89.65375,"unit":"percent","family":"livebench","index":77.76023696744375,"aboveFit":false,"run":"xhigh effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Reasoning","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.2892287128712871,"latency":null},{"benchmarkId":"vals-ai:mmmu:overall","benchmarkName":"MMMU Pro","category":"multimodal","domain":null,"eligible":true,"ranked":true,"score":88.266,"unit":"percent","family":"mmmu-pro","index":70.02327002084391,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmmu","measuredAt":"2026-09-01","stderr":0.774,"costPerTest":0.043319,"latency":54.147},{"benchmarkId":"vals-ai:mmlu_pro:overall","benchmarkName":"MMLU Pro","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":88.144,"unit":"percent","family":"mmlu-pro","index":55.531617861355706,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","measuredAt":"2026-09-01","stderr":0.318,"costPerTest":0.027891,"latency":42.114},{"benchmarkId":"mrcrv2_128_256","benchmarkName":"OpenAI MRCR v2 8-needle 128K-256K","category":"reasoning","score":87.5,"unit":"percent","family":"mrcrv2_128_256","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-language:overall","benchmarkName":"LiveBench Language","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":87.363,"unit":"percent","family":"livebench:livebench-language:overall","index":73.34966247978248,"aboveFit":false,"run":"xhigh effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Language","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.18987533333333334,"latency":null},{"benchmarkId":"vals-ai:medscribe:overall","benchmarkName":"MedScribe","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":86.868,"unit":"percent","family":"vals-ai:medscribe:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medscribe","measuredAt":"2026-10-29","stderr":1.932,"costPerTest":0.142988,"latency":132.761},{"benchmarkId":"vals-ai:legal_bench:overall","benchmarkName":"LegalBench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":86.515,"unit":"percent","family":"vals-ai:legal_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","measuredAt":"2026-10-13","stderr":0.41,"costPerTest":0.015403,"latency":18.139},{"benchmarkId":"vals-ai:lcb:overall","benchmarkName":"LiveCodeBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":85.296,"unit":"percent","family":"livecodebench","index":60.11737437251642,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","measuredAt":"2026-09-01","stderr":1.018,"costPerTest":0.0666,"latency":166.845},{"benchmarkId":"epoch-ai:epoch-frontiermath-tiers-1-3-v2-private:overall","benchmarkName":"FrontierMath-Tiers-1-3-v2-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":85.26315789473684,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tiers-1-3-v2-private:overall","index":75.77310537395043,"aboveFit":false,"run":"xhigh effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.1034091168852487,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-coding-2:overall","benchmarkName":"LiveBench Coding 2.0","category":"coding","domain":null,"eligible":true,"ranked":true,"score":85.1,"unit":"percent","family":"livebench:livebench-coding-2:overall","index":69.81198284668119,"aboveFit":false,"run":"xhigh effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Coding","measuredAt":"2026-10-14","stderr":null,"costPerTest":4.207644827586207,"latency":null},{"benchmarkId":"arc-prize:v2-semi-private:overall","benchmarkName":"ARC-AGI-2 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":85,"unit":"percent","family":"arc-prize:v2-semi-private:overall","index":78.21496498521581,"aboveFit":false,"run":"xhigh effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":1.87,"latency":null},{"benchmarkId":"reactNativeEvals","benchmarkName":"React Native Evals","category":"coding","score":84.7,"unit":"percent","family":"reactNativeEvals","index":61.859637325072796,"aboveFit":false,"run":null,"source":"Callstack","sourceUrl":"https://rn-evals.vercel.app/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"browseComp","benchmarkName":"BrowseComp","category":"agentic","score":84.4,"unit":"percent","family":"browseComp","index":66.58666567694021,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaTerminalBench21","benchmarkName":"Artificial Analysis Terminal-Bench v2.1","category":"agentic","score":84.3,"unit":"percent","family":"aaTerminalBench21","index":66.89613078917513,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/terminalbench-v2-1","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":84.3,"unit":"percent","family":"lcr","index":65.13463776604186,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmmuProPython","benchmarkName":"MMMU-Pro with Python","category":"multimodal","score":83.2,"unit":"percent","family":"mmmuProPython","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mrcrv2_64_128","benchmarkName":"OpenAI MRCR v2 8-needle 64K-128K","category":"reasoning","score":83.1,"unit":"percent","family":"mrcrv2_64_128","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:swebench:overall","benchmarkName":"SWE-bench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":82.6,"unit":"percent","family":"swe-bench-verified","index":62.1501834544046,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/swebench","measuredAt":"2026-09-01","stderr":1.695,"costPerTest":1.361765,"latency":426.425},{"benchmarkId":"livebench:livebench-coding:overall","benchmarkName":"LiveBench Coding","category":"coding","domain":null,"eligible":true,"ranked":true,"score":82.14949999999999,"unit":"percent","family":"livebench:livebench-coding:overall","index":70.35991533458477,"aboveFit":false,"run":"xhigh effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Coding","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.15368974358974358,"latency":null},{"benchmarkId":"terminalBench2","benchmarkName":"Terminal-Bench 2.0","category":"agentic","score":82,"unit":"percent","family":"terminalBench2","index":72.22796931380128,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"cyberGym","benchmarkName":"CyberGym","category":"agentic","score":81.8,"unit":"percent","family":"cyberGym","index":67.81910808314663,"aboveFit":false,"run":null,"source":"Zhun Wang, Tianneng Shi, Jingxuan He, Matthew Cai, Jialin Zhang, Dawn Song","sourceUrl":"https://www.cybergym.io/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-data-analysis:overall","benchmarkName":"LiveBench Data Analysis","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":81.57566666666666,"unit":"percent","family":"livebench","index":67.29561319373884,"aboveFit":false,"run":"xhigh effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Data%20Analysis","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.40549999999999997,"latency":null},{"benchmarkId":"mmmuPro","benchmarkName":"Massive Multi-discipline Multimodal Understanding Pro","category":"multimodal","score":81.2,"unit":"percent","family":"mmmu-pro","index":60.82749095551189,"aboveFit":false,"run":null,"source":"MMMU-Pro authors","sourceUrl":"https://arxiv.org/abs/2409.02813","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-swe-bench-verified:overall","benchmarkName":"SWE-Bench verified","category":"coding","domain":null,"eligible":true,"ranked":true,"score":80.57851239669421,"unit":"percent","family":"swe-bench-verified","index":60.65921995419062,"aboveFit":false,"run":"xhigh effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.8000198869416815,"costPerTest":null,"latency":null},{"benchmarkId":"aaMmmuPro","benchmarkName":"Artificial Analysis MMMU-Pro","category":"multimodal","score":79.9,"unit":"percent","family":"mmmu-pro","index":59.13565501085698,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmmu-pro","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"osWorldVerified","benchmarkName":"OSWorld-Verified","category":"agentic","score":78.7,"unit":"percent","family":"osWorldVerified","index":64.01214130581246,"aboveFit":false,"run":null,"source":"Tianbao Xie, Danyang Zhang, Jixuan Chen, Xiaochuan Li, Siheng Zhao, Ruisheng Cao, Toh Jing Hua, Zhoujun Cheng, Dongchan Shin, Fangyu Lei, Yitao Liu, Yiheng Xu, Shuyan Zhou, Silvio Savarese, Caiming Xiong, Victor Zhong, Tao Yu","sourceUrl":"https://os-world.github.io/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lifeVal","benchmarkName":"LifeVal: everyday life tasks","category":"agentic","score":77.4,"unit":"percent","family":"lifeVal","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:terminal-bench-2-1:overall","benchmarkName":"Terminal-Bench 2.1","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":76.404,"unit":"percent","family":"terminal-bench-2-1","index":65.8264104745484,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2-1","measuredAt":"2026-10-01","stderr":0.649,"costPerTest":0.741883,"latency":427.412},{"benchmarkId":"aaIfBench","benchmarkName":"Artificial Analysis IFBench","category":"instruction","score":75.9,"unit":"percent","family":"ifbench-family","index":59.95298505762419,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mcpAtlas","benchmarkName":"MCP Atlas","category":"agentic","score":75.3,"unit":"percent","family":"mcpAtlas","index":61.76897169581417,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:tax_eval_v2:overall","benchmarkName":"TaxEval v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":74.98,"unit":"percent","family":"vals-ai:tax_eval_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","measuredAt":"2026-09-01","stderr":0.854,"costPerTest":0.06042,"latency":95.83},{"benchmarkId":"vals-ai:terminal-bench-2:overall","benchmarkName":"Terminal-Bench 2.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":73.202,"unit":"percent","family":"vals-ai:terminal-bench-2:overall","index":70.62701264891805,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2","measuredAt":"2026-06-04","stderr":3.978,"costPerTest":0.760845,"latency":462.765},{"benchmarkId":"gertLabs","benchmarkName":"Gert Labs Composite Game Benchmark","category":"agentic","score":72.93,"unit":"percent","family":"gertLabs","index":74.59940212818346,"aboveFit":false,"run":null,"source":"Gert Labs","sourceUrl":"https://gertlabs.com/rankings","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tier-4-v2-private:overall","benchmarkName":"FrontierMath-Tier-4-v2-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":72.5,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tier-4-v2-private:overall","index":76.96993230970565,"aboveFit":false,"run":"xhigh effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":7.149950690165275,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-instruction-following:overall","benchmarkName":"LiveBench Instruction Following","category":"instruction","domain":null,"eligible":true,"ranked":true,"score":70.72925000000001,"unit":"percent","family":"livebench:livebench-instruction-following:overall","index":69.70297874226245,"aboveFit":false,"run":"xhigh effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=IF","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.2274965,"latency":null},{"benchmarkId":"livebench:livebench-design:overall","benchmarkName":"LiveBench Design","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":70.3,"unit":"percent","family":"livebench-2-agentic","index":70.00204755527832,"aboveFit":false,"run":"xhigh effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Design","measuredAt":"2026-10-14","stderr":null,"costPerTest":1.190356,"latency":null},{"benchmarkId":"vals-ai:vibe-code:overall","benchmarkName":"Vibe Code Bench v1.1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":69.847,"unit":"percent","family":"vals-ai:vibe-code:overall","index":67.45141756963429,"aboveFit":false,"run":"OpenHands","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","measuredAt":"2026-11-09","stderr":4.535,"costPerTest":16.661911,"latency":1911.567},{"benchmarkId":"vals-ai:mortgage_tax:overall","benchmarkName":"MortgageTax","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":68.76,"unit":"percent","family":"vals-ai:mortgage_tax:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mortgage_tax","measuredAt":"2026-09-01","stderr":0.912,"costPerTest":0.071407,"latency":28.243},{"benchmarkId":"vals-ai:corp_fin_v2:overall","benchmarkName":"CorpFin v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":68.415,"unit":"percent","family":"vals-ai:corp_fin_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","measuredAt":"2026-08-12","stderr":0.916,"costPerTest":0.43166,"latency":23.304},{"benchmarkId":"parseBenchLlamaIndex","benchmarkName":"ParseBench: document parsing for AI agents (LlamaIndex leaderboard)","category":"multimodal","score":66.42,"unit":"percent","family":"parseBenchLlamaIndex","index":66.39144309640623,"aboveFit":false,"run":null,"source":"Boyang Zhang, Sebastián G. Acosta, Preston Carlson, Sacha Bron, Pierre-Loïc Doulcet, Daniel B. Ospina and Simon Suo (LlamaIndex)","sourceUrl":"https://arxiv.org/abs/2604.08538","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:case_law_v2:overall","benchmarkName":"CaseLaw v2","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":66.238,"unit":"percent","family":"vals-ai:case_law_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/case_law_v2","measuredAt":"2026-05-04","stderr":0.322,"costPerTest":0.155011,"latency":25.2},{"benchmarkId":"vals-ai:emb:overall","benchmarkName":"EMB","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":64.543,"unit":"percent","family":"vals-ai:emb:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/emb","measuredAt":"2026-10-20","stderr":2.869,"costPerTest":3.273352,"latency":910.575},{"benchmarkId":"epoch-ai:epoch-simpleqa-verified:overall","benchmarkName":"SimpleQA Verified","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":63,"unit":"percent","family":"epoch-ai:epoch-simpleqa-verified:overall","index":76.40361384148811,"aboveFit":false,"run":"xhigh effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.5275252316519465,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-chess-puzzles:overall","benchmarkName":"Chess Puzzles","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":63,"unit":"percent","family":"epoch-ai:epoch-chess-puzzles:overall","index":81.802633019595,"aboveFit":false,"run":"xhigh effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":4.8523658709390975,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:skillsbench:overall","benchmarkName":"SkillsBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":62.209,"unit":"percent","family":"vals-ai:skillsbench:overall","index":71.44552452502461,"aboveFit":false,"run":"OpenHands","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/skillsbench","measuredAt":"2026-10-29","stderr":4.523,"costPerTest":2.54346,"latency":414.615},{"benchmarkId":"livebench:livebench-research:overall","benchmarkName":"LiveBench Research","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":61.6,"unit":"percent","family":"livebench-2-agentic","index":63.63946702947035,"aboveFit":false,"run":"xhigh effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Research","measuredAt":"2026-10-14","stderr":null,"costPerTest":17.137886363636365,"latency":null},{"benchmarkId":"vals-ai:public-benefits-bench:overall","benchmarkName":"Public Benefits Bench v1.1","category":null,"domain":"Public benefits","eligible":false,"ranked":true,"score":60.893,"unit":"percent","family":"vals-ai:public-benefits-bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/public-benefits-bench","measuredAt":"2026-10-12","stderr":1.269,"costPerTest":3.971165,"latency":2377.308},{"benchmarkId":"terminalBenchHard","benchmarkName":"Terminal-Bench Hard","category":"agentic","score":60.6,"unit":"percent","family":"terminalBenchHard","index":68.83239215957695,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/terminalbench-hard","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:finance_agent:overall","benchmarkName":"Finance Agent v1.1","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":59.963,"unit":"percent","family":"vals-ai:finance_agent:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/finance_agent","measuredAt":"2026-06-04","stderr":2.835,"costPerTest":1.32831,"latency":855.159},{"benchmarkId":"cursorBench31","benchmarkName":"cursorBench31","category":"coding","score":59.2,"unit":"percent","family":"cursorBench31","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"swePro","benchmarkName":"SWE-bench Pro","category":"coding","score":58.6,"unit":"percent","family":"swePro","index":58.743728571529324,"aboveFit":false,"run":null,"source":"Xiang Deng, Jeff Da, Edwin Pan, Yannis Yiming He, Charles Ide, Kanak Garg, Niklas Lauffer, Andrew Park, Nitin Pasari, Chetan Rane, Karmini Sampath, Maya Krishnan, Srivatsa Kundurthy, Sean Hendryx, Zifan Wang, Chen Bo Calvin Zhang, Noah Jacobson, Bing Liu, Brad Kenstler","sourceUrl":"https://arxiv.org/abs/2509.16941","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"cursorBench32","benchmarkName":"cursorBench32","category":"coding","score":58.4,"unit":"percent","family":"cursorBench32","index":63.8213014119234,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"omniscienceAccuracy","benchmarkName":"Artificial Analysis Omniscience Accuracy","category":"knowledge","score":58,"unit":"percent","family":"omniscienceAccuracy","index":83.78479754207768,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:public-benefits-bench-v1:overall","benchmarkName":"Public Benefits Bench v1","category":null,"domain":"Public benefits","eligible":false,"ranked":true,"score":57.24,"unit":"percent","family":"vals-ai:public-benefits-bench-v1:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/public-benefits-bench-v1","measuredAt":"2026-06-09","stderr":1.287,"costPerTest":2.018596,"latency":1580.527},{"benchmarkId":"epoch-ai:epoch-mystery-game-puzzles:overall","benchmarkName":"Mystery Game Puzzles","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":56.00000000000001,"unit":"percent","family":"epoch-ai:epoch-mystery-game-puzzles:overall","index":82.04823241755982,"aboveFit":false,"run":"xhigh effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":4.988876515698588,"costPerTest":null,"latency":null},{"benchmarkId":"aaSciCode","benchmarkName":"Artificial Analysis SciCode","category":"coding","score":55.8,"unit":"percent","family":"scicode-family","index":68.58576441245337,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/scicode","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"toolathlon","benchmarkName":"Toolathlon","category":"agentic","score":55.6,"unit":"percent","family":"toolathlon","index":67.17598906360567,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"officeQaPro","benchmarkName":"OfficeQA Pro","category":"multimodal","score":54.1,"unit":"percent","family":"officeQaPro","index":64.10863855367151,"aboveFit":false,"run":null,"source":"OfficeQA Pro authors","sourceUrl":"https://arxiv.org/abs/2603.08655","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-agentic-coding:overall","benchmarkName":"LiveBench Agentic Coding","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":53.989666666666665,"unit":"percent","family":"livebench:livebench-agentic-coding:overall","index":64.84131406120413,"aboveFit":false,"run":"xhigh effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Agentic%20Coding","measuredAt":"2026-06-25","stderr":null,"costPerTest":2.1797819444444446,"latency":null},{"benchmarkId":"hle","benchmarkName":"Humanity's Last Exam","category":"knowledge","score":52.2,"unit":"percent","family":"hle","index":76.75852560386875,"aboveFit":false,"run":null,"source":"Center for AI Safety, Scale AI, and thousands of expert contributors","sourceUrl":"https://lastexam.ai/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:fabv2:overall","benchmarkName":"Finance Agent (v2)","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":51.76,"unit":"percent","family":"vals-ai:fabv2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/fabv2","measuredAt":"2026-10-29","stderr":0.55,"costPerTest":4.149702,"latency":662.269},{"benchmarkId":"epoch-ai:epoch-frontiermath-2025-02-28-private:overall","benchmarkName":"FrontierMath-2025-02-28-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":51.7,"unit":"percent","family":"epoch-ai:epoch-frontiermath-2025-02-28-private:overall","index":75.55377664071112,"aboveFit":false,"run":"xhigh effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.9000000000000004,"costPerTest":null,"latency":null},{"benchmarkId":"aaAnalystAgent","benchmarkName":"Artificial Analysis AnalystAgent","category":"agentic","score":50,"unit":"percent","family":"aaAnalystAgent","index":72.01108090108372,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/aa-analyst-agent","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:medcode:overall","benchmarkName":"MedCode","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":49.1,"unit":"percent","family":"vals-ai:medcode:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medcode","measuredAt":"2026-10-29","stderr":2.188,"costPerTest":0.160759,"latency":159.546},{"benchmarkId":"livebench:livebench-browser:overall","benchmarkName":"LiveBench Browser Use","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":48,"unit":"percent","family":"livebench-2-agentic","index":53.693364138552134,"aboveFit":false,"run":"xhigh effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Browser","measuredAt":"2026-10-14","stderr":null,"costPerTest":5.62,"latency":null},{"benchmarkId":"aaEnterpriseOpsGym","benchmarkName":"Artificial Analysis EnterpriseOps-Gym","category":"agentic","score":46.6,"unit":"percent","family":"aaEnterpriseOpsGym","index":null,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/enterprise-ops-gym-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaItbench","benchmarkName":"Artificial Analysis ITBench-AA","category":"agentic","score":45.8,"unit":"percent","family":"aaItbench","index":66.06898840316519,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/itbench-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":45.8,"unit":"percent","family":"hle","index":70.33745203567806,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:code-migration:overall","benchmarkName":"Code Migration","category":"coding","domain":null,"eligible":true,"ranked":true,"score":45.161,"unit":"percent","family":"vals-ai:code-migration:overall","index":69.18047131628416,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/code-migration","measuredAt":"2026-10-29","stderr":4.164,"costPerTest":6.442253,"latency":1894.953},{"benchmarkId":"livebench:livebench-long-horizon:overall","benchmarkName":"LiveBench Long Agentic","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":44.6,"unit":"percent","family":"livebench-2-agentic","index":51.20683841582258,"aboveFit":false,"run":"xhigh effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Long%20Agentic","measuredAt":"2026-10-14","stderr":null,"costPerTest":13.851483333333332,"latency":null},{"benchmarkId":"tau2-bench:banking-knowledge:overall","benchmarkName":"τ²-bench Banking","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":44.58762886597938,"unit":"percent","family":"tau2-bench","index":32.37659178514244,"aboveFit":false,"run":"xhigh effort · Sierra","source":"Sierra Research","sourceUrl":"https://github.com/sierra-research/tau2-bench","measuredAt":"2026-10-20","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-furniture-assembly:overall","benchmarkName":"Furniture Assembly","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":44.166666666666664,"unit":"percent","family":"epoch-ai:epoch-furniture-assembly:overall","index":67.25298505902941,"aboveFit":false,"run":"xhigh effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":6.410142328406845,"costPerTest":null,"latency":null},{"benchmarkId":"frontierCode","benchmarkName":"FrontierCode 1.1 Main","category":"coding","score":43,"unit":"percent","family":"frontierCode","index":67.98454652816015,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"jobBench","benchmarkName":"JobBench","category":"agentic","score":42.7,"unit":"percent","family":"jobBench","index":60.11092568723948,"aboveFit":false,"run":null,"source":"Yuetai Li, Yichen Feng, Zhangchen Xu, Zixian Ma, Kaiyuan Zheng, Fengqing Jiang, Xinghua Sun, Rulin Shao, Zichen Chen, Yue Huang, Xinyang Han, Brian Lee, Kayla Xu, Shenglai Zeng, Hang Hua, Xiangliang Zhang, Basel Alomair, Ranjay Krishna, Luke Zettlemoyer, Pang Wei Koh, Bhaskar Ramasubramanian, Luyao Niu, Xiang Yue, Radha Poovendran","sourceUrl":"https://arxiv.org/abs/2605.26329","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gdpvalAaNormalized","benchmarkName":"GDPval-AA normalized","category":"agentic","score":42.5,"unit":"percent","family":"gdpvalAaNormalized","index":64.42028728909564,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hleNoTools","benchmarkName":"Humanity's Last Exam without tools","category":"knowledge","score":41.4,"unit":"percent","family":"hle","index":65.92296395754695,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:legal_research:overall","benchmarkName":"Legal Research Bench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":40.385,"unit":"percent","family":"vals-ai:legal_research:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_research","measuredAt":"2026-11-09","stderr":3.41,"costPerTest":7.402471,"latency":2265.105},{"benchmarkId":"aaTau3Banking","benchmarkName":"Artificial Analysis Tau3-Banking","category":"agentic","score":39,"unit":"percent","family":"aaTau3Banking","index":62.76845690273903,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/tau3-banking","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"apexAgentsAa","benchmarkName":"APEX-Agents-AA","category":"agentic","score":37.7,"unit":"percent","family":"apexAgentsAa","index":67.98364116337693,"aboveFit":false,"run":null,"source":"Artificial Analysis / Mercor","sourceUrl":"https://artificialanalysis.ai/evaluations/apex-agents-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-oeis-open-lite:overall","benchmarkName":"OEIS Open Lite","category":"math","domain":null,"eligible":true,"ranked":true,"score":36,"unit":"percent","family":"epoch-ai:epoch-oeis-open-lite:overall","index":null,"aboveFit":false,"run":"medium effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":4.824181513244217,"costPerTest":null,"latency":null},{"benchmarkId":"bioBasicsBench","benchmarkName":"BioBasicsBench: everyday wet-lab science","category":"reasoning","score":35.4,"unit":"percent","family":"bioBasicsBench","index":null,"aboveFit":false,"run":null,"source":"OpenAI, Ligo","sourceUrl":"https://openai.com/index/measuring-how-ai-agents-handle-wet-lab-science/","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-private:overall","benchmarkName":"FrontierMath-Tier-4-2025-07-01-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":35.4,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-private:overall","index":79.15752460665063,"aboveFit":false,"run":"xhigh effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":6.9,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-ebr-bench:overall","benchmarkName":"EBR-bench","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":34.28571428571428,"unit":"percent","family":"epoch-ai:epoch-ebr-bench:overall","index":68.16143180584898,"aboveFit":false,"run":"xhigh effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.9825390471106021,"costPerTest":null,"latency":null},{"benchmarkId":"postTrainBench11","benchmarkName":"PostTrainBench v1.1","category":"coding","score":27.23,"unit":"percent","family":"postTrainBench11","index":63.71863593718666,"aboveFit":false,"run":null,"source":"PostTrainBench team","sourceUrl":"https://posttrainbench.com/?version=v1.1","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":27.1,"unit":"percent","family":"critpt","index":89.31155389531276,"aboveFit":true,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-oeis-open:overall","benchmarkName":"OEIS Open","category":"math","domain":null,"eligible":true,"ranked":true,"score":26.21951219512195,"unit":"percent","family":"epoch-ai:epoch-oeis-open:overall","index":null,"aboveFit":false,"run":"medium effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.9849168516251945,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-mirrorcode:overall","benchmarkName":"MirrorCode","category":"coding","domain":null,"eligible":true,"ranked":true,"score":23.333333333333332,"unit":"percent","family":"epoch-ai:epoch-mirrorcode:overall","index":59.81297261588808,"aboveFit":false,"run":"xhigh effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":10.131411680782824,"costPerTest":null,"latency":null},{"benchmarkId":"frontierCode2","benchmarkName":"FrontierCode 2.0","category":"coding","score":22.6,"unit":"percent","family":"frontierCode2","index":68.78233763635576,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:tax_agent_bench:overall","benchmarkName":"Tax Agent Bench","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":21.762,"unit":"percent","family":"vals-ai:tax_agent_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench","measuredAt":"2026-11-09","stderr":2.978,"costPerTest":4.073277,"latency":951.811},{"benchmarkId":"apprenticeBench","benchmarkName":"ApprenticeBench: end-to-end computer use, continual learning, and long-horizon agency on a real accounts-payable job","category":"agentic","score":20,"unit":"percent","family":"apprenticeBench","index":68.3475536955693,"aboveFit":false,"run":null,"source":"NeoCognition","sourceUrl":"https://neocognition.io/blog/apprentice-bench/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"researchClawBench","benchmarkName":"ResearchClawBench","category":"agentic","score":17,"unit":"percent","family":"researchClawBench","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"exploitGym","benchmarkName":"ExploitGym","category":"agentic","score":13.4,"unit":"percent","family":"exploitGym","index":67.11710554071945,"aboveFit":false,"run":null,"source":"Zhun Wang, Nico Schiller, Hongwei Li, Srijiith Sesha Narayana, Milad Nasr, Nicholas Carlini, Xiangyu Qi, Eric Wallace, Elie Bursztein, Luca Invernizzi, Kurt Thomas, Yan Shoshitaishvili, Wenbo Guo, Jingxuan He, Thorsten Holz, Dawn Song","sourceUrl":"https://arxiv.org/abs/2605.11086","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"osWorld2","benchmarkName":"OSWorld 2.0","category":"agentic","score":13,"unit":"percent","family":"osWorld2","index":59.32745312531354,"aboveFit":false,"run":null,"source":"Mengqi Yuan, Zilong Zhou, Xinzhuang Xiong, Weiming Wu, Jiayang Sun, Jiamin Song, Kaiqian Cui, Bowen Wang, Haoyuan Wu, Yitong Li, Dunjie Lu, Haikong Lu, Qi Zhen, Xinyuan Wang, Jiaqi Deng, Yuhao Yang, Cheng Chen, Boyuan Zheng, Alex Su, Xiao Yu, Hao Zou, Saaket Agashe, Xing Han Lu, Manpreet Kaur, Zhengyang Qi, Vincent Sunn Chen, Frederic Sala, Dayiheng Liu, Junyang Lin, Zhou Yu, Yu Su, Siva Reddy, Xin Eric Wang, Peng Qi, Tianbao Xie, Tao Yu","sourceUrl":"https://arxiv.org/abs/2606.29537","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lmarena:agent-bash-recovery:overall","benchmarkName":"Agent Arena command recovery","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":9.54738256695622,"unit":"index","family":"agent-arena","index":74.38024653651614,"aboveFit":false,"run":"xhigh effort","source":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset","measuredAt":"2026-10-08","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:time_horizon_index:overall","benchmarkName":"Time Horizon Index: KSP","category":null,"domain":"Other","eligible":false,"ranked":true,"score":8.333,"unit":"percent","family":"vals-ai:time_horizon_index:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/time_horizon_index","measuredAt":"2026-10-07","stderr":0,"costPerTest":516.360641,"latency":0},{"benchmarkId":"vals-ai:srebench:overall","benchmarkName":"SRE Bench","category":null,"domain":"Other","eligible":false,"ranked":true,"score":3.817,"unit":"percent","family":"vals-ai:srebench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/srebench","measuredAt":"2026-11-05","stderr":1.186,"costPerTest":17.773692,"latency":2787.997},{"benchmarkId":"vals-ai:programbench:overall","benchmarkName":"ProgramBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":0.5,"unit":"percent","family":"vals-ai:programbench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/programbench","measuredAt":"2026-11-09","stderr":0.5,"costPerTest":6.95454,"latency":1322.959},{"benchmarkId":"arc-prize:v3-semi-private:overall","benchmarkName":"ARC-AGI-3 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":0.43,"unit":"percent","family":"arc-prize:v3-semi-private:overall","index":65.48827835901378,"aboveFit":false,"run":"high effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mazeBench","benchmarkName":"MazeBench: visual spatial reasoning in a 3D open world","category":"agentic","score":0,"unit":"percent","family":"mazeBench","index":61.97866734834783,"aboveFit":false,"run":null,"source":"Jonathan Pappas and David Pappas","sourceUrl":"https://mazebench.com/blog?post=maze-bench-results","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-erdos:overall","benchmarkName":"FrontierMath-Erdos","category":"math","domain":null,"eligible":true,"ranked":true,"score":0,"unit":"percent","family":"epoch-ai:epoch-frontiermath-erdos:overall","index":null,"aboveFit":false,"run":"xhigh effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":5,"outputPerMillion":30,"batch":{"inputPerMillion":2.5,"outputPerMillion":15},"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":52,"timeToFirstToken":2.7275,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":{"elo":1466.118516989652,"eloUpper":1469.7897925658694,"eloLower":1462.4472414134343,"votes":69937,"rank":36,"rankUpper":null,"rankLower":null,"arenaName":"GPT-5.5","organization":"openai","updatedAt":"2026-11-09"},"quality":{"coveragePercent":95,"benchmarkCount":68,"categoryCount":7,"confidence":"high","state":"available"},"hosts":[{"host":"databricks","name":"Databricks","kind":"cloud","url":"https://docs.databricks.com/aws/en/machine-learning/foundation-models/","inputPerMillion":5,"outputPerMillion":30,"cacheReadPerMillion":0.5,"contextTokens":1050000,"maxOutputTokens":128000,"deprecated":false},{"host":"digitalocean","name":"DigitalOcean","kind":"cloud","url":"https://docs.digitalocean.com/products/gradient-ai-platform/details/models/","inputPerMillion":5,"outputPerMillion":30,"cacheReadPerMillion":0.5,"contextTokens":1000000,"maxOutputTokens":128000,"deprecated":false},{"host":"gmi-cloud","name":"GMI Cloud","kind":"host","url":"https://docs.gmicloud.ai/inference-engine/api-reference/llm-api-reference","inputPerMillion":5,"outputPerMillion":30,"cacheReadPerMillion":0.5,"contextTokens":1050000,"maxOutputTokens":128000,"deprecated":false},{"host":"azure","name":"Microsoft Azure","kind":"cloud","url":"https://learn.microsoft.com/en-us/azure/ai-services/openai/concepts/models","inputPerMillion":5,"outputPerMillion":30,"cacheReadPerMillion":0.5,"contextTokens":1050000,"maxOutputTokens":128000,"deprecated":false},{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":5,"outputPerMillion":30,"cacheReadPerMillion":0.5,"contextTokens":1050000,"maxOutputTokens":128000,"deprecated":false},{"host":"sap-ai-core","name":"SAP AI Core","kind":"cloud","url":"https://help.sap.com/docs/sap-ai-core","inputPerMillion":5,"outputPerMillion":30,"cacheReadPerMillion":0.5,"contextTokens":1050000,"maxOutputTokens":128000,"deprecated":false},{"host":"snowflake","name":"Snowflake Cortex","kind":"cloud","url":"https://docs.snowflake.com/en/user-guide/snowflake-cortex/cortex-rest-api","inputPerMillion":5,"outputPerMillion":30,"cacheReadPerMillion":0.5,"contextTokens":1050000,"maxOutputTokens":128000,"deprecated":false},{"host":"amazon-bedrock","name":"Amazon Bedrock","kind":"cloud","url":"https://docs.aws.amazon.com/bedrock/latest/userguide/models-supported.html","inputPerMillion":5.5,"outputPerMillion":33,"cacheReadPerMillion":0.55,"contextTokens":1000000,"maxOutputTokens":128000,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:livebench","name":"LiveBench","url":"https://livebench.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:arc-prize-foundation","name":"ARC Prize Foundation","url":"https://arcprize.org","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:sierra-research","name":"Sierra Research","url":"https://github.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:lmarena","name":"LMArena","url":"https://huggingface.co","license":"CC BY 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gpt-5-6-terra","providerId":"openai","slug":"gpt-5-6-terra","name":"GPT-5.6 Terra","provider":"OpenAI","description":"GPT-5.6 Terra is a reasoning model from OpenAI in the GPT-5.6 family. 66 benchmarks count toward its index, in 8 categories.","logoUrl":null,"releaseDate":"2026-07-09","contextWindowTokens":1050000,"modalities":["file","image","text"],"openWeights":false,"details":{"apiId":"openai/gpt-5.6-terra","maxOutputTokens":128000,"knowledgeCutoff":"2026-02-16","retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.19999999999999998,"weightsUrl":null},"reasoning":true,"overallScore":68.86275547289002,"overallScoreRange":{"low":66.8723660637542,"high":70.85314488202583},"badges":[],"capabilityScores":{"agentic":68.20170855087576,"coding":68.44562548518739,"reasoning":72.54753285896183,"multimodal":67.32295623927907,"knowledge":65.47526499698148,"multilingual":60.66439108168465,"instruction":56.447429792049775,"math":72.8219262605474},"capabilityCounts":{"math":5,"reasoning":9,"coding":16,"knowledge":8,"multimodal":3,"agentic":22,"instruction":2,"multilingual":1},"benchmarks":[{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":99.72222222222223,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":66.93688599272527,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0.2777777777777778,"costPerTest":null,"latency":null},{"benchmarkId":"arc-prize:v1-public-eval:overall","benchmarkName":"ARC-AGI-1 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":98.25,"unit":"percent","family":"arc-prize:v1-public-eval:overall","index":null,"aboveFit":false,"run":"max effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.32,"latency":null},{"benchmarkId":"arc-prize:v1-semi-private:overall","benchmarkName":"ARC-AGI-1 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":96.5,"unit":"percent","family":"arc-prize:v1-semi-private:overall","index":71.35189279722005,"aboveFit":false,"run":"max effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.55,"latency":null},{"benchmarkId":"vals-ai:swebench:overall","benchmarkName":"SWE-bench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":95.4,"unit":"percent","family":"swe-bench-verified","index":71.59092045003587,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/swebench","measuredAt":"2026-09-01","stderr":0.938,"costPerTest":0.400788,"latency":180.284},{"benchmarkId":"livebench:livebench-math:overall","benchmarkName":"LiveBench Mathematics","category":"math","domain":null,"eligible":true,"ranked":true,"score":94.90825000000001,"unit":"percent","family":"livebench:livebench-math:overall","index":72.05028338942266,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Mathematics","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.1321815303430079,"latency":null},{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":93.3080808080808,"unit":"percent","family":"gpqa-diamond","index":62.788722804228456,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.5363672147634115,"costPerTest":null,"latency":null},{"benchmarkId":"gpqa","benchmarkName":"Graduate-Level Google-Proof Q&A","category":"knowledge","score":92.9,"unit":"percent","family":"gpqa-diamond","index":62.41444223789757,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gpqaDiamond","benchmarkName":"GPQA Diamond","category":"knowledge","score":92.9,"unit":"percent","family":"gpqa-diamond","index":62.41444223789757,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":92.5,"unit":"percent","family":"gpqa-diamond","index":62.04757316792964,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"extractBenchLlamaIndex","benchmarkName":"ExtractBench: schema-guided enterprise document extraction (LlamaIndex leaderboard)","category":"multimodal","score":92.26,"unit":"percent","family":"extractBenchLlamaIndex","index":74.50761719436824,"aboveFit":false,"run":null,"source":"Boyang Zhang, Adrian Lyjak, Eli Stewart, Zhaoqi Li and Simon Suo (LlamaIndex)","sourceUrl":"https://arxiv.org/abs/2607.29677","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"arc-prize:v2-public-eval:overall","benchmarkName":"ARC-AGI-2 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":91.32000000000001,"unit":"percent","family":"arc-prize:v2-public-eval:overall","index":null,"aboveFit":false,"run":"max effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":1.18,"latency":null},{"benchmarkId":"vals-ai:gpqa:overall","benchmarkName":"GPQA Diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":90.909,"unit":"percent","family":"gpqa-diamond","index":60.58835144213227,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","measuredAt":"2026-09-01","stderr":1.906,"costPerTest":0.033997,"latency":36.764},{"benchmarkId":"livebench:livebench-reasoning:overall","benchmarkName":"LiveBench Reasoning","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":90.6345,"unit":"percent","family":"livebench","index":79.03073385916449,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Reasoning","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.23393861386138615,"latency":null},{"benchmarkId":"aaTerminalBench21","benchmarkName":"Artificial Analysis Terminal-Bench v2.1","category":"agentic","score":88,"unit":"percent","family":"aaTerminalBench21","index":70.64945825748858,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/terminalbench-v2-1","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:ioi:overall","benchmarkName":"IOI","category":"coding","domain":null,"eligible":true,"ranked":true,"score":87.611,"unit":"percent","family":"vals-ai:ioi:overall","index":79.00776612967158,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/ioi","measuredAt":"2026-10-29","stderr":6.531,"costPerTest":8.664009,"latency":4992.838},{"benchmarkId":"browseComp","benchmarkName":"BrowseComp","category":"agentic","score":87.5,"unit":"percent","family":"browseComp","index":69.03572707553812,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"terminalBench21","benchmarkName":"Terminal-Bench 2.1","category":"agentic","score":87.4,"unit":"percent","family":"terminal-bench-2-1","index":72.25748006229325,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:sage:overall","benchmarkName":"SAGE","category":null,"domain":"Education","eligible":false,"ranked":true,"score":87.038,"unit":"percent","family":"vals-ai:sage:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/sage","measuredAt":"2026-11-05","stderr":1.997,"costPerTest":0.110435,"latency":42.919},{"benchmarkId":"vulcanBench","benchmarkName":"VulcanBench v3","category":"coding","score":87,"unit":"percent","family":"vulcanBench","index":68.4175418120868,"aboveFit":false,"run":null,"source":"VulcanBench contributors","sourceUrl":"https://github.com/morganlinton/VulcanBench/tree/main","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:mmlu_pro:overall","benchmarkName":"MMLU Pro","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":86.659,"unit":"percent","family":"mmlu-pro","index":53.66069561128655,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","measuredAt":"2026-09-01","stderr":0.333,"costPerTest":0.008039,"latency":8.173},{"benchmarkId":"vals-ai:mmmu:overall","benchmarkName":"MMMU Pro","category":"multimodal","domain":null,"eligible":true,"ranked":true,"score":86.474,"unit":"percent","family":"mmmu-pro","index":67.69113924175036,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmmu","measuredAt":"2026-09-01","stderr":0.822,"costPerTest":0.016375,"latency":13.033},{"benchmarkId":"tau2Bench","benchmarkName":"τ²-Bench Tool-Agent-User Evaluation","category":"agentic","score":86.3,"unit":"percent","family":"tau2-bench","index":60.05579970417346,"aboveFit":false,"run":null,"source":"Victor Barres, Honghua Dong, Soham Ray, Xujie Si, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2506.07982","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-coding-2:overall","benchmarkName":"LiveBench Coding 2.0","category":"coding","domain":null,"eligible":true,"ranked":true,"score":86.2,"unit":"percent","family":"livebench:livebench-coding-2:overall","index":70.60874043029663,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Coding","measuredAt":"2026-10-14","stderr":null,"costPerTest":2.7147310344827584,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tiers-1-3-v2-private:overall","benchmarkName":"FrontierMath-Tiers-1-3-v2-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":85.96491228070175,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tiers-1-3-v2-private:overall","index":76.14247499591445,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.0611471958164294,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:lcb:overall","benchmarkName":"LiveCodeBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":85.93,"unit":"percent","family":"livecodebench","index":60.64343031170416,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","measuredAt":"2026-09-01","stderr":1.017,"costPerTest":0.031103,"latency":44.331},{"benchmarkId":"vals-ai:legal_bench:overall","benchmarkName":"LegalBench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":85.105,"unit":"percent","family":"vals-ai:legal_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","measuredAt":"2026-10-13","stderr":0.45,"costPerTest":0.002643,"latency":2.782},{"benchmarkId":"livebench:livebench-browser:overall","benchmarkName":"LiveBench Browser Use","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":84,"unit":"percent","family":"livebench-2-agentic","index":80.02128355568858,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Browser","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.847168,"latency":null},{"benchmarkId":"arc-prize:v2-semi-private:overall","benchmarkName":"ARC-AGI-2 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":83.89999999999999,"unit":"percent","family":"arc-prize:v2-semi-private:overall","index":77.70991681414203,"aboveFit":false,"run":"max effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":1.09,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":83,"unit":"percent","family":"lcr","index":64.2427330827805,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-language:overall","benchmarkName":"LiveBench Language","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":82.89266666666667,"unit":"percent","family":"livebench:livebench-language:overall","index":68.29456646340041,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Language","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.173422,"latency":null},{"benchmarkId":"vals-ai:medscribe:overall","benchmarkName":"MedScribe","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":82.869,"unit":"percent","family":"vals-ai:medscribe:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medscribe","measuredAt":"2026-10-29","stderr":1.948,"costPerTest":0.062588,"latency":35.526},{"benchmarkId":"mmmuProPython","benchmarkName":"MMMU-Pro with Python","category":"multimodal","score":82,"unit":"percent","family":"mmmuProPython","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"cyberGym","benchmarkName":"CyberGym","category":"agentic","score":81.8,"unit":"percent","family":"cyberGym","index":67.81910808314663,"aboveFit":false,"run":null,"source":"Zhun Wang, Tianneng Shi, Jingxuan He, Matthew Cai, Jialin Zhang, Dawn Song","sourceUrl":"https://www.cybergym.io/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"labBench2","benchmarkName":"LABBench2: An Improved Benchmark for AI Systems Performing Biology Research","category":"knowledge","score":81.2,"unit":"percent","family":"labBench2","index":null,"aboveFit":false,"run":null,"source":"Jon M. Laurent, Albert Bou, Michael Pieler, Conor Igoe, Alex Andonian, Siddharth Narayanan, James Braza, Alexandros Sanchez Vassopoulos, Jacob L. Steenwyk, Blake Lash, Andrew D. White, Samuel G. Rodriques","sourceUrl":"https://arxiv.org/abs/2604.09554","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmmuPro","benchmarkName":"Massive Multi-discipline Multimodal Understanding Pro","category":"multimodal","score":80.7,"unit":"percent","family":"mmmu-pro","index":60.17678482295231,"aboveFit":false,"run":null,"source":"MMMU-Pro authors","sourceUrl":"https://arxiv.org/abs/2409.02813","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaMmmuPro","benchmarkName":"Artificial Analysis MMMU-Pro","category":"multimodal","score":80.7,"unit":"percent","family":"mmmu-pro","index":60.17678482295231,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmmu-pro","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-data-analysis:overall","benchmarkName":"LiveBench Data Analysis","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":79.30666666666667,"unit":"percent","family":"livebench","index":64.35627345648231,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Data%20Analysis","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.27776133333333336,"latency":null},{"benchmarkId":"livebench:livebench-coding:overall","benchmarkName":"LiveBench Coding","category":"coding","domain":null,"eligible":true,"ranked":true,"score":78.24549999999999,"unit":"percent","family":"livebench:livebench-coding:overall","index":64.86937815421757,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Coding","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.16919829059829058,"latency":null},{"benchmarkId":"vals-ai:terminal-bench-2-1:overall","benchmarkName":"Terminal-Bench 2.1","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":77.528,"unit":"percent","family":"terminal-bench-2-1","index":66.48378790421603,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2-1","measuredAt":"2026-10-01","stderr":2.247,"costPerTest":0.472552,"latency":519.739},{"benchmarkId":"vals-ai:tax_eval_v2:overall","benchmarkName":"TaxEval v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":76.166,"unit":"percent","family":"vals-ai:tax_eval_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","measuredAt":"2026-09-01","stderr":0.836,"costPerTest":0.035909,"latency":37.28},{"benchmarkId":"livebench:livebench-design:overall","benchmarkName":"LiveBench Design","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":76,"unit":"percent","family":"livebench-2-agentic","index":74.17063479632493,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Design","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.620068,"latency":null},{"benchmarkId":"vals-ai:vibe-code:overall","benchmarkName":"Vibe Code Bench v1.1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":74.594,"unit":"percent","family":"vals-ai:vibe-code:overall","index":69.16420975059782,"aboveFit":false,"run":"OpenHands","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","measuredAt":"2026-11-09","stderr":4.203,"costPerTest":7.887015,"latency":1152.433},{"benchmarkId":"vals-ai:proof_bench:overall","benchmarkName":"ProofBench v1.1","category":"math","domain":null,"eligible":true,"ranked":true,"score":74,"unit":"percent","family":"vals-ai:proof_bench:overall","index":72.70472846405326,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/proof_bench","measuredAt":"2026-11-09","stderr":4.408,"costPerTest":1.274209,"latency":777.341},{"benchmarkId":"vals-ai:cyber:overall","benchmarkName":"CyberBench v1.1","category":null,"domain":"Other","eligible":false,"ranked":true,"score":72.084,"unit":"percent","family":"vals-ai:cyber:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/cyber","measuredAt":"2026-11-05","stderr":5.408,"costPerTest":3.308232,"latency":1023.932},{"benchmarkId":"aaIfBench","benchmarkName":"Artificial Analysis IFBench","category":"instruction","score":71.2,"unit":"percent","family":"ifbench-family","index":55.56078516792552,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tier-4-v2-private:overall","benchmarkName":"FrontierMath-Tier-4-v2-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":70.73170731707317,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tier-4-v2-private:overall","index":76.27525846062139,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":7.194088392074452,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":70.575,"unit":"percent","family":"euroeval","index":67.97427161364845,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":0.4808326112068523,"costPerTest":null,"latency":null},{"benchmarkId":"deepSwe","benchmarkName":"DeepSWE","category":"agentic","score":69.6,"unit":"percent","family":"deepSwe","index":71.6739776788454,"aboveFit":false,"run":null,"source":"Datacurve AI","sourceUrl":"https://deepswe.datacurve.ai/blog","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-french:overall","benchmarkName":"EuroEval French","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":68.87125,"unit":"percent","family":"euroeval","index":65.66267488535489,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/french","measuredAt":null,"stderr":0.4304321955895028,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-italian:overall","benchmarkName":"EuroEval Italian","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":67.825,"unit":"percent","family":"euroeval","index":64.24315437062141,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/italian","measuredAt":null,"stderr":0.37845078940332516,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:mortgage_tax:overall","benchmarkName":"MortgageTax","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":67.329,"unit":"percent","family":"vals-ai:mortgage_tax:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mortgage_tax","measuredAt":"2026-09-01","stderr":0.925,"costPerTest":0.022706,"latency":6.316},{"benchmarkId":"parseBenchLlamaIndex","benchmarkName":"ParseBench: document parsing for AI agents (LlamaIndex leaderboard)","category":"multimodal","score":67.06,"unit":"percent","family":"parseBenchLlamaIndex","index":67.28446670051665,"aboveFit":false,"run":null,"source":"Boyang Zhang, Sebastián G. Acosta, Preston Carlson, Sacha Bron, Pierre-Loïc Doulcet, Daniel B. Ospina and Simon Suo (LlamaIndex)","sourceUrl":"https://arxiv.org/abs/2604.08538","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:emb:overall","benchmarkName":"EMB","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":66.205,"unit":"percent","family":"vals-ai:emb:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/emb","measuredAt":"2026-10-20","stderr":3.11,"costPerTest":2.193037,"latency":813.024},{"benchmarkId":"euroeval:euroeval-portuguese:overall","benchmarkName":"EuroEval Portuguese","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":65.39125000000001,"unit":"percent","family":"euroeval","index":60.941115610542504,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/portuguese","measuredAt":null,"stderr":0.49018172650558894,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:corp_fin_v2:overall","benchmarkName":"CorpFin v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":65.307,"unit":"percent","family":"vals-ai:corp_fin_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","measuredAt":"2026-08-12","stderr":0.938,"costPerTest":0.171459,"latency":7.717},{"benchmarkId":"vals-ai:ioi-v1:overall","benchmarkName":"IOI v1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":65.25,"unit":"percent","family":"vals-ai:ioi-v1:overall","index":72.60598831604867,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/ioi-v1","measuredAt":"2026-08-09","stderr":0.917,"costPerTest":0.97448,"latency":1330.749},{"benchmarkId":"euroeval:euroeval-dutch:overall","benchmarkName":"EuroEval Dutch","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":64.98333333333332,"unit":"percent","family":"euroeval","index":60.387666552826786,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/dutch","measuredAt":null,"stderr":0.4886741952833222,"costPerTest":null,"latency":null},{"benchmarkId":"cursorBench32","benchmarkName":"cursorBench32","category":"coding","score":64.9,"unit":"percent","family":"cursorBench32","index":69.16306441717876,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-instruction-following:overall","benchmarkName":"LiveBench Instruction Following","category":"instruction","domain":null,"eligible":true,"ranked":true,"score":64.6165,"unit":"percent","family":"livebench:livebench-instruction-following:overall","index":57.334074416174026,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=IF","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.331823,"latency":null},{"benchmarkId":"swePro","benchmarkName":"SWE-bench Pro","category":"coding","score":63.4,"unit":"percent","family":"swePro","index":62.83145648123164,"aboveFit":false,"run":null,"source":"Xiang Deng, Jeff Da, Edwin Pan, Yannis Yiming He, Charles Ide, Kanak Garg, Niklas Lauffer, Andrew Park, Nitin Pasari, Chetan Rane, Karmini Sampath, Maya Krishnan, Srivatsa Kundurthy, Sean Hendryx, Zifan Wang, Chen Bo Calvin Zhang, Noah Jacobson, Bing Liu, Brad Kenstler","sourceUrl":"https://arxiv.org/abs/2509.16941","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:public-benefits-bench:overall","benchmarkName":"Public Benefits Bench v1.1","category":null,"domain":"Public benefits","eligible":false,"ranked":true,"score":62.382,"unit":"percent","family":"vals-ai:public-benefits-bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/public-benefits-bench","measuredAt":"2026-10-12","stderr":1.26,"costPerTest":1.199723,"latency":1171.486},{"benchmarkId":"euroeval:euroeval-spanish:overall","benchmarkName":"EuroEval Spanish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":62.114999999999995,"unit":"percent","family":"euroeval","index":56.49599820419072,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/spanish","measuredAt":null,"stderr":0.4125227266466661,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-research:overall","benchmarkName":"LiveBench Research","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":60.4,"unit":"percent","family":"livebench-2-agentic","index":62.7618697155658,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Research","measuredAt":"2026-10-14","stderr":null,"costPerTest":7.7186318181818185,"latency":null},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":59.73,"unit":"percent","family":"euroeval","index":53.260101977056365,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.5807780341920655,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":58.95857142857143,"unit":"percent","family":"euroeval","index":52.21345090628515,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.4921008690361382,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:skillsbench:overall","benchmarkName":"SkillsBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":58.895,"unit":"percent","family":"vals-ai:skillsbench:overall","index":68.9790065614874,"aboveFit":false,"run":"OpenHands","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/skillsbench","measuredAt":"2026-10-29","stderr":4.472,"costPerTest":1.75247,"latency":415.068},{"benchmarkId":"healthBenchProfessional","benchmarkName":"HealthBench Professional","category":"knowledge","score":57.7,"unit":"percent","family":"healthBenchProfessional","index":70.04817912525553,"aboveFit":false,"run":null,"source":"Rebecca Soskin Hicks, Mikhail Trofimov, Dominick Lim, Rahul K. Arora, Foivos Tsimpourlas, Preston Bowman, Michael Sharman, Chi Tong, Kavin Karthik, Arnav Dugar, Akshay Jagadeesh, Khaled Saab, Johannes Heidecke, Ashley Alexander, Nate Gross, Karan Singhal","sourceUrl":"https://arxiv.org/abs/2604.27470","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"frontierCode11Extended","benchmarkName":"FrontierCode 1.1 Extended","category":"coding","score":55.8,"unit":"percent","family":"frontierCode11Extended","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaSciCode","benchmarkName":"Artificial Analysis SciCode","category":"coding","score":55,"unit":"percent","family":"scicode-family","index":67.54281149532159,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/scicode","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-agentic-coding:overall","benchmarkName":"LiveBench Agentic Coding","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":54.949666666666666,"unit":"percent","family":"livebench:livebench-agentic-coding:overall","index":65.83134571079079,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Agentic%20Coding","measuredAt":"2026-06-25","stderr":null,"costPerTest":1.3491166666666667,"latency":null},{"benchmarkId":"vals-ai:fabv2:overall","benchmarkName":"Finance Agent (v2)","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":54.436,"unit":"percent","family":"vals-ai:fabv2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/fabv2","measuredAt":"2026-10-29","stderr":2.071,"costPerTest":3.629197,"latency":1449.852},{"benchmarkId":"epoch-ai:epoch-furniture-assembly:overall","benchmarkName":"Furniture Assembly","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":54.166666666666664,"unit":"percent","family":"epoch-ai:epoch-furniture-assembly:overall","index":71.91474680693977,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":6.432138878020231,"costPerTest":null,"latency":null},{"benchmarkId":"toolathlon","benchmarkName":"Toolathlon","category":"agentic","score":53.1,"unit":"percent","family":"toolathlon","index":65.16423264321784,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hleVerified","benchmarkName":"HLE-Verified","category":"knowledge","score":51.1,"unit":"percent","family":"hleVerified","index":null,"aboveFit":false,"run":null,"source":"Weiqi Zhai, Zhihai Wang, Jinghang Wang, Boyu Yang, Xiaogang Li, Xiang Xu, Bohan Wang, Peng Wang, Xingzhe Wu, Anfeng Li, Qiyuan Feng, Yuhao Zhou, Shoulin Han, Wenjie Luo, Yiyuan Li, Yaxuan Wang, Ruixian Luo, Guojie Lin, Peiyao Xiao, Chengliang Xu, Ben Wang, Zeyu Wang, Zichao Chen, Jianan Ye, Yijie Hu, Jialong Chen, Zongwen Shen, Yuliang Xu, An Yang, Bowen Yu, Dayiheng Liu, Junyang Lin, Hu Wei, Que Shen, Bing Zhao","sourceUrl":"https://arxiv.org/abs/2602.13964","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaItbench","benchmarkName":"Artificial Analysis ITBench-AA","category":"agentic","score":51,"unit":"percent","family":"aaItbench","index":73.19213915256054,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/itbench-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"osWorld2","benchmarkName":"OSWorld 2.0","category":"agentic","score":50.2,"unit":"percent","family":"osWorld2","index":74.36859016078488,"aboveFit":false,"run":null,"source":"Mengqi Yuan, Zilong Zhou, Xinzhuang Xiong, Weiming Wu, Jiayang Sun, Jiamin Song, Kaiqian Cui, Bowen Wang, Haoyuan Wu, Yitong Li, Dunjie Lu, Haikong Lu, Qi Zhen, Xinyuan Wang, Jiaqi Deng, Yuhao Yang, Cheng Chen, Boyuan Zheng, Alex Su, Xiao Yu, Hao Zou, Saaket Agashe, Xing Han Lu, Manpreet Kaur, Zhengyang Qi, Vincent Sunn Chen, Frederic Sala, Dayiheng Liu, Junyang Lin, Zhou Yu, Yu Su, Siva Reddy, Xin Eric Wang, Peng Qi, Tianbao Xie, Tao Yu","sourceUrl":"https://arxiv.org/abs/2606.29537","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-long-horizon:overall","benchmarkName":"LiveBench Long Agentic","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":48.4,"unit":"percent","family":"livebench-2-agentic","index":53.98589657652032,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Long%20Agentic","measuredAt":"2026-10-14","stderr":null,"costPerTest":11.756905555555555,"latency":null},{"benchmarkId":"vals-ai:code-migration:overall","benchmarkName":"Code Migration","category":"coding","domain":null,"eligible":true,"ranked":true,"score":47.804,"unit":"percent","family":"vals-ai:code-migration:overall","index":70.58453888393699,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/code-migration","measuredAt":"2026-10-29","stderr":4.28,"costPerTest":8.127254,"latency":2914.132},{"benchmarkId":"gdpvalAaNormalized","benchmarkName":"GDPval-AA normalized","category":"agentic","score":47.6,"unit":"percent","family":"gdpvalAaNormalized","index":68.06666877372399,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-chess-puzzles:overall","benchmarkName":"Chess Puzzles","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":47,"unit":"percent","family":"epoch-ai:epoch-chess-puzzles:overall","index":69.65838989719346,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":5.016135580465917,"costPerTest":null,"latency":null},{"benchmarkId":"omniscienceAccuracy","benchmarkName":"Artificial Analysis Omniscience Accuracy","category":"knowledge","score":46.8,"unit":"percent","family":"omniscienceAccuracy","index":69.66993052061088,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:medcode:overall","benchmarkName":"MedCode","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":43.414,"unit":"percent","family":"vals-ai:medcode:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medcode","measuredAt":"2026-10-29","stderr":2.173,"costPerTest":0.046558,"latency":18.411},{"benchmarkId":"epoch-ai:epoch-simpleqa-verified:overall","benchmarkName":"SimpleQA Verified","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":43.2,"unit":"percent","family":"epoch-ai:epoch-simpleqa-verified:overall","index":59.90846593068676,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.5672320237336324,"costPerTest":null,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":42.9,"unit":"percent","family":"hle","index":67.42790307509165,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:legal_research:overall","benchmarkName":"Legal Research Bench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":41.346,"unit":"percent","family":"vals-ai:legal_research:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_research","measuredAt":"2026-11-09","stderr":3.423,"costPerTest":7.907299,"latency":4172.869},{"benchmarkId":"cursorBench40","benchmarkName":"cursorBench40","category":"coding","score":41.3,"unit":"percent","family":"cursorBench40","index":68.01780679226098,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-mirrorcode:overall","benchmarkName":"MirrorCode","category":"coding","domain":null,"eligible":true,"ranked":true,"score":41.11111111111111,"unit":"percent","family":"epoch-ai:epoch-mirrorcode:overall","index":67.62045997253124,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":12.372809695177825,"costPerTest":null,"latency":null},{"benchmarkId":"aaTau3Banking","benchmarkName":"Artificial Analysis Tau3-Banking","category":"agentic","score":40.2,"unit":"percent","family":"aaTau3Banking","index":64.25329882479917,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/tau3-banking","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"apexAgentsAa","benchmarkName":"APEX-Agents-AA","category":"agentic","score":38.9,"unit":"percent","family":"apexAgentsAa","index":68.80829390821867,"aboveFit":false,"run":null,"source":"Artificial Analysis / Mercor","sourceUrl":"https://artificialanalysis.ai/evaluations/apex-agents-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-mystery-game-puzzles:overall","benchmarkName":"Mystery Game Puzzles","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":35,"unit":"percent","family":"epoch-ai:epoch-mystery-game-puzzles:overall","index":65.7855195867512,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":4.793724854411022,"costPerTest":null,"latency":null},{"benchmarkId":"healthBenchHard","benchmarkName":"HealthBench Hard","category":"knowledge","score":32.7,"unit":"percent","family":"healthBenchHard","index":72.37793701162254,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:tax_agent_bench:overall","benchmarkName":"Tax Agent Bench","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":30.035,"unit":"percent","family":"vals-ai:tax_agent_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench","measuredAt":"2026-11-09","stderr":3.307,"costPerTest":4.829225,"latency":3630.72},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":30,"unit":"percent","family":"critpt","index":94.92561551373953,"aboveFit":true,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaDeepsecBench","benchmarkName":"Artificial Analysis DeepsecBench-AA","category":"agentic","score":25.1,"unit":"percent","family":"aaDeepsecBench","index":69.59211195708964,"aboveFit":false,"run":null,"source":"Artificial Analysis; dataset by Vercel","sourceUrl":"https://artificialanalysis.ai/evaluations/deepsecbench-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"exploitGym","benchmarkName":"ExploitGym","category":"agentic","score":23.2,"unit":"percent","family":"exploitGym","index":73.06391299288816,"aboveFit":false,"run":null,"source":"Zhun Wang, Nico Schiller, Hongwei Li, Srijiith Sesha Narayana, Milad Nasr, Nicholas Carlini, Xiangyu Qi, Eric Wallace, Elie Bursztein, Luca Invernizzi, Kurt Thomas, Yan Shoshitaishvili, Wenbo Guo, Jingxuan He, Thorsten Holz, Dawn Song","sourceUrl":"https://arxiv.org/abs/2605.11086","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:terminal-bench-4:overall","benchmarkName":"Terminal-Bench 4.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":22.727,"unit":"percent","family":"terminal-bench-4","index":65.30767104211206,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-4","measuredAt":"2026-11-09","stderr":2.314,"costPerTest":5.602118,"latency":1888.568},{"benchmarkId":"terminal-bench:terminal-bench-4-0-0:overall","benchmarkName":"Terminal-Bench 4.0.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":21.52,"unit":"percent","family":"terminal-bench-4","index":64.73454081398951,"aboveFit":false,"run":"max effort · Codex","source":"Terminal-Bench","sourceUrl":"https://www.tbench.ai/leaderboard","measuredAt":"2026-09-29","stderr":1.6581632653061225,"costPerTest":5.2530909090909095,"latency":2514.6},{"benchmarkId":"terminal-bench:terminal-bench-4-1-0:overall","benchmarkName":"Terminal-Bench 4.1.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":21.52,"unit":"percent","family":"terminal-bench:terminal-bench-4-1-0:overall","index":63.01346990309695,"aboveFit":false,"run":"max effort · Codex","source":"Terminal-Bench","sourceUrl":"https://www.tbench.ai/leaderboard","measuredAt":"2026-09-29","stderr":1.6581632653061225,"costPerTest":5.2530909090909095,"latency":2514.6},{"benchmarkId":"frontierBench","benchmarkName":"Terminal-Bench 3.0","category":"agentic","score":20.81,"unit":"percent","family":"frontierBench","index":67.66959288922104,"aboveFit":false,"run":null,"source":"Ryan Marten, Alex Shaw, Andy Konwinski, Harbor, and the Laude Institute","sourceUrl":"https://www.frontierbench.ai/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"apprenticeBench","benchmarkName":"ApprenticeBench: end-to-end computer use, continual learning, and long-horizon agency on a real accounts-payable job","category":"agentic","score":16,"unit":"percent","family":"apprenticeBench","index":66.3197592285453,"aboveFit":false,"run":null,"source":"NeoCognition","sourceUrl":"https://neocognition.io/blog/apprentice-bench/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:vcb-1-100:overall","benchmarkName":"Vibe Code Bench 1-100","category":"coding","domain":null,"eligible":true,"ranked":true,"score":14.822,"unit":"percent","family":"vals-ai:vcb-1-100:overall","index":63.48288780439073,"aboveFit":false,"run":"OpenHands","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vcb-1-100","measuredAt":"2026-10-29","stderr":3.123,"costPerTest":33.326696,"latency":6536.122},{"benchmarkId":"vals-ai:terminal-bench-science:overall","benchmarkName":"Terminal-Bench-Science","category":null,"domain":"Science","eligible":false,"ranked":true,"score":10,"unit":"percent","family":"vals-ai:terminal-bench-science:overall","index":null,"aboveFit":false,"run":"Mini-SWE-agent","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-science","measuredAt":"2026-11-09","stderr":3.612,"costPerTest":5.19351,"latency":7443.23},{"benchmarkId":"vals-ai:srebench:overall","benchmarkName":"SRE Bench","category":null,"domain":"Other","eligible":false,"ranked":true,"score":9.542,"unit":"percent","family":"vals-ai:srebench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/srebench","measuredAt":"2026-11-05","stderr":1.819,"costPerTest":30.855656,"latency":11800.798},{"benchmarkId":"mazeBench","benchmarkName":"MazeBench: visual spatial reasoning in a 3D open world","category":"agentic","score":6,"unit":"percent","family":"mazeBench","index":72.89857107281941,"aboveFit":false,"run":null,"source":"Jonathan Pappas and David Pappas","sourceUrl":"https://mazebench.com/blog?post=maze-bench-results","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lmarena:agent-bash-recovery:overall","benchmarkName":"Agent Arena command recovery","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":0.8842784639766732,"unit":"index","family":"agent-arena","index":67.25781683753468,"aboveFit":false,"run":"xhigh effort","source":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset","measuredAt":"2026-10-08","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"arc-prize:v3-semi-private:overall","benchmarkName":"ARC-AGI-3 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":0.8,"unit":"percent","family":"arc-prize:v3-semi-private:overall","index":65.64547757406649,"aboveFit":false,"run":"max effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:programbench:overall","benchmarkName":"ProgramBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":0.5,"unit":"percent","family":"vals-ai:programbench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/programbench","measuredAt":"2026-11-09","stderr":0.5,"costPerTest":5.377882,"latency":2016.05},{"benchmarkId":"lmarena:agent-steerability:overall","benchmarkName":"Agent Arena steerability","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":-0.5196860142374526,"unit":"index","family":"agent-arena","index":66.10353793506738,"aboveFit":false,"run":"xhigh effort","source":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset","measuredAt":"2026-11-11","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lmarena:agent-task-outcome:overall","benchmarkName":"Agent Arena task outcome","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":-7.2245662214023625,"unit":"index","family":"agent-arena","index":60.59107527180564,"aboveFit":false,"run":"xhigh effort","source":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset","measuredAt":"2026-11-11","stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":2,"outputPerMillion":12,"batch":{"inputPerMillion":1,"outputPerMillion":6},"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":66,"timeToFirstToken":2.1375,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":{"elo":1448.7394563793828,"eloUpper":1452.887684109945,"eloLower":1444.5912286488206,"votes":48825,"rank":55,"rankUpper":null,"rankLower":null,"arenaName":"GPT-5.6 Terra-xhigh","organization":"openai","updatedAt":"2026-11-09"},"quality":{"coveragePercent":100,"benchmarkCount":66,"categoryCount":8,"confidence":"high","state":"available"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:arc-prize-foundation","name":"ARC Prize Foundation","url":"https://arcprize.org","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:livebench","name":"LiveBench","url":"https://livebench.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:terminal-bench","name":"Terminal-Bench","url":"https://www.tbench.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:lmarena","name":"LMArena","url":"https://huggingface.co","license":"CC BY 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"deepseek-v4-1-flash","providerId":"deepseek","slug":"deepseek-v4-1-flash","name":"DeepSeek V4.1 Flash 1017","provider":"DeepSeek","description":"DeepSeek V4.1 Flash 1017 is a reasoning model from DeepSeek in the DeepSeek V4.1 Flash family. 61 benchmarks count toward its index, in 7 categories.","logoUrl":null,"releaseDate":"2026-10-17","contextWindowTokens":1048576,"modalities":["text","image"],"openWeights":true,"details":{"apiId":"deepseek/deepseek-v4.1-flash","maxOutputTokens":943718,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.004,"weightsUrl":"https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash"},"reasoning":true,"overallScore":68.50210603700033,"overallScoreRange":{"low":65.9317496884854,"high":71.07246238551527},"badges":["fastest"],"capabilityScores":{"agentic":72.24115517898,"coding":69.80148455548691,"reasoning":65.01490475192367,"multimodal":62.18201529824591,"knowledge":64.23891593958487,"multilingual":null,"instruction":68.29465067208821,"math":65.42111709295662},"capabilityCounts":{"math":5,"reasoning":11,"knowledge":5,"agentic":20,"coding":16,"multimodal":3,"instruction":1},"benchmarks":[{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":98.33333333333333,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":66.12883870698487,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0.6405884289881728,"costPerTest":null,"latency":null},{"benchmarkId":"arc-prize:v1-public-eval:overall","benchmarkName":"ARC-AGI-1 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":98,"unit":"percent","family":"arc-prize:v1-public-eval:overall","index":null,"aboveFit":false,"run":"max effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.04453587,"latency":null},{"benchmarkId":"arc-prize:v1-semi-private:overall","benchmarkName":"ARC-AGI-1 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":94.5,"unit":"percent","family":"arc-prize:v1-semi-private:overall","index":70.43242059127125,"aboveFit":false,"run":"max effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.06568805399999998,"latency":null},{"benchmarkId":"livebench:livebench-math:overall","benchmarkName":"LiveBench Mathematics","category":"math","domain":null,"eligible":true,"ranked":true,"score":93.28975,"unit":"percent","family":"livebench:livebench-math:overall","index":69.79454871290326,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Mathematics","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.021698416886543538,"latency":null},{"benchmarkId":"babyVisionPython","benchmarkName":"BabyVision with Python","category":"multimodal","score":92.5,"unit":"percent","family":"babyVisionPython","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gpqa","benchmarkName":"Graduate-Level Google-Proof Q&A","category":"knowledge","score":90.9,"unit":"percent","family":"gpqa-diamond","index":60.580096888057994,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gpqaDiamond","benchmarkName":"GPQA Diamond","category":"knowledge","score":90.9,"unit":"percent","family":"gpqa-diamond","index":60.580096888057994,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"terminalBench21","benchmarkName":"Terminal-Bench 2.1","category":"agentic","score":90.6,"unit":"percent","family":"terminal-bench-2-1","index":74.12901722860678,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-coding-2:overall","benchmarkName":"LiveBench Coding 2.0","category":"coding","domain":null,"eligible":true,"ranked":true,"score":90.2,"unit":"percent","family":"livebench:livebench-coding-2:overall","index":73.50604073435277,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Coding","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.23771379310344828,"latency":null},{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":89.83585858585859,"unit":"percent","family":"gpqa-diamond","index":59.60409546075699,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.7287759372442393,"costPerTest":null,"latency":null},{"benchmarkId":"cyberGym","benchmarkName":"CyberGym","category":"agentic","score":88.1,"unit":"percent","family":"cyberGym","index":71.71388027302925,"aboveFit":false,"run":null,"source":"Zhun Wang, Tianneng Shi, Jingxuan He, Matthew Cai, Jialin Zhang, Dawn Song","sourceUrl":"https://www.cybergym.io/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"extractBenchLlamaIndex","benchmarkName":"ExtractBench: schema-guided enterprise document extraction (LlamaIndex leaderboard)","category":"multimodal","score":87.11,"unit":"percent","family":"extractBenchLlamaIndex","index":67.63955609442206,"aboveFit":false,"run":null,"source":"Boyang Zhang, Adrian Lyjak, Eli Stewart, Zhaoqi Li and Simon Suo (LlamaIndex)","sourceUrl":"https://arxiv.org/abs/2607.29677","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-research:overall","benchmarkName":"LiveBench Research","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":86.8,"unit":"percent","family":"livebench-2-agentic","index":82.06901062146586,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Research","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.4086636363636364,"latency":null},{"benchmarkId":"livebench:livebench-reasoning:overall","benchmarkName":"LiveBench Reasoning","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":86.69225,"unit":"percent","family":"livebench","index":73.92380918785572,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Reasoning","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.022825742574257426,"latency":null},{"benchmarkId":"vals-ai:medscribe:overall","benchmarkName":"MedScribe","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":85.5,"unit":"percent","family":"vals-ai:medscribe:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medscribe","measuredAt":"2026-10-29","stderr":1.918,"costPerTest":0.015397,"latency":53.826},{"benchmarkId":"vals-ai:sage:overall","benchmarkName":"SAGE","category":null,"domain":"Education","eligible":false,"ranked":true,"score":85.384,"unit":"percent","family":"vals-ai:sage:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/sage","measuredAt":"2026-11-05","stderr":1.96,"costPerTest":0.011399,"latency":33.106},{"benchmarkId":"vals-ai:vibe-code:overall","benchmarkName":"Vibe Code Bench v1.1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":84.739,"unit":"percent","family":"vals-ai:vibe-code:overall","index":72.82468514050198,"aboveFit":false,"run":"OpenHands","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","measuredAt":"2026-11-09","stderr":2.891,"costPerTest":0.407445,"latency":928.101},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":84,"unit":"percent","family":"lcr","index":64.92881360836616,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-browser:overall","benchmarkName":"LiveBench Browser Use","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":84,"unit":"percent","family":"livebench-2-agentic","index":80.02128355568858,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Browser","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.348648,"latency":null},{"benchmarkId":"vals-ai:legal_bench:overall","benchmarkName":"LegalBench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":83.277,"unit":"percent","family":"vals-ai:legal_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","measuredAt":"2026-10-13","stderr":0.458,"costPerTest":0.001419,"latency":6.533},{"benchmarkId":"chartographyWithTools","benchmarkName":"Chartography with image and code tools","category":"multimodal","score":83,"unit":"percent","family":"chartographyWithTools","index":null,"aboveFit":false,"run":null,"source":"Surge AI and Anthropic","sourceUrl":"https://surgehq.ai/blog/chartography","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"arc-prize:v2-public-eval:overall","benchmarkName":"ARC-AGI-2 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":81.66666666666667,"unit":"percent","family":"arc-prize:v2-public-eval:overall","index":null,"aboveFit":false,"run":"max effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.137258925,"latency":null},{"benchmarkId":"livebench:livebench-language:overall","benchmarkName":"LiveBench Language","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":81.18966666666667,"unit":"percent","family":"livebench:livebench-language:overall","index":66.36879765065743,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Language","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.017129333333333333,"latency":null},{"benchmarkId":"livebench:livebench-coding:overall","benchmarkName":"LiveBench Coding","category":"coding","domain":null,"eligible":true,"ranked":true,"score":80.037,"unit":"percent","family":"livebench:livebench-coding:overall","index":67.38892153501365,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Coding","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.012968376068376069,"latency":null},{"benchmarkId":"livebench:livebench-data-analysis:overall","benchmarkName":"LiveBench Data Analysis","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":79.25333333333333,"unit":"percent","family":"livebench","index":64.2871836433545,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Data%20Analysis","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.03555333333333333,"latency":null},{"benchmarkId":"livebench:livebench-design:overall","benchmarkName":"LiveBench Design","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":79.2,"unit":"percent","family":"livebench-2-agentic","index":76.51089430007039,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Design","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.174804,"latency":null},{"benchmarkId":"livebench:livebench-agentic-coding:overall","benchmarkName":"LiveBench Agentic Coding","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":77.27266666666667,"unit":"percent","family":"livebench:livebench-agentic-coding:overall","index":88.85267541258554,"aboveFit":true,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Agentic%20Coding","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.06419861111111111,"latency":null},{"benchmarkId":"deepSwe","benchmarkName":"DeepSWE","category":"agentic","score":77,"unit":"percent","family":"deepSwe","index":77.05357110040067,"aboveFit":false,"run":null,"source":"Datacurve AI","sourceUrl":"https://deepswe.datacurve.ai/blog","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaMmmuPro","benchmarkName":"Artificial Analysis MMMU-Pro","category":"multimodal","score":77,"unit":"percent","family":"mmmu-pro","index":55.3615594420114,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmmu-pro","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:terminal-bench-2-1:overall","benchmarkName":"Terminal-Bench 2.1","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":74.532,"unit":"percent","family":"terminal-bench-2-1","index":64.73156123225498,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2-1","measuredAt":"2026-10-01","stderr":1.633,"costPerTest":0.098877,"latency":604.506},{"benchmarkId":"vals-ai:cyber:overall","benchmarkName":"CyberBench v1.1","category":null,"domain":"Other","eligible":false,"ranked":true,"score":73.69,"unit":"percent","family":"vals-ai:cyber:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/cyber","measuredAt":"2026-11-05","stderr":5.476,"costPerTest":0.105475,"latency":830.108},{"benchmarkId":"arc-prize:v2-semi-private:overall","benchmarkName":"ARC-AGI-2 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":72.91666666666666,"unit":"percent","family":"arc-prize:v2-semi-private:overall","index":72.66708734842058,"aboveFit":false,"run":"max effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.12938378500000003,"latency":null},{"benchmarkId":"livebench:livebench-instruction-following:overall","benchmarkName":"LiveBench Instruction Following","category":"instruction","domain":null,"eligible":true,"ranked":true,"score":70.03325000000001,"unit":"percent","family":"livebench:livebench-instruction-following:overall","index":68.29465067208821,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=IF","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.01592,"latency":null},{"benchmarkId":"vals-ai:skillsbench:overall","benchmarkName":"SkillsBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":69.802,"unit":"percent","family":"vals-ai:skillsbench:overall","index":77.09678309386531,"aboveFit":false,"run":"OpenHands","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/skillsbench","measuredAt":"2026-10-29","stderr":3.879,"costPerTest":0.085117,"latency":223.451},{"benchmarkId":"scarfBench","benchmarkName":"ScarfBench: Cross-Framework Application Migration in Enterprise Java","category":"coding","score":69,"unit":"percent","family":"scarfBench","index":73.61566760331243,"aboveFit":false,"run":null,"source":"Advait Pavuluri, Bridget McGinn, Ashita Saxena, George Safta, Srikanth Tamilselvam, Raju Pavuluri, Michele Merler, Baishakhi Ray, Rahul Krishna (IBM Research)","sourceUrl":"https://arxiv.org/abs/2605.06754","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaCweBench","benchmarkName":"Artificial Analysis CWE-Bench-AA","category":"agentic","score":68.3,"unit":"percent","family":"aaCweBench","index":78.92115781667178,"aboveFit":false,"run":null,"source":"Artificial Analysis; dataset by Collinear AI","sourceUrl":"https://artificialanalysis.ai/evaluations/cwe-bench-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"nl2Repo","benchmarkName":"NL2Repo","category":"coding","score":68.2,"unit":"percent","family":"nl2Repo","index":74.86642531001075,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:biomysterybench:overall","benchmarkName":"BioMysteryBench","category":null,"domain":"Science","eligible":false,"ranked":true,"score":67.778,"unit":"percent","family":"vals-ai:biomysterybench:overall","index":null,"aboveFit":false,"run":"Terminus 2","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/biomysterybench","measuredAt":"2026-11-09","stderr":1.111,"costPerTest":0.233183,"latency":890.067},{"benchmarkId":"epoch-ai:epoch-frontiermath-tiers-1-3-v2-private:overall","benchmarkName":"FrontierMath-Tiers-1-3-v2-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":67.36842105263158,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tiers-1-3-v2-private:overall","index":66.35418001386807,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.7821981570324446,"costPerTest":null,"latency":null},{"benchmarkId":"apex","benchmarkName":"Apex","category":"math","score":65.6,"unit":"percent","family":"apex","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-mirrorcode:overall","benchmarkName":"MirrorCode","category":"coding","domain":null,"eligible":true,"ranked":true,"score":64.44444444444444,"unit":"percent","family":"epoch-ai:epoch-mirrorcode:overall","index":77.86778712812537,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":11.369999821616648,"costPerTest":null,"latency":null},{"benchmarkId":"parseBenchLlamaIndex","benchmarkName":"ParseBench: document parsing for AI agents (LlamaIndex leaderboard)","category":"multimodal","score":64.38,"unit":"percent","family":"parseBenchLlamaIndex","index":63.54493035830426,"aboveFit":false,"run":null,"source":"Boyang Zhang, Sebastián G. Acosta, Preston Carlson, Sacha Bron, Pierre-Loïc Doulcet, Daniel B. Ospina and Simon Suo (LlamaIndex)","sourceUrl":"https://arxiv.org/abs/2604.08538","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:public-benefits-bench:overall","benchmarkName":"Public Benefits Bench v1.1","category":null,"domain":"Public benefits","eligible":false,"ranked":true,"score":64.276,"unit":"percent","family":"vals-ai:public-benefits-bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/public-benefits-bench","measuredAt":"2026-10-12","stderr":1.246,"costPerTest":0.073234,"latency":598.96},{"benchmarkId":"hleWithTools","benchmarkName":"Humanity's Last Exam with tools","category":"agentic","score":63.9,"unit":"percent","family":"hleWithTools","index":71.88549456836446,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"automationBench","benchmarkName":"AutomationBench","category":"agentic","score":60.5,"unit":"percent","family":"automationBench","index":95,"aboveFit":true,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"openHarmonyBench","benchmarkName":"OpenHarmony Bench v1.0","category":"coding","score":60.3,"unit":"percent","family":"openHarmonyBench","index":66.92810594462074,"aboveFit":false,"run":null,"source":"OpenHarmony Bench authors","sourceUrl":"https://arxiv.org/abs/2608.16022","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"cweBenchV2","benchmarkName":"CWE-bench v2","category":"agentic","score":57.6,"unit":"percent","family":"cweBenchV2","index":72.20785960403899,"aboveFit":false,"run":null,"source":"Collinear AI","sourceUrl":"https://cwe-bench.com/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:emb:overall","benchmarkName":"EMB","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":57.208,"unit":"percent","family":"vals-ai:emb:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/emb","measuredAt":"2026-10-20","stderr":3.227,"costPerTest":0.234152,"latency":780.458},{"benchmarkId":"gdpvalAaNormalized","benchmarkName":"GDPval-AA normalized","category":"agentic","score":55,"unit":"percent","family":"gdpvalAaNormalized","index":73.35749681024355,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"cweBenchV1","benchmarkName":"CWE-bench v1","category":"agentic","score":55,"unit":"percent","family":"cweBenchV1","index":68.55293574761009,"aboveFit":false,"run":null,"source":"Collinear AI","sourceUrl":"https://cwe-bench.com/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:proof_bench:overall","benchmarkName":"ProofBench v1.1","category":"math","domain":null,"eligible":true,"ranked":true,"score":54,"unit":"percent","family":"vals-ai:proof_bench:overall","index":65.79983444422184,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/proof_bench","measuredAt":"2026-11-09","stderr":5.009,"costPerTest":0.126437,"latency":546.595},{"benchmarkId":"vals-ai:fabv2:overall","benchmarkName":"Finance Agent (v2)","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":53.481,"unit":"percent","family":"vals-ai:fabv2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/fabv2","measuredAt":"2026-10-29","stderr":0.39,"costPerTest":0.207465,"latency":364.376},{"benchmarkId":"aaSciCode","benchmarkName":"Artificial Analysis SciCode","category":"coding","score":51.9,"unit":"percent","family":"scicode-family","index":63.50136894143591,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/scicode","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"zeroBenchPython","benchmarkName":"ZeroBench_main with Python","category":"multimodal","score":51,"unit":"percent","family":"zeroBenchPython","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-chess-puzzles:overall","benchmarkName":"Chess Puzzles","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":48,"unit":"percent","family":"epoch-ai:epoch-chess-puzzles:overall","index":70.41740509234356,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":5.021167315686783,"costPerTest":null,"latency":null},{"benchmarkId":"cursorBench40","benchmarkName":"cursorBench40","category":"coding","score":47.7,"unit":"percent","family":"cursorBench40","index":73.16829471299272,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaItbench","benchmarkName":"Artificial Analysis ITBench-AA","category":"agentic","score":46.9,"unit":"percent","family":"aaItbench","index":67.57580875399883,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/itbench-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"omniscienceAccuracy","benchmarkName":"Artificial Analysis Omniscience Accuracy","category":"knowledge","score":46.4,"unit":"percent","family":"omniscienceAccuracy","index":69.16582812698707,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:code-migration:overall","benchmarkName":"Code Migration","category":"coding","domain":null,"eligible":true,"ranked":true,"score":45.623,"unit":"percent","family":"vals-ai:code-migration:overall","index":69.42590423957421,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/code-migration","measuredAt":"2026-10-29","stderr":4.285,"costPerTest":0.944694,"latency":6325.865},{"benchmarkId":"epoch-ai:epoch-mystery-game-puzzles:overall","benchmarkName":"Mystery Game Puzzles","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":43,"unit":"percent","family":"epoch-ai:epoch-mystery-game-puzzles:overall","index":71.98083876039257,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":4.97569851956243,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-simpleqa-verified:overall","benchmarkName":"SimpleQA Verified","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":42.9,"unit":"percent","family":"epoch-ai:epoch-simpleqa-verified:overall","index":59.65853944718977,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.5658997547870177,"costPerTest":null,"latency":null},{"benchmarkId":"hle","benchmarkName":"Humanity's Last Exam","category":"knowledge","score":42.6,"unit":"percent","family":"hle","index":67.12691525158272,"aboveFit":false,"run":null,"source":"Center for AI Safety, Scale AI, and thousands of expert contributors","sourceUrl":"https://lastexam.ai/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:legal_research:overall","benchmarkName":"Legal Research Bench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":41.346,"unit":"percent","family":"vals-ai:legal_research:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_research","measuredAt":"2026-11-09","stderr":3.423,"costPerTest":0.24793,"latency":701.783},{"benchmarkId":"vals-ai:medcode:overall","benchmarkName":"MedCode","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":41.173,"unit":"percent","family":"vals-ai:medcode:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medcode","measuredAt":"2026-10-29","stderr":2.042,"costPerTest":0.01245,"latency":35.126},{"benchmarkId":"aaCyberIndex","benchmarkName":"Artificial Analysis Cyber Index","category":"agentic","score":40.8,"unit":"percent","family":"aaCyberIndex","index":69.12231293377714,"aboveFit":false,"run":null,"source":"Artificial Analysis with the Cyber Index Alliance (Collinear AI, IBM, NVIDIA, Pasta Lab, Vercel)","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-cyber-index","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:ioi:overall","benchmarkName":"IOI","category":"coding","domain":null,"eligible":true,"ranked":true,"score":40.278,"unit":"percent","family":"vals-ai:ioi:overall","index":61.237897632655404,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/ioi","measuredAt":"2026-10-29","stderr":2.557,"costPerTest":0.24255,"latency":1109.577},{"benchmarkId":"livebench:livebench-long-horizon:overall","benchmarkName":"LiveBench Long Agentic","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":40.1,"unit":"percent","family":"livebench-2-agentic","index":47.915848488680524,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Long%20Agentic","measuredAt":"2026-10-14","stderr":null,"costPerTest":1.646052,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":39.2,"unit":"percent","family":"hle","index":63.71571991848142,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"bug-hunt-bench","benchmarkName":"Bug Hunt Bench","category":"coding","score":39.1,"unit":"percent","family":"bug-hunt-bench","index":75.91313647011724,"aboveFit":false,"run":null,"source":"Pawel Huryn","sourceUrl":"https://bughunt.productcompass.pm/method","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"agentsLastExam","benchmarkName":"Agents' Last Exam","category":"agentic","score":35.4,"unit":"percent","family":"agentsLastExam","index":70.44630200765641,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-furniture-assembly:overall","benchmarkName":"Furniture Assembly","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":34.166666666666664,"unit":"percent","family":"epoch-ai:epoch-furniture-assembly:overall","index":62.59122331111903,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":6.000431560123094,"costPerTest":null,"latency":null},{"benchmarkId":"aaDeepsecBench","benchmarkName":"Artificial Analysis DeepsecBench-AA","category":"agentic","score":31.2,"unit":"percent","family":"aaDeepsecBench","index":74.2845465265532,"aboveFit":false,"run":null,"source":"Artificial Analysis; dataset by Vercel","sourceUrl":"https://artificialanalysis.ai/evaluations/deepsecbench-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"terminalBench3","benchmarkName":"terminalBench3","category":"agentic","score":30,"unit":"percent","family":"terminalBench3","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"programBench","benchmarkName":"ProgramBench: Can Language Models Rebuild Programs From Scratch?","category":"coding","score":29.1,"unit":"percent","family":"programBench","index":58.21941693109842,"aboveFit":false,"run":null,"source":"John Yang, Kilian Lieret, Jeffrey Ma, Parth Thakkar, Dmitrii Pedchenko, Sten Sootla, Emily McMilin, Pengcheng Yin, Rui Hou, Gabriel Synnaeve, Diyi Yang, Ofir Press","sourceUrl":"https://programbench.com/static/paper.pdf","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tier-4-v2-private:overall","benchmarkName":"FrontierMath-Tier-4-v2-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":26.82926829268293,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tier-4-v2-private:overall","index":59.02818358680503,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":7.005564203095157,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:tax_agent_bench:overall","benchmarkName":"Tax Agent Bench","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":26.63,"unit":"percent","family":"vals-ai:tax_agent_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench","measuredAt":"2026-11-09","stderr":3.141,"costPerTest":0.134662,"latency":349.076},{"benchmarkId":"aaCyberGymE2e","benchmarkName":"Artificial Analysis CyberGym-E2E-AA","category":"agentic","score":22.9,"unit":"percent","family":"aaCyberGymE2e","index":52.36094340789298,"aboveFit":false,"run":null,"source":"Artificial Analysis; dataset by Berkeley RDI","sourceUrl":"https://artificialanalysis.ai/evaluations/cybergym-e2e-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:mysterymechanism:overall","benchmarkName":"MysteryMechanism","category":null,"domain":"Science","eligible":false,"ranked":true,"score":21.171,"unit":"percent","family":"vals-ai:mysterymechanism:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mysterymechanism","measuredAt":"2026-11-11","stderr":2.748,"costPerTest":0.164089,"latency":484.327},{"benchmarkId":"vals-ai:terminal-bench-4:overall","benchmarkName":"Terminal-Bench 4.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":19.697,"unit":"percent","family":"terminal-bench-4","index":63.86890998891293,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-4","measuredAt":"2026-11-09","stderr":1.75,"costPerTest":0.499473,"latency":2902.323},{"benchmarkId":"frontierCode2","benchmarkName":"FrontierCode 2.0","category":"coding","score":19.3,"unit":"percent","family":"frontierCode2","index":66.33584336290015,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:vcb-1-100:overall","benchmarkName":"Vibe Code Bench 1-100","category":"coding","domain":null,"eligible":true,"ranked":true,"score":16.378,"unit":"percent","family":"vals-ai:vcb-1-100:overall","index":64.92747410721344,"aboveFit":false,"run":"OpenHands","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vcb-1-100","measuredAt":"2026-10-29","stderr":3.194,"costPerTest":0.688342,"latency":1701.009},{"benchmarkId":"exploitGym","benchmarkName":"ExploitGym","category":"agentic","score":15.3,"unit":"percent","family":"exploitGym","index":68.27005800593584,"aboveFit":false,"run":null,"source":"Zhun Wang, Nico Schiller, Hongwei Li, Srijiith Sesha Narayana, Milad Nasr, Nicholas Carlini, Xiangyu Qi, Eric Wallace, Elie Bursztein, Luca Invernizzi, Kurt Thomas, Yan Shoshitaishvili, Wenbo Guo, Jingxuan He, Thorsten Holz, Dawn Song","sourceUrl":"https://arxiv.org/abs/2605.11086","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":14.3,"unit":"percent","family":"critpt","index":64.532247441567,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lmarena:agent-bash-recovery:overall","benchmarkName":"Agent Arena command recovery","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":7.85815025696297,"unit":"index","family":"agent-arena","index":72.99143275450965,"aboveFit":false,"run":"max effort","source":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset","measuredAt":"2026-10-08","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"jevBench15","benchmarkName":"jevBench15","category":"reasoning","score":6.645505039231519,"unit":"percent","family":"jevBench15","index":55.49825241443918,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"jevBench14","benchmarkName":"jevBench14","category":"reasoning","score":4.768648223526218,"unit":"percent","family":"jevBench14","index":47.34273932195467,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lmarena:agent-steerability:overall","benchmarkName":"Agent Arena steerability","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":4.042197262679842,"unit":"index","family":"agent-arena","index":69.85412116119154,"aboveFit":false,"run":"max effort","source":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset","measuredAt":"2026-11-11","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"stemBench120","benchmarkName":"STEMBench120: six-domain scientific research progress","category":"reasoning","score":2.538,"unit":"percent","family":"stemBench120","index":null,"aboveFit":false,"run":null,"source":"Ulam","sourceUrl":"https://www.ulam.ai/stembench","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lmarena:agent-task-outcome:overall","benchmarkName":"Agent Arena task outcome","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":2.341887046054003,"unit":"index","family":"agent-arena","index":68.45619960323494,"aboveFit":false,"run":"max effort","source":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset","measuredAt":"2026-11-11","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mazeBench","benchmarkName":"MazeBench: visual spatial reasoning in a 3D open world","category":"agentic","score":1,"unit":"percent","family":"mazeBench","index":63.79865130242642,"aboveFit":false,"run":null,"source":"Jonathan Pappas and David Pappas","sourceUrl":"https://mazebench.com/blog?post=maze-bench-results","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"arc-prize:v3-semi-private:overall","benchmarkName":"ARC-AGI-3 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":0.8516646657220222,"unit":"percent","family":"arc-prize:v3-semi-private:overall","index":65.66742796568126,"aboveFit":false,"run":"max effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:srebench:overall","benchmarkName":"SRE Bench","category":null,"domain":"Other","eligible":false,"ranked":true,"score":0.763,"unit":"percent","family":"vals-ai:srebench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/srebench","measuredAt":"2026-11-05","stderr":0.539,"costPerTest":0.548583,"latency":3663.235},{"benchmarkId":"vals-ai:programbench:overall","benchmarkName":"ProgramBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":0.5,"unit":"percent","family":"vals-ai:programbench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/programbench","measuredAt":"2026-11-09","stderr":0.5,"costPerTest":0.896935,"latency":3709.707},{"benchmarkId":"vals-ai:terminal-bench-science:overall","benchmarkName":"Terminal-Bench-Science","category":null,"domain":"Science","eligible":false,"ranked":true,"score":0,"unit":"percent","family":"vals-ai:terminal-bench-science:overall","index":null,"aboveFit":false,"run":"Mini-SWE-agent","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-science","measuredAt":"2026-11-09","stderr":0,"costPerTest":0.473154,"latency":10621.573}],"price":{"inputPerMillion":0.022,"outputPerMillion":0.39999999999999997,"batch":{"inputPerMillion":0.011,"outputPerMillion":0.19999999999999998},"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":261,"timeToFirstToken":0.194,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":{"elo":1463.390381391253,"eloUpper":1469.4948625180182,"eloLower":1457.2859002644877,"votes":13580,"rank":39,"rankUpper":null,"rankLower":null,"arenaName":"DeepSeek-V4.1-Flash-20260910-max","organization":"deepseek","updatedAt":"2026-11-09"},"quality":{"coveragePercent":95,"benchmarkCount":61,"categoryCount":7,"confidence":"high","state":"available"},"hosts":[{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":0.022,"outputPerMillion":0.4,"cacheReadPerMillion":0.004,"contextTokens":1048576,"maxOutputTokens":943718,"deprecated":false},{"host":"vultr","name":"Vultr","kind":"cloud","url":"https://api.vultrinference.com/","inputPerMillion":0.15,"outputPerMillion":0.6,"cacheReadPerMillion":null,"contextTokens":1048576,"maxOutputTokens":384000,"deprecated":false},{"host":"deepinfra","name":"Deep Infra","kind":"host","url":"https://deepinfra.com/models","inputPerMillion":0.2,"outputPerMillion":0.6,"cacheReadPerMillion":0.006,"contextTokens":1048576,"maxOutputTokens":384000,"deprecated":false},{"host":"coreweave","name":"CoreWeave","kind":"host","url":"https://docs.wandb.ai/inference","inputPerMillion":0.13,"outputPerMillion":0.65,"cacheReadPerMillion":0.002,"contextTokens":1048576,"maxOutputTokens":1048576,"deprecated":false},{"host":"arcee-ai","name":"Arcee AI","kind":"host","url":"https://docs.arcee.ai","inputPerMillion":0.22,"outputPerMillion":0.66,"cacheReadPerMillion":0.007,"contextTokens":1000000,"maxOutputTokens":384000,"deprecated":false},{"host":"fireworks-ai","name":"Fireworks AI","kind":"host","url":"https://fireworks.ai/docs/","inputPerMillion":0.21,"outputPerMillion":0.84,"cacheReadPerMillion":0.0042,"contextTokens":1000000,"maxOutputTokens":384000,"deprecated":false},{"host":"baseten","name":"Baseten","kind":"host","url":"https://docs.baseten.co/inference/model-apis/overview","inputPerMillion":0.3,"outputPerMillion":1.2,"cacheReadPerMillion":0.007,"contextTokens":1048576,"maxOutputTokens":32768,"deprecated":false},{"host":"digitalocean","name":"DigitalOcean","kind":"cloud","url":"https://docs.digitalocean.com/products/gradient-ai-platform/details/models/","inputPerMillion":0.3,"outputPerMillion":1.2,"cacheReadPerMillion":0.006,"contextTokens":1048576,"maxOutputTokens":1048576,"deprecated":false},{"host":"azure","name":"Microsoft Azure","kind":"cloud","url":"https://learn.microsoft.com/en-us/azure/ai-services/openai/concepts/models","inputPerMillion":0.3,"outputPerMillion":1.2,"cacheReadPerMillion":0.006,"contextTokens":1000000,"maxOutputTokens":128000,"deprecated":false},{"host":"modal","name":"Modal","kind":"host","url":"https://modal.com/docs/guide/endpoints","inputPerMillion":0.3,"outputPerMillion":1.2,"cacheReadPerMillion":0.03,"contextTokens":1000000,"maxOutputTokens":384000,"deprecated":false},{"host":"novita-ai","name":"Novita AI","kind":"host","url":"https://novita.ai/docs/guides/introduction","inputPerMillion":0.3,"outputPerMillion":1.2,"cacheReadPerMillion":0.006,"contextTokens":1048576,"maxOutputTokens":393216,"deprecated":false},{"host":"together-ai","name":"Together AI","kind":"host","url":"https://docs.together.ai/docs/serverless-models","inputPerMillion":0.3,"outputPerMillion":1.2,"cacheReadPerMillion":0.006,"contextTokens":1048576,"maxOutputTokens":384000,"deprecated":false},{"host":"tinfoil","name":"Tinfoil","kind":"host","url":"https://docs.tinfoil.sh","inputPerMillion":0.6,"outputPerMillion":1.45,"cacheReadPerMillion":0.01,"contextTokens":1048576,"maxOutputTokens":384000,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:arc-prize-foundation","name":"ARC Prize Foundation","url":"https://arcprize.org","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:livebench","name":"LiveBench","url":"https://livebench.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:lmarena","name":"LMArena","url":"https://huggingface.co","license":"CC BY 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gemini-3-7-flash","providerId":"google","slug":"gemini-3-7-flash","name":"Gemini 3.7 Flash","provider":"Google","description":"Gemini 3.7 Flash is a reasoning model from Google. 53 benchmarks count toward its index, in 8 categories.","logoUrl":null,"releaseDate":"2026-08-13","contextWindowTokens":1048576,"modalities":["text","image","video","file","audio"],"openWeights":false,"details":{"apiId":"google/gemini-3.7-flash","maxOutputTokens":65536,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.075,"weightsUrl":null},"reasoning":true,"overallScore":68.46242934077264,"overallScoreRange":{"low":66.24130902172234,"high":70.68354965982294},"badges":[],"capabilityScores":{"agentic":65.06719640767051,"coding":66.8818526894305,"reasoning":68.72718170943867,"multimodal":67.74465569740158,"knowledge":71.24860529701672,"multilingual":72.25290204605,"instruction":88.31071571542654,"math":66.82749158825978},"capabilityCounts":{"math":5,"reasoning":8,"knowledge":6,"multimodal":4,"coding":12,"agentic":16,"instruction":1,"multilingual":1},"benchmarks":[{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":97.22222222222221,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":65.48240087839254,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.8586784946171873,"costPerTest":null,"latency":null},{"benchmarkId":"mrcrv2_64_128","benchmarkName":"OpenAI MRCR v2 8-needle 64K-128K","category":"reasoning","score":97,"unit":"percent","family":"mrcrv2_64_128","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"arc-prize:v1-public-eval:overall","benchmarkName":"ARC-AGI-1 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":96.625,"unit":"percent","family":"arc-prize:v1-public-eval:overall","index":null,"aboveFit":false,"run":"high effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.09120528,"latency":null},{"benchmarkId":"arc-prize:v1-semi-private:overall","benchmarkName":"ARC-AGI-1 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":95.5,"unit":"percent","family":"arc-prize:v1-semi-private:overall","index":70.89215669424564,"aboveFit":false,"run":"high effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.12013233,"latency":null},{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":94.82323232323232,"unit":"percent","family":"gpqa-diamond","index":64.17837837228873,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.345652529224626,"costPerTest":null,"latency":null},{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":94.5,"unit":"percent","family":"gpqa-diamond","index":63.881918517769215,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:gpqa:overall","benchmarkName":"GPQA Diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":93.94,"unit":"percent","family":"gpqa-diamond","index":63.368301819814135,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","measuredAt":"2026-09-01","stderr":1.488,"costPerTest":0.023122,"latency":8.773},{"benchmarkId":"livebench:livebench-math:overall","benchmarkName":"LiveBench Mathematics","category":"math","domain":null,"eligible":true,"ranked":true,"score":93.46799999999999,"unit":"percent","family":"livebench:livebench-math:overall","index":70.04297917696849,"aboveFit":false,"run":"high effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Mathematics","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.07440395778364116,"latency":null},{"benchmarkId":"aaHarveyLab","benchmarkName":"Artificial Analysis Harvey LAB-AA v1.0","category":"agentic","score":90.7,"unit":"percent","family":"aaHarveyLab","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:mmlu_pro:overall","benchmarkName":"MMLU Pro","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":90.121,"unit":"percent","family":"mmlu-pro","index":58.02240122053867,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","measuredAt":"2026-09-01","stderr":0.296,"costPerTest":0.010817,"latency":4.159},{"benchmarkId":"vals-ai:mmmu:overall","benchmarkName":"MMMU Pro","category":"multimodal","domain":null,"eligible":true,"ranked":true,"score":88.96,"unit":"percent","family":"mmmu-pro","index":70.9264501328366,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmmu","measuredAt":"2026-09-01","stderr":0.753,"costPerTest":0.014052,"latency":6.283},{"benchmarkId":"charxiv","benchmarkName":"CharXiv Reasoning","category":"multimodal","score":88.7,"unit":"percent","family":"charxiv","index":65.16901137916365,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:lcb:overall","benchmarkName":"LiveCodeBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":88.652,"unit":"percent","family":"livecodebench","index":62.901985937049616,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","measuredAt":"2026-09-01","stderr":0.924,"costPerTest":0.040218,"latency":14.533},{"benchmarkId":"livebench:livebench-reasoning:overall","benchmarkName":"LiveBench Reasoning","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":87.798,"unit":"percent","family":"livebench","index":75.35623532909477,"aboveFit":false,"run":"high effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Reasoning","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.0688980198019802,"latency":null},{"benchmarkId":"vals-ai:legal_bench:overall","benchmarkName":"LegalBench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":87.256,"unit":"percent","family":"vals-ai:legal_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","measuredAt":"2026-10-13","stderr":0.416,"costPerTest":0.00409,"latency":2.227},{"benchmarkId":"bioMysteryBenchHumanSolvable","benchmarkName":"BioMysteryBench Human Solvable","category":"knowledge","score":87.1,"unit":"percent","family":"bioMysteryBenchHumanSolvable","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"terminalBench21","benchmarkName":"Terminal-Bench 2.1","category":"agentic","score":85.8,"unit":"percent","family":"terminal-bench-2-1","index":71.3217114791365,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaTerminalBench21","benchmarkName":"Artificial Analysis Terminal-Bench v2.1","category":"agentic","score":85.8,"unit":"percent","family":"aaTerminalBench21","index":68.41775003308598,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/terminalbench-v2-1","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaMmmuPro","benchmarkName":"Artificial Analysis MMMU-Pro","category":"multimodal","score":85.5,"unit":"percent","family":"mmmu-pro","index":66.4235636955243,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmmu-pro","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-language:overall","benchmarkName":"LiveBench Language","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":85.455,"unit":"percent","family":"livebench:livebench-language:overall","index":71.19207769134232,"aboveFit":false,"run":"high effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Language","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.03095133333333333,"latency":null},{"benchmarkId":"lvBench","benchmarkName":"LVBench","category":"multimodal","score":85.4,"unit":"percent","family":"lvBench","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"arc-prize:v2-public-eval:overall","benchmarkName":"ARC-AGI-2 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":85.27777777777777,"unit":"percent","family":"arc-prize:v2-public-eval:overall","index":null,"aboveFit":false,"run":"high effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.27265338124999994,"latency":null},{"benchmarkId":"arc-prize:v2-semi-private:overall","benchmarkName":"ARC-AGI-2 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":84.58333333333333,"unit":"percent","family":"arc-prize:v2-semi-private:overall","index":78.02365885980907,"aboveFit":false,"run":"high effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.24944951875000002,"latency":null},{"benchmarkId":"charxivNoTools","benchmarkName":"CharXiv Reasoning without tools","category":"multimodal","score":84.5,"unit":"percent","family":"charxivNoTools","index":65.23153134323701,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:medscribe:overall","benchmarkName":"MedScribe","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":83.942,"unit":"percent","family":"vals-ai:medscribe:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medscribe","measuredAt":"2026-10-29","stderr":2.004,"costPerTest":0.058736,"latency":16.818},{"benchmarkId":"labBench2","benchmarkName":"LABBench2: An Improved Benchmark for AI Systems Performing Biology Research","category":"knowledge","score":82.1,"unit":"percent","family":"labBench2","index":null,"aboveFit":false,"run":null,"source":"Jon M. Laurent, Albert Bou, Michael Pieler, Conor Igoe, Alex Andonian, Siddharth Narayanan, James Braza, Alexandros Sanchez Vassopoulos, Jacob L. Steenwyk, Blake Lash, Andrew D. White, Samuel G. Rodriques","sourceUrl":"https://arxiv.org/abs/2604.09554","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":81.7,"unit":"percent","family":"lcr","index":63.350828399519145,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:swebench:overall","benchmarkName":"SWE-bench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":80.8,"unit":"percent","family":"swe-bench-verified","index":60.822579814393954,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/swebench","measuredAt":"2026-09-01","stderr":1.763,"costPerTest":1.436459,"latency":350.439},{"benchmarkId":"livebench:livebench-coding-2:overall","benchmarkName":"LiveBench Coding 2.0","category":"coding","domain":null,"eligible":true,"ranked":true,"score":80.5,"unit":"percent","family":"livebench:livebench-coding-2:overall","index":66.48008749701663,"aboveFit":false,"run":"high effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Coding","measuredAt":"2026-10-14","stderr":null,"costPerTest":1.731706896551724,"latency":null},{"benchmarkId":"livebench:livebench-instruction-following:overall","benchmarkName":"LiveBench Instruction Following","category":"instruction","domain":null,"eligible":true,"ranked":true,"score":79.92525,"unit":"percent","family":"livebench:livebench-instruction-following:overall","index":88.31071571542654,"aboveFit":true,"run":"high effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=IF","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.097494,"latency":null},{"benchmarkId":"livebench:livebench-coding:overall","benchmarkName":"LiveBench Coding","category":"coding","domain":null,"eligible":true,"ranked":true,"score":78.8885,"unit":"percent","family":"livebench:livebench-coding:overall","index":65.77368537936515,"aboveFit":false,"run":"high effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Coding","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.051431623931623933,"latency":null},{"benchmarkId":"vals-ai:terminal-bench-2-1:overall","benchmarkName":"Terminal-Bench 2.1","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":77.528,"unit":"percent","family":"terminal-bench-2-1","index":66.48378790421603,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2-1","measuredAt":"2026-10-01","stderr":0.649,"costPerTest":1.022219,"latency":492.202},{"benchmarkId":"euroeval:euroeval-french:overall","benchmarkName":"EuroEval French","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":77.13125000000001,"unit":"percent","family":"euroeval","index":76.86959431350157,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/french","measuredAt":null,"stderr":0.3487925601270761,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-italian:overall","benchmarkName":"EuroEval Italian","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":76.85625,"unit":"percent","family":"euroeval","index":76.49648258919885,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/italian","measuredAt":null,"stderr":0.3361895856507158,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":76.64874999999999,"unit":"percent","family":"euroeval","index":76.21495283358861,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":0.45767585418066353,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-research:overall","benchmarkName":"LiveBench Research","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":76.5,"unit":"percent","family":"livebench-2-agentic","index":74.53630034378516,"aboveFit":false,"run":"high effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Research","measuredAt":"2026-10-14","stderr":null,"costPerTest":2.6836272727272728,"latency":null},{"benchmarkId":"vals-ai:tax_eval_v2:overall","benchmarkName":"TaxEval v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":74.734,"unit":"percent","family":"vals-ai:tax_eval_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","measuredAt":"2026-09-01","stderr":0.85,"costPerTest":0.018681,"latency":7.832},{"benchmarkId":"euroeval:euroeval-portuguese:overall","benchmarkName":"EuroEval Portuguese","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":74.35374999999999,"unit":"percent","family":"euroeval","index":73.10116589804423,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/portuguese","measuredAt":null,"stderr":0.4653778706599616,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-design:overall","benchmarkName":"LiveBench Design","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":73.2,"unit":"percent","family":"livebench-2-agentic","index":72.12290773054765,"aboveFit":false,"run":"high effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Design","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.62314,"latency":null},{"benchmarkId":"euroeval:euroeval-dutch:overall","benchmarkName":"EuroEval Dutch","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":73.10333333333334,"unit":"percent","family":"euroeval","index":71.40463819405574,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/dutch","measuredAt":null,"stderr":0.39125155306756004,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tiers-1-3-v2-private:overall","benchmarkName":"FrontierMath-Tiers-1-3-v2-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":71.57894736842105,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tiers-1-3-v2-private:overall","index":68.57039774565216,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.6764156649364654,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:emb:overall","benchmarkName":"EMB","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":71.326,"unit":"percent","family":"vals-ai:emb:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/emb","measuredAt":"2026-10-20","stderr":2.262,"costPerTest":5.694001,"latency":650.914},{"benchmarkId":"vals-ai:vibe-code:overall","benchmarkName":"Vibe Code Bench v1.1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":70.395,"unit":"percent","family":"vals-ai:vibe-code:overall","index":67.64914457936001,"aboveFit":false,"run":"OpenHands","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","measuredAt":"2026-11-09","stderr":4.837,"costPerTest":4.827643,"latency":1423.494},{"benchmarkId":"parseBenchLlamaIndex","benchmarkName":"ParseBench: document parsing for AI agents (LlamaIndex leaderboard)","category":"multimodal","score":70.37,"unit":"percent","family":"parseBenchLlamaIndex","index":71.90307315302522,"aboveFit":false,"run":null,"source":"Boyang Zhang, Sebastián G. Acosta, Preston Carlson, Sacha Bron, Pierre-Loïc Doulcet, Daniel B. Ospina and Simon Suo (LlamaIndex)","sourceUrl":"https://arxiv.org/abs/2604.08538","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-simpleqa-verified:overall","benchmarkName":"SimpleQA Verified","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":69.19999999999999,"unit":"percent","family":"epoch-ai:epoch-simpleqa-verified:overall","index":81.56876116709256,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.460648312734278,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-spanish:overall","benchmarkName":"EuroEval Spanish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":68.30125,"unit":"percent","family":"euroeval","index":64.88931603861836,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/spanish","measuredAt":null,"stderr":0.41128229356489443,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-data-analysis:overall","benchmarkName":"LiveBench Data Analysis","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":67.96433333333333,"unit":"percent","family":"livebench","index":49.663029260856014,"aboveFit":false,"run":"high effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Data%20Analysis","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.16665666666666668,"latency":null},{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":67.86571428571428,"unit":"percent","family":"euroeval","index":64.29839428824543,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.4215399368593051,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:ioi:overall","benchmarkName":"IOI","category":"coding","domain":null,"eligible":true,"ranked":true,"score":67.833,"unit":"percent","family":"vals-ai:ioi:overall","index":71.58266188667017,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/ioi","measuredAt":"2026-10-29","stderr":3.606,"costPerTest":3.398069,"latency":1269.021},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":67.33,"unit":"percent","family":"euroeval","index":63.57155326687654,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.525600846650764,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:mortgage_tax:overall","benchmarkName":"MortgageTax","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":66.654,"unit":"percent","family":"vals-ai:mortgage_tax:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mortgage_tax","measuredAt":"2026-09-01","stderr":0.923,"costPerTest":0.011071,"latency":4.114},{"benchmarkId":"epoch-ai:epoch-chess-puzzles:overall","benchmarkName":"Chess Puzzles","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":66,"unit":"percent","family":"epoch-ai:epoch-chess-puzzles:overall","index":84.0796786050453,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":4.760952285695233,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:skillsbench:overall","benchmarkName":"SkillsBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":65.891,"unit":"percent","family":"vals-ai:skillsbench:overall","index":74.18593464625093,"aboveFit":false,"run":"OpenHands","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/skillsbench","measuredAt":"2026-10-29","stderr":4.554,"costPerTest":1.799454,"latency":185.651},{"benchmarkId":"deepSwe","benchmarkName":"DeepSWE","category":"agentic","score":65.3,"unit":"percent","family":"deepSwe","index":68.5479977176714,"aboveFit":false,"run":null,"source":"Datacurve AI","sourceUrl":"https://deepswe.datacurve.ai/blog","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaAnalystAgent","benchmarkName":"Artificial Analysis AnalystAgent","category":"agentic","score":60,"unit":"percent","family":"aaAnalystAgent","index":83.31205193486775,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/aa-analyst-agent","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:fabv2:overall","benchmarkName":"Finance Agent (v2)","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":59.042,"unit":"percent","family":"vals-ai:fabv2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/fabv2","measuredAt":"2026-10-29","stderr":0.272,"costPerTest":1.477884,"latency":181.334},{"benchmarkId":"livebench:livebench-agentic-coding:overall","benchmarkName":"LiveBench Agentic Coding","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":58.282999999999994,"unit":"percent","family":"livebench:livebench-agentic-coding:overall","index":69.2689556051889,"aboveFit":false,"run":"high effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Agentic%20Coding","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.8334597222222222,"latency":null},{"benchmarkId":"vals-ai:proof_bench:overall","benchmarkName":"ProofBench v1.1","category":"math","domain":null,"eligible":true,"ranked":true,"score":58,"unit":"percent","family":"vals-ai:proof_bench:overall","index":67.18081324818813,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/proof_bench","measuredAt":"2026-11-09","stderr":4.96,"costPerTest":0.558508,"latency":423.698},{"benchmarkId":"aaSciCode","benchmarkName":"Artificial Analysis SciCode","category":"coding","score":57.2,"unit":"percent","family":"scicode-family","index":70.41093201743399,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/scicode","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-browser:overall","benchmarkName":"LiveBench Browser Use","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":56,"unit":"percent","family":"livebench-2-agentic","index":59.54401289791579,"aboveFit":false,"run":"high effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Browser","measuredAt":"2026-10-14","stderr":null,"costPerTest":1.069328,"latency":null},{"benchmarkId":"omniscienceAccuracy","benchmarkName":"Artificial Analysis Omniscience Accuracy","category":"knowledge","score":55.3,"unit":"percent","family":"omniscienceAccuracy","index":80.38210638511694,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hleVerified","benchmarkName":"HLE-Verified","category":"knowledge","score":53.6,"unit":"percent","family":"hleVerified","index":null,"aboveFit":false,"run":null,"source":"Weiqi Zhai, Zhihai Wang, Jinghang Wang, Boyu Yang, Xiaogang Li, Xiang Xu, Bohan Wang, Peng Wang, Xingzhe Wu, Anfeng Li, Qiyuan Feng, Yuhao Zhou, Shoulin Han, Wenjie Luo, Yiyuan Li, Yaxuan Wang, Ruixian Luo, Guojie Lin, Peiyao Xiao, Chengliang Xu, Ben Wang, Zeyu Wang, Zichao Chen, Jianan Ye, Yijie Hu, Jialong Chen, Zongwen Shen, Yuliang Xu, An Yang, Bowen Yu, Dayiheng Liu, Junyang Lin, Hu Wei, Que Shen, Bing Zhao","sourceUrl":"https://arxiv.org/abs/2602.13964","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:medcode:overall","benchmarkName":"MedCode","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":53.39,"unit":"percent","family":"vals-ai:medcode:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medcode","measuredAt":"2026-10-29","stderr":2.12,"costPerTest":0.038331,"latency":9.164},{"benchmarkId":"osWorld2","benchmarkName":"OSWorld 2.0","category":"agentic","score":47.9,"unit":"percent","family":"osWorld2","index":73.43862738708638,"aboveFit":false,"run":null,"source":"Mengqi Yuan, Zilong Zhou, Xinzhuang Xiong, Weiming Wu, Jiayang Sun, Jiamin Song, Kaiqian Cui, Bowen Wang, Haoyuan Wu, Yitong Li, Dunjie Lu, Haikong Lu, Qi Zhen, Xinyuan Wang, Jiaqi Deng, Yuhao Yang, Cheng Chen, Boyuan Zheng, Alex Su, Xiao Yu, Hao Zou, Saaket Agashe, Xing Han Lu, Manpreet Kaur, Zhengyang Qi, Vincent Sunn Chen, Frederic Sala, Dayiheng Liu, Junyang Lin, Zhou Yu, Yu Su, Siva Reddy, Xin Eric Wang, Peng Qi, Tianbao Xie, Tao Yu","sourceUrl":"https://arxiv.org/abs/2606.29537","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":47.9,"unit":"percent","family":"hle","index":72.44436680024063,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gdpvalAaNormalized","benchmarkName":"GDPval-AA normalized","category":"agentic","score":44.6,"unit":"percent","family":"gdpvalAaNormalized","index":65.9217384886485,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:cyber:overall","benchmarkName":"CyberBench v1.1","category":null,"domain":"Other","eligible":false,"ranked":true,"score":43.75,"unit":"percent","family":"vals-ai:cyber:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/cyber","measuredAt":"2026-11-05","stderr":2.21,"costPerTest":2.146121,"latency":453.761},{"benchmarkId":"frontierCode","benchmarkName":"FrontierCode 1.1 Main","category":"coding","score":43.6,"unit":"percent","family":"frontierCode","index":68.47608521868456,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"bioMysteryBenchHumanDifficult","benchmarkName":"BioMysteryBench Human Difficult","category":"knowledge","score":43.5,"unit":"percent","family":"bioMysteryBenchHumanDifficult","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-mirrorcode:overall","benchmarkName":"MirrorCode","category":"coding","domain":null,"eligible":true,"ranked":true,"score":41.11111111111111,"unit":"percent","family":"epoch-ai:epoch-mirrorcode:overall","index":67.62045997253124,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":12.157113738962925,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-mystery-game-puzzles:overall","benchmarkName":"Mystery Game Puzzles","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":37,"unit":"percent","family":"epoch-ai:epoch-mystery-game-puzzles:overall","index":67.33434938016154,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":4.8523658709390975,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tier-4-v2-private:overall","benchmarkName":"FrontierMath-Tier-4-v2-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":36.58536585365854,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tier-4-v2-private:overall","index":62.860866892097555,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":7.6158512175590225,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:code-migration:overall","benchmarkName":"Code Migration","category":"coding","domain":null,"eligible":true,"ranked":true,"score":34.8,"unit":"percent","family":"vals-ai:code-migration:overall","index":63.67629270544388,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/code-migration","measuredAt":"2026-10-29","stderr":4.225,"costPerTest":21.464006,"latency":9520.131},{"benchmarkId":"vals-ai:legal_research:overall","benchmarkName":"Legal Research Bench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":34.615,"unit":"percent","family":"vals-ai:legal_research:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_research","measuredAt":"2026-11-09","stderr":3.307,"costPerTest":0.760397,"latency":288.96},{"benchmarkId":"automationBench","benchmarkName":"AutomationBench","category":"agentic","score":30.4,"unit":"percent","family":"automationBench","index":39.914178490173455,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-long-horizon:overall","benchmarkName":"LiveBench Long Agentic","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":28.3,"unit":"percent","family":"livebench-2-agentic","index":39.28614156861913,"aboveFit":false,"run":"high effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Long%20Agentic","measuredAt":"2026-10-14","stderr":null,"costPerTest":5.558494444444444,"latency":null},{"benchmarkId":"epoch-ai:epoch-furniture-assembly:overall","benchmarkName":"Furniture Assembly","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":26.666666666666668,"unit":"percent","family":"epoch-ai:epoch-furniture-assembly:overall","index":59.09490200018625,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":5.6331678510110805,"costPerTest":null,"latency":null},{"benchmarkId":"agentsLastExam","benchmarkName":"Agents' Last Exam","category":"agentic","score":26.3,"unit":"percent","family":"agentsLastExam","index":62.891843635136006,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:tax_agent_bench:overall","benchmarkName":"Tax Agent Bench","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":22.798,"unit":"percent","family":"vals-ai:tax_agent_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench","measuredAt":"2026-11-09","stderr":3.028,"costPerTest":0.483978,"latency":87.24},{"benchmarkId":"frontierSweV2","benchmarkName":"FrontierSWE v2","category":"coding","score":20.3,"unit":"percent","family":"frontierSweV2","index":63.002382618965854,"aboveFit":false,"run":null,"source":"Proximal","sourceUrl":"https://www.frontierswe.com/blog/v2","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"apprenticeBench","benchmarkName":"ApprenticeBench: end-to-end computer use, continual learning, and long-horizon agency on a real accounts-payable job","category":"agentic","score":16,"unit":"percent","family":"apprenticeBench","index":66.3197592285453,"aboveFit":false,"run":null,"source":"NeoCognition","sourceUrl":"https://neocognition.io/blog/apprentice-bench/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"frontierBench","benchmarkName":"Terminal-Bench 3.0","category":"agentic","score":14.9,"unit":"percent","family":"frontierBench","index":63.62317056407581,"aboveFit":false,"run":null,"source":"Ryan Marten, Alex Shaw, Andy Konwinski, Harbor, and the Laude Institute","sourceUrl":"https://www.frontierbench.ai/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":14.3,"unit":"percent","family":"critpt","index":64.532247441567,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"terminal-bench:terminal-bench-4-1-0:overall","benchmarkName":"Terminal-Bench 4.1.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":12.42,"unit":"percent","family":"terminal-bench:terminal-bench-4-1-0:overall","index":58.77368602905727,"aboveFit":false,"run":"high effort · mini-SWE-agent","source":"Terminal-Bench","sourceUrl":"https://www.tbench.ai/leaderboard","measuredAt":"2026-09-29","stderr":1.2857142857142858,"costPerTest":3.8238484848484844,"latency":1688.9},{"benchmarkId":"vals-ai:terminal-bench-4:overall","benchmarkName":"Terminal-Bench 4.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":12.121,"unit":"percent","family":"terminal-bench-4","index":60.27153251728368,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-4","measuredAt":"2026-11-09","stderr":3.03,"costPerTest":8.705677,"latency":5026.717},{"benchmarkId":"terminal-bench:terminal-bench-4-0-0:overall","benchmarkName":"Terminal-Bench 4.0.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":11.21,"unit":"percent","family":"terminal-bench-4","index":59.83895452406111,"aboveFit":false,"run":"high effort · mini-SWE-agent","source":"Terminal-Bench","sourceUrl":"https://www.tbench.ai/leaderboard","measuredAt":"2026-09-29","stderr":1.2500000000000002,"costPerTest":3.8238484848484844,"latency":1688.9},{"benchmarkId":"vals-ai:terminal-bench-science:overall","benchmarkName":"Terminal-Bench-Science","category":null,"domain":"Science","eligible":false,"ranked":true,"score":7.143,"unit":"percent","family":"vals-ai:terminal-bench-science:overall","index":null,"aboveFit":false,"run":"Mini-SWE-agent","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-science","measuredAt":"2026-11-09","stderr":3.1,"costPerTest":7.190413,"latency":4825.144},{"benchmarkId":"vals-ai:srebench:overall","benchmarkName":"SRE Bench","category":null,"domain":"Other","eligible":false,"ranked":true,"score":4.58,"unit":"percent","family":"vals-ai:srebench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/srebench","measuredAt":"2026-11-05","stderr":1.294,"costPerTest":21.474171,"latency":6606.13},{"benchmarkId":"mazeBench","benchmarkName":"MazeBench: visual spatial reasoning in a 3D open world","category":"agentic","score":1,"unit":"percent","family":"mazeBench","index":63.79865130242642,"aboveFit":false,"run":null,"source":"Jonathan Pappas and David Pappas","sourceUrl":"https://mazebench.com/blog?post=maze-bench-results","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:programbench:overall","benchmarkName":"ProgramBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":0,"unit":"percent","family":"vals-ai:programbench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/programbench","measuredAt":"2026-11-09","stderr":0,"costPerTest":7.130366,"latency":1290.211},{"benchmarkId":"lmarena:agent-task-outcome:overall","benchmarkName":"Agent Arena task outcome","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":-3.4723863780244724,"unit":"index","family":"agent-arena","index":63.6759553372781,"aboveFit":false,"run":"high effort","source":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset","measuredAt":"2026-11-11","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lmarena:agent-bash-recovery:overall","benchmarkName":"Agent Arena command recovery","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":-5.443636647576189,"unit":"index","family":"agent-arena","index":62.05527858018459,"aboveFit":false,"run":"high effort","source":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset","measuredAt":"2026-10-08","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lmarena:agent-steerability:overall","benchmarkName":"Agent Arena steerability","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":-7.347897858383345,"unit":"index","family":"agent-arena","index":60.4896773313428,"aboveFit":false,"run":"high effort","source":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset","measuredAt":"2026-11-11","stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":0.75,"outputPerMillion":3.75,"batch":{"inputPerMillion":0.375,"outputPerMillion":1.875},"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":85,"timeToFirstToken":2.4105,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":{"elo":1488.8172106766838,"eloUpper":1493.4922054948925,"eloLower":1484.142215858475,"votes":33300,"rank":15,"rankUpper":null,"rankLower":null,"arenaName":"Gemini 3.7 Flash-high","organization":"google","updatedAt":"2026-11-09"},"quality":{"coveragePercent":100,"benchmarkCount":53,"categoryCount":8,"confidence":"high","state":"available"},"hosts":[{"host":"google-ai-studio","name":"Google AI Studio","kind":"maker","url":"https://ai.google.dev/gemini-api/docs/models","inputPerMillion":0.75,"outputPerMillion":3.75,"cacheReadPerMillion":0.075,"contextTokens":1048576,"maxOutputTokens":65536,"deprecated":false},{"host":"google-vertex","name":"Google Vertex AI","kind":"maker","url":"https://cloud.google.com/vertex-ai/generative-ai/docs/models","inputPerMillion":0.75,"outputPerMillion":3.75,"cacheReadPerMillion":0.075,"contextTokens":1048576,"maxOutputTokens":65536,"deprecated":false},{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":0.75,"outputPerMillion":3.75,"cacheReadPerMillion":0.075,"contextTokens":1048576,"maxOutputTokens":65536,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:arc-prize-foundation","name":"ARC Prize Foundation","url":"https://arcprize.org","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:livebench","name":"LiveBench","url":"https://livebench.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:terminal-bench","name":"Terminal-Bench","url":"https://www.tbench.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:lmarena","name":"LMArena","url":"https://huggingface.co","license":"CC BY 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"step-5-preview","providerId":"stepfun","slug":"step-5-preview","name":"Step 5 Preview","provider":"StepFun","description":"Step 5 Preview is a reasoning model from StepFun in the Step 5 family. 37 benchmarks count toward its index, in 6 categories.","logoUrl":null,"releaseDate":"2026-09-20","contextWindowTokens":1000000,"modalities":["text","image","video"],"openWeights":true,"details":{"apiId":"stepfun/step-5-preview","maxOutputTokens":64000,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.09999999999999999,"weightsUrl":"https://huggingface.co/stepfun-ai/Step-5-Preview"},"reasoning":true,"overallScore":68.39429489260567,"overallScoreRange":{"low":64.73596326787805,"high":72.0526265173333},"badges":[],"capabilityScores":{"agentic":68.81269551624524,"coding":71.38466188146899,"reasoning":67.40549078392581,"multimodal":62.17228985382497,"knowledge":63.52259036214669,"multilingual":null,"instruction":null,"math":61.65689803232299},"capabilityCounts":{"knowledge":3,"agentic":22,"reasoning":3,"coding":6,"multimodal":2,"math":1},"benchmarks":[{"benchmarkId":"gpqaDiamond","benchmarkName":"GPQA Diamond","category":"knowledge","score":93.5,"unit":"percent","family":"gpqa-diamond","index":62.96474584284943,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaHarveyLab","benchmarkName":"Artificial Analysis Harvey LAB-AA v1.0","category":"agentic","score":93.4,"unit":"percent","family":"aaHarveyLab","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"browseComp","benchmarkName":"BrowseComp","category":"agentic","score":88.7,"unit":"percent","family":"browseComp","index":69.98375084273731,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":88.3,"unit":"percent","family":"lcr","index":67.8789598683845,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mcpAtlas","benchmarkName":"MCP Atlas","category":"agentic","score":85.6,"unit":"percent","family":"mcpAtlas","index":68.70494633535353,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-coding-2:overall","benchmarkName":"LiveBench Coding 2.0","category":"coding","domain":null,"eligible":true,"ranked":true,"score":85.6,"unit":"percent","family":"livebench:livebench-coding-2:overall","index":70.1741453846882,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Coding","measuredAt":"2026-10-14","stderr":null,"costPerTest":1.0953172413793104,"latency":null},{"benchmarkId":"terminalBench21","benchmarkName":"Terminal-Bench 2.1","category":"agentic","score":85,"unit":"percent","family":"terminal-bench-2-1","index":70.85382718755811,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"cyberGym","benchmarkName":"CyberGym","category":"agentic","score":84.7,"unit":"percent","family":"cyberGym","index":69.61193972610846,"aboveFit":false,"run":null,"source":"Zhun Wang, Tianneng Shi, Jingxuan He, Matthew Cai, Jialin Zhang, Dawn Song","sourceUrl":"https://www.cybergym.io/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-reasoning:overall","benchmarkName":"LiveBench Reasoning","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":83.8,"unit":"percent","family":"livebench","index":70.17709021250084,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Reasoning","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.048211178217821775,"latency":null},{"benchmarkId":"draco","benchmarkName":"Data Research and Analysis with Complex Operations","category":"agentic","score":83.3,"unit":"percent","family":"draco","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"programBench","benchmarkName":"ProgramBench: Can Language Models Rebuild Programs From Scratch?","category":"coding","score":80.5,"unit":"percent","family":"programBench","index":91.55703946066714,"aboveFit":false,"run":null,"source":"John Yang, Kilian Lieret, Jeffrey Ma, Parth Thakkar, Dmitrii Pedchenko, Sten Sootla, Emily McMilin, Pengcheng Yin, Rui Hou, Gabriel Synnaeve, Diyi Yang, Ofir Press","sourceUrl":"https://programbench.com/static/paper.pdf","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-design:overall","benchmarkName":"LiveBench Design","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":76.9,"unit":"percent","family":"livebench-2-agentic","index":74.82883278175335,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Design","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.7456879999999999,"latency":null},{"benchmarkId":"aaMmmuPro","benchmarkName":"Artificial Analysis MMMU-Pro","category":"multimodal","score":76.4,"unit":"percent","family":"mmmu-pro","index":54.580712082939904,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmmu-pro","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmmuPro","benchmarkName":"Massive Multi-discipline Multimodal Understanding Pro","category":"multimodal","score":76,"unit":"percent","family":"mmmu-pro","index":54.06014717689223,"aboveFit":false,"run":null,"source":"MMMU-Pro authors","sourceUrl":"https://arxiv.org/abs/2409.02813","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaCyberGymE2e","benchmarkName":"Artificial Analysis CyberGym-E2E-AA","category":"agentic","score":74.8,"unit":"percent","family":"aaCyberGymE2e","index":95,"aboveFit":true,"run":null,"source":"Artificial Analysis; dataset by Berkeley RDI","sourceUrl":"https://artificialanalysis.ai/evaluations/cybergym-e2e-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"toolathlonVerified","benchmarkName":"Toolathlon-Verified","category":"agentic","score":74.1,"unit":"percent","family":"toolathlonVerified","index":68.74724142498236,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-research:overall","benchmarkName":"LiveBench Research","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":72.8,"unit":"percent","family":"livebench-2-agentic","index":71.83037529257948,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Research","measuredAt":"2026-10-14","stderr":null,"costPerTest":1.2289136363636364,"latency":null},{"benchmarkId":"sweMarathon","benchmarkName":"SWE-Marathon","category":"coding","score":72.7,"unit":"percent","family":"sweMarathon","index":null,"aboveFit":false,"run":null,"source":"Abundant AI and BenchFlow","sourceUrl":"https://www.swe-marathon.org/","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:vibe-code:overall","benchmarkName":"Vibe Code Bench v1.1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":70.399,"unit":"percent","family":"vals-ai:vibe-code:overall","index":67.65058784220473,"aboveFit":false,"run":"OpenHands","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","measuredAt":"2026-11-09","stderr":4.267,"costPerTest":5.856257,"latency":5466.358},{"benchmarkId":"hermesBench","benchmarkName":"Hermes Bench: 150 agent tasks graded on the files and state the agent leaves behind","category":"agentic","score":69.2,"unit":"percent","family":"hermesBench","index":null,"aboveFit":false,"run":null,"source":"Nous Research","sourceUrl":"https://portal.nousresearch.com/bench","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"deepSwe","benchmarkName":"DeepSWE","category":"agentic","score":67.7,"unit":"percent","family":"deepSwe","index":70.2927307192569,"aboveFit":false,"run":null,"source":"Datacurve AI","sourceUrl":"https://deepswe.datacurve.ai/blog","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:biomysterybench:overall","benchmarkName":"BioMysteryBench","category":null,"domain":"Science","eligible":false,"ranked":true,"score":67.407,"unit":"percent","family":"vals-ai:biomysterybench:overall","index":null,"aboveFit":false,"run":"Terminus 2","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/biomysterybench","measuredAt":"2026-11-09","stderr":0.37,"costPerTest":0.494625,"latency":1294.751},{"benchmarkId":"livebench:livebench-data-analysis:overall","benchmarkName":"LiveBench Data Analysis","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":63.5,"unit":"percent","family":"livebench","index":43.87978009072574,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Data%20Analysis","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.06389812666666667,"latency":null},{"benchmarkId":"officeQaPro","benchmarkName":"OfficeQA Pro","category":"multimodal","score":60.3,"unit":"percent","family":"officeQaPro","index":70.02415007773388,"aboveFit":false,"run":null,"source":"OfficeQA Pro authors","sourceUrl":"https://arxiv.org/abs/2603.08655","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"jobBench","benchmarkName":"JobBench","category":"agentic","score":59,"unit":"percent","family":"jobBench","index":69.12644379502828,"aboveFit":false,"run":null,"source":"Yuetai Li, Yichen Feng, Zhangchen Xu, Zixian Ma, Kaiyuan Zheng, Fengqing Jiang, Xinghua Sun, Rulin Shao, Zichen Chen, Yue Huang, Xinyang Han, Brian Lee, Kayla Xu, Shenglai Zeng, Hang Hua, Xiangliang Zhang, Basel Alomair, Ranjay Krishna, Luke Zettlemoyer, Pang Wei Koh, Bhaskar Ramasubramanian, Luyao Niu, Xiang Yue, Radha Poovendran","sourceUrl":"https://arxiv.org/abs/2605.26329","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"sciCode","benchmarkName":"Scientific Code Benchmark","category":"coding","score":58.9,"unit":"percent","family":"scicode-family","index":72.62720696633903,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaSciCode","benchmarkName":"Artificial Analysis SciCode","category":"coding","score":58.9,"unit":"percent","family":"scicode-family","index":72.62720696633903,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/scicode","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:public-benefits-bench:overall","benchmarkName":"Public Benefits Bench v1.1","category":null,"domain":"Public benefits","eligible":false,"ranked":true,"score":58.119,"unit":"percent","family":"vals-ai:public-benefits-bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/public-benefits-bench","measuredAt":"2026-10-12","stderr":1.283,"costPerTest":0.504407,"latency":1639.157},{"benchmarkId":"vals-ai:emb:overall","benchmarkName":"EMB","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":57.116,"unit":"percent","family":"vals-ai:emb:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/emb","measuredAt":"2026-10-20","stderr":3.165,"costPerTest":1.329318,"latency":2615.096},{"benchmarkId":"aaItbench","benchmarkName":"Artificial Analysis ITBench-AA","category":"agentic","score":55.6,"unit":"percent","family":"aaItbench","index":79.49338789241025,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/itbench-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gdpvalAaNormalized","benchmarkName":"GDPval-AA normalized","category":"agentic","score":54.3,"unit":"percent","family":"gdpvalAaNormalized","index":72.85701307705926,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaCweBench","benchmarkName":"Artificial Analysis CWE-Bench-AA","category":"agentic","score":52.5,"unit":"percent","family":"aaCweBench","index":65.51830280627749,"aboveFit":false,"run":null,"source":"Artificial Analysis; dataset by Collinear AI","sourceUrl":"https://artificialanalysis.ai/evaluations/cwe-bench-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:fabv2:overall","benchmarkName":"Finance Agent (v2)","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":50.671,"unit":"percent","family":"vals-ai:fabv2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/fabv2","measuredAt":"2026-10-29","stderr":0.324,"costPerTest":0.619283,"latency":747.585},{"benchmarkId":"aaCyberIndex","benchmarkName":"Artificial Analysis Cyber Index","category":"agentic","score":47.7,"unit":"percent","family":"aaCyberIndex","index":76.20126844052263,"aboveFit":false,"run":null,"source":"Artificial Analysis with the Cyber Index Alliance (Collinear AI, IBM, NVIDIA, Pasta Lab, Vercel)","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-cyber-index","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaEnterpriseOpsGym","benchmarkName":"Artificial Analysis EnterpriseOps-Gym","category":"agentic","score":47.2,"unit":"percent","family":"aaEnterpriseOpsGym","index":null,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/enterprise-ops-gym-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hle","benchmarkName":"Humanity's Last Exam","category":"knowledge","score":46.5,"unit":"percent","family":"hle","index":71.03975695719892,"aboveFit":false,"run":null,"source":"Center for AI Safety, Scale AI, and thousands of expert contributors","sourceUrl":"https://lastexam.ai/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":46.5,"unit":"percent","family":"hle","index":71.03975695719892,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"automationBench","benchmarkName":"AutomationBench","category":"agentic","score":44,"unit":"percent","family":"automationBench","index":65.78252054233977,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:proof_bench:overall","benchmarkName":"ProofBench v1.1","category":"math","domain":null,"eligible":true,"ranked":true,"score":42,"unit":"percent","family":"vals-ai:proof_bench:overall","index":61.65689803232299,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/proof_bench","measuredAt":"2026-11-09","stderr":4.96,"costPerTest":0.714564,"latency":2313.592},{"benchmarkId":"mlsBenchLite","benchmarkName":"MLS-Bench Lite","category":"coding","score":40.5,"unit":"percent","family":"mlsBenchLite","index":null,"aboveFit":false,"run":null,"source":"MLS-Bench","sourceUrl":"https://mls-bench.com/","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:legal_research:overall","benchmarkName":"Legal Research Bench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":40.385,"unit":"percent","family":"vals-ai:legal_research:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_research","measuredAt":"2026-11-09","stderr":3.41,"costPerTest":0.838645,"latency":2168.413},{"benchmarkId":"vals-ai:code-migration:overall","benchmarkName":"Code Migration","category":"coding","domain":null,"eligible":true,"ranked":true,"score":38.977,"unit":"percent","family":"vals-ai:code-migration:overall","index":65.89528257683466,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/code-migration","measuredAt":"2026-10-29","stderr":4.533,"costPerTest":7.606885,"latency":12856.895},{"benchmarkId":"vals-ai:ioi:overall","benchmarkName":"IOI","category":"coding","domain":null,"eligible":true,"ranked":true,"score":38.056,"unit":"percent","family":"vals-ai:ioi:overall","index":60.40370905808016,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/ioi","measuredAt":"2026-10-29","stderr":6.69,"costPerTest":2.025817,"latency":4820.902},{"benchmarkId":"apexAgentsAa","benchmarkName":"APEX-Agents-AA","category":"agentic","score":38,"unit":"percent","family":"apexAgentsAa","index":68.18980434958736,"aboveFit":false,"run":null,"source":"Artificial Analysis / Mercor","sourceUrl":"https://artificialanalysis.ai/evaluations/apex-agents-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"apexAgents","benchmarkName":"APEX-Agents","category":"agentic","score":37.8,"unit":"percent","family":"apexAgents","index":63.162517324287215,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"omniscienceAccuracy","benchmarkName":"Artificial Analysis Omniscience Accuracy","category":"knowledge","score":36.4,"unit":"percent","family":"omniscienceAccuracy","index":56.563268286391704,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hermesIndex","benchmarkName":"Hermes Index: agentic capability and cost per task inside Hermes Agent","category":"agentic","score":32.83,"unit":"percent","family":"hermesIndex","index":63.04785162093387,"aboveFit":false,"run":null,"source":"Nous Research","sourceUrl":"https://portal.nousresearch.com/bench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:terminal-bench-4:overall","benchmarkName":"Terminal-Bench 4.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":31.818,"unit":"percent","family":"terminal-bench-4","index":69.62442904034089,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-4","measuredAt":"2026-11-09","stderr":2.624,"costPerTest":3.113224,"latency":7066.968},{"benchmarkId":"agentsLastExam","benchmarkName":"Agents' Last Exam","category":"agentic","score":29.5,"unit":"percent","family":"agentsLastExam","index":65.5483564694289,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mka","benchmarkName":"Multimodal Knowledge Agent Benchmark","category":"agentic","score":27.6,"unit":"percent","family":"mka","index":58.63475686625052,"aboveFit":false,"run":null,"source":"Google DeepMind, in partnership with Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mka","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-long-horizon:overall","benchmarkName":"LiveBench Long Agentic","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":27.5,"unit":"percent","family":"livebench-2-agentic","index":38.70107669268277,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Long%20Agentic","measuredAt":"2026-10-14","stderr":null,"costPerTest":10.215691666666666,"latency":null},{"benchmarkId":"vals-ai:tax_agent_bench:overall","benchmarkName":"Tax Agent Bench","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":27.273,"unit":"percent","family":"vals-ai:tax_agent_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench","measuredAt":"2026-11-09","stderr":3.113,"costPerTest":0.55338,"latency":1288.515},{"benchmarkId":"livebench:livebench-browser:overall","benchmarkName":"LiveBench Browser Use","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":24,"unit":"percent","family":"livebench-2-agentic","index":36.14141786046117,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Browser","measuredAt":"2026-10-14","stderr":null,"costPerTest":2.791736,"latency":null},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":20.9,"unit":"percent","family":"critpt","index":77.30907733177966,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:mysterymechanism:overall","benchmarkName":"MysteryMechanism","category":null,"domain":"Science","eligible":false,"ranked":true,"score":18.468,"unit":"percent","family":"vals-ai:mysterymechanism:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mysterymechanism","measuredAt":"2026-11-11","stderr":2.61,"costPerTest":0.597179,"latency":1760.418},{"benchmarkId":"aaDeepsecBench","benchmarkName":"Artificial Analysis DeepsecBench-AA","category":"agentic","score":15.8,"unit":"percent","family":"aaDeepsecBench","index":62.438072367579615,"aboveFit":false,"run":null,"source":"Artificial Analysis; dataset by Vercel","sourceUrl":"https://artificialanalysis.ai/evaluations/deepsecbench-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:hlab:overall","benchmarkName":"HLAB-Vals-1.1","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":12.917,"unit":"percent","family":"vals-ai:hlab:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/hlab","measuredAt":"2026-11-09","stderr":2.807,"costPerTest":1.12706,"latency":993.244},{"benchmarkId":"vals-ai:terminal-bench-science:overall","benchmarkName":"Terminal-Bench-Science","category":null,"domain":"Science","eligible":false,"ranked":true,"score":2.857,"unit":"percent","family":"vals-ai:terminal-bench-science:overall","index":null,"aboveFit":false,"run":"Mini-SWE-agent","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-science","measuredAt":"2026-11-09","stderr":2.006,"costPerTest":4.250933,"latency":8997.76},{"benchmarkId":"vals-ai:srebench:overall","benchmarkName":"SRE Bench","category":null,"domain":"Other","eligible":false,"ranked":true,"score":0.382,"unit":"percent","family":"vals-ai:srebench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/srebench","measuredAt":"2026-11-05","stderr":0.382,"costPerTest":3.06713,"latency":5625.323},{"benchmarkId":"lmarena:agent-bash-recovery:overall","benchmarkName":"Agent Arena command recovery","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":0.22841904814647762,"unit":"index","family":"agent-arena","index":66.71859757777085,"aboveFit":false,"run":null,"source":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset","measuredAt":"2026-10-08","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lmarena:agent-steerability:overall","benchmarkName":"Agent Arena steerability","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":-0.8956768339035914,"unit":"index","family":"agent-arena","index":65.7944145367218,"aboveFit":false,"run":null,"source":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset","measuredAt":"2026-11-11","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lmarena:agent-task-outcome:overall","benchmarkName":"Agent Arena task outcome","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":-1.4926886755538105,"unit":"index","family":"agent-arena","index":65.30357720854626,"aboveFit":false,"run":null,"source":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset","measuredAt":"2026-11-11","stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":1,"outputPerMillion":2.7,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":105,"timeToFirstToken":3.484,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":{"elo":1443.1071976314038,"eloUpper":1449.3475475397868,"eloLower":1436.8668477230208,"votes":12362,"rank":70,"rankUpper":null,"rankLower":null,"arenaName":"Step 5 Preview","organization":"stepfun","updatedAt":"2026-11-09"},"quality":{"coveragePercent":90,"benchmarkCount":37,"categoryCount":6,"confidence":"medium","state":"available"},"hosts":[{"host":"deepinfra","name":"Deep Infra","kind":"host","url":"https://deepinfra.com/models","inputPerMillion":1,"outputPerMillion":2.7,"cacheReadPerMillion":0.1,"contextTokens":1048576,"maxOutputTokens":1000000,"deprecated":true},{"host":"novita-ai","name":"Novita AI","kind":"host","url":"https://novita.ai/docs/guides/introduction","inputPerMillion":1,"outputPerMillion":2.7,"cacheReadPerMillion":0.1,"contextTokens":1000000,"maxOutputTokens":64000,"deprecated":false},{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":1,"outputPerMillion":2.7,"cacheReadPerMillion":0.1,"contextTokens":1000000,"maxOutputTokens":64000,"deprecated":false},{"host":"stepfun","name":"StepFun","kind":"maker","url":"https://platform.stepfun.ai/docs/en/overview/concept","inputPerMillion":1,"outputPerMillion":2.7,"cacheReadPerMillion":0.1,"contextTokens":1024000,"maxOutputTokens":65536,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:livebench","name":"LiveBench","url":"https://livebench.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:lmarena","name":"LMArena","url":"https://huggingface.co","license":"CC BY 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gemini-3-8-flash","providerId":"google","slug":"gemini-3-8-flash","name":"Gemini 3.8 Flash","provider":"Google","description":"Gemini 3.8 Flash is a reasoning model from Google. 66 benchmarks count toward its index, in 7 categories.","logoUrl":null,"releaseDate":"2026-09-02","contextWindowTokens":1048576,"modalities":["text","image","video","file","audio"],"openWeights":false,"details":{"apiId":"google/gemini-3.8-flash","maxOutputTokens":65536,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.0825,"weightsUrl":null},"reasoning":true,"overallScore":67.84981508280814,"overallScoreRange":{"low":65.35942567367232,"high":70.34020449194395},"badges":[],"capabilityScores":{"agentic":65.33213625431927,"coding":66.15337242841697,"reasoning":68.2525690715614,"multimodal":67.114100269464,"knowledge":70.87696668159273,"multilingual":null,"instruction":91.32010640848202,"math":64.31803726250128},"capabilityCounts":{"math":5,"reasoning":10,"knowledge":6,"coding":16,"agentic":24,"multimodal":4,"instruction":1},"benchmarks":[{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":98.88888888888889,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":66.45205762128103,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0.8724007409153889,"costPerTest":null,"latency":null},{"benchmarkId":"arc-prize:v1-semi-private:overall","benchmarkName":"ARC-AGI-1 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":98.5,"unit":"percent","family":"arc-prize:v1-semi-private:overall","index":72.27136500316884,"aboveFit":false,"run":"high effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.20570699249999985,"latency":null},{"benchmarkId":"arc-prize:v1-public-eval:overall","benchmarkName":"ARC-AGI-1 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":98,"unit":"percent","family":"arc-prize:v1-public-eval:overall","index":null,"aboveFit":false,"run":"high effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.12971782687499991,"latency":null},{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":95.39141414141415,"unit":"percent","family":"gpqa-diamond","index":64.69949921031134,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.401136318210652,"costPerTest":null,"latency":null},{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":95.3,"unit":"percent","family":"gpqa-diamond","index":64.61565665770505,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:gpqa:overall","benchmarkName":"GPQA Diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":94.444,"unit":"percent","family":"gpqa-diamond","index":63.83055684797371,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","measuredAt":"2026-09-01","stderr":1.477,"costPerTest":0.056302,"latency":20.18},{"benchmarkId":"arc-prize:v2-public-eval:overall","benchmarkName":"ARC-AGI-2 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":92.5,"unit":"percent","family":"arc-prize:v2-public-eval:overall","index":null,"aboveFit":false,"run":"high effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.41753459999999976,"latency":null},{"benchmarkId":"livebench:livebench-math:overall","benchmarkName":"LiveBench Mathematics","category":"math","domain":null,"eligible":true,"ranked":true,"score":91.56425,"unit":"percent","family":"livebench:livebench-math:overall","index":67.38968607198007,"aboveFit":false,"run":"high effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Mathematics","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.12769498680738786,"latency":null},{"benchmarkId":"vals-ai:mmlu_pro:overall","benchmarkName":"MMLU Pro","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":90.221,"unit":"percent","family":"mmlu-pro","index":58.148389250846364,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","measuredAt":"2026-09-01","stderr":0.295,"costPerTest":0.022457,"latency":7.82},{"benchmarkId":"vals-ai:lcb:overall","benchmarkName":"LiveCodeBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":89.484,"unit":"percent","family":"livecodebench","index":63.592330639579885,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","measuredAt":"2026-09-01","stderr":0.897,"costPerTest":0.07124,"latency":24.07},{"benchmarkId":"terminalBench21","benchmarkName":"Terminal-Bench 2.1","category":"agentic","score":89.4,"unit":"percent","family":"terminal-bench-2-1","index":73.42719079123921,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-reasoning:overall","benchmarkName":"LiveBench Reasoning","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":89.29325,"unit":"percent","family":"livebench","index":77.29323301183291,"aboveFit":false,"run":"high effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Reasoning","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.14755148514851485,"latency":null},{"benchmarkId":"arc-prize:v2-semi-private:overall","benchmarkName":"ARC-AGI-2 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":89.16666666666667,"unit":"percent","family":"arc-prize:v2-semi-private:overall","index":80.12802623928313,"aboveFit":false,"run":"high effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.39972941250000005,"latency":null},{"benchmarkId":"vals-ai:mmmu:overall","benchmarkName":"MMMU Pro","category":"multimodal","domain":null,"eligible":true,"ranked":true,"score":89.075,"unit":"percent","family":"mmmu-pro","index":71.07611254332531,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmmu","measuredAt":"2026-09-01","stderr":0.75,"costPerTest":0.004472,"latency":12.028},{"benchmarkId":"bioMysteryBenchHumanSolvable","benchmarkName":"BioMysteryBench Human Solvable","category":"knowledge","score":88.8,"unit":"percent","family":"bioMysteryBenchHumanSolvable","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:sage:overall","benchmarkName":"SAGE","category":null,"domain":"Education","eligible":false,"ranked":true,"score":88.242,"unit":"percent","family":"vals-ai:sage:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/sage","measuredAt":"2026-11-05","stderr":1.718,"costPerTest":0.076449,"latency":67.459},{"benchmarkId":"livebench:livebench-language:overall","benchmarkName":"LiveBench Language","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":87.79266666666666,"unit":"percent","family":"livebench:livebench-language:overall","index":73.83553368738195,"aboveFit":false,"run":"high effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Language","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.08808933333333334,"latency":null},{"benchmarkId":"aaTerminalBench21","benchmarkName":"Artificial Analysis Terminal-Bench v2.1","category":"agentic","score":87.6,"unit":"percent","family":"aaTerminalBench21","index":70.24369312577902,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/terminalbench-v2-1","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lvBench","benchmarkName":"LVBench","category":"multimodal","score":87.1,"unit":"percent","family":"lvBench","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:legal_bench:overall","benchmarkName":"LegalBench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":86.993,"unit":"percent","family":"vals-ai:legal_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","measuredAt":"2026-10-13","stderr":0.432,"costPerTest":0.008109,"latency":3.32},{"benchmarkId":"charxivNoTools","benchmarkName":"CharXiv Reasoning without tools","category":"multimodal","score":86.2,"unit":"percent","family":"charxivNoTools","index":67.47578933243959,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"labBench2","benchmarkName":"LABBench2: An Improved Benchmark for AI Systems Performing Biology Research","category":"knowledge","score":86.2,"unit":"percent","family":"labBench2","index":null,"aboveFit":false,"run":null,"source":"Jon M. Laurent, Albert Bou, Michael Pieler, Conor Igoe, Alex Andonian, Siddharth Narayanan, James Braza, Alexandros Sanchez Vassopoulos, Jacob L. Steenwyk, Blake Lash, Andrew D. White, Samuel G. Rodriques","sourceUrl":"https://arxiv.org/abs/2604.09554","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaMmmuPro","benchmarkName":"Artificial Analysis MMMU-Pro","category":"multimodal","score":85.6,"unit":"percent","family":"mmmu-pro","index":66.5537049220362,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmmu-pro","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:medscribe:overall","benchmarkName":"MedScribe","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":84.496,"unit":"percent","family":"vals-ai:medscribe:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medscribe","measuredAt":"2026-10-29","stderr":1.943,"costPerTest":0.025238,"latency":21.145},{"benchmarkId":"livebench:livebench-coding-2:overall","benchmarkName":"LiveBench Coding 2.0","category":"coding","domain":null,"eligible":true,"ranked":true,"score":82.2,"unit":"percent","family":"livebench:livebench-coding-2:overall","index":67.71144012624049,"aboveFit":false,"run":"high effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Coding","measuredAt":"2026-10-14","stderr":null,"costPerTest":2.296427586206897,"latency":null},{"benchmarkId":"livebench:livebench-instruction-following:overall","benchmarkName":"LiveBench Instruction Following","category":"instruction","domain":null,"eligible":true,"ranked":true,"score":81.4125,"unit":"percent","family":"livebench:livebench-instruction-following:overall","index":91.32010640848202,"aboveFit":true,"run":"high effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=IF","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.169038,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":81.3,"unit":"percent","family":"lcr","index":63.07639618928488,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:terminal-bench-2-1:overall","benchmarkName":"Terminal-Bench 2.1","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":81.273,"unit":"percent","family":"terminal-bench-2-1","index":68.67407124416732,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2-1","measuredAt":"2026-10-01","stderr":0.375,"costPerTest":1.544972,"latency":554.797},{"benchmarkId":"extractBenchLlamaIndex","benchmarkName":"ExtractBench: schema-guided enterprise document extraction (LlamaIndex leaderboard)","category":"multimodal","score":80.71,"unit":"percent","family":"extractBenchLlamaIndex","index":59.10448987312972,"aboveFit":false,"run":null,"source":"Boyang Zhang, Adrian Lyjak, Eli Stewart, Zhaoqi Li and Simon Suo (LlamaIndex)","sourceUrl":"https://arxiv.org/abs/2607.29677","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:swebench:overall","benchmarkName":"SWE-bench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":80,"unit":"percent","family":"swe-bench-verified","index":60.232533752167,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/swebench","measuredAt":"2026-09-01","stderr":1.791,"costPerTest":2.1911,"latency":683.446},{"benchmarkId":"livebench:livebench-research:overall","benchmarkName":"LiveBench Research","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":79.1,"unit":"percent","family":"livebench-2-agentic","index":76.43776119057834,"aboveFit":false,"run":"high effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Research","measuredAt":"2026-10-14","stderr":null,"costPerTest":2.6380545454545454,"latency":null},{"benchmarkId":"vals-ai:vibe-code:overall","benchmarkName":"Vibe Code Bench v1.1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":78.651,"unit":"percent","family":"vals-ai:vibe-code:overall","index":70.6280390908483,"aboveFit":false,"run":"OpenHands","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","measuredAt":"2026-11-09","stderr":3.884,"costPerTest":6.865141,"latency":519.104},{"benchmarkId":"vals-ai:tax_eval_v2:overall","benchmarkName":"TaxEval v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":74.448,"unit":"percent","family":"vals-ai:tax_eval_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","measuredAt":"2026-09-01","stderr":0.848,"costPerTest":0.023535,"latency":8.759},{"benchmarkId":"deepSwe","benchmarkName":"DeepSWE","category":"agentic","score":73.8,"unit":"percent","family":"deepSwe","index":74.72726043162002,"aboveFit":false,"run":null,"source":"Datacurve AI","sourceUrl":"https://deepswe.datacurve.ai/blog","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-design:overall","benchmarkName":"LiveBench Design","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":72.7,"unit":"percent","family":"livebench-2-agentic","index":71.75724218308743,"aboveFit":false,"run":"high effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Design","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.979588,"latency":null},{"benchmarkId":"livebench:livebench-coding:overall","benchmarkName":"LiveBench Coding","category":"coding","domain":null,"eligible":true,"ranked":true,"score":72.489,"unit":"percent","family":"livebench:livebench-coding:overall","index":56.77350794961107,"aboveFit":false,"run":"high effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Coding","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.18472735042735042,"latency":null},{"benchmarkId":"vals-ai:emb:overall","benchmarkName":"EMB","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":72.196,"unit":"percent","family":"vals-ai:emb:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/emb","measuredAt":"2026-10-20","stderr":2.42,"costPerTest":8.243862,"latency":769.197},{"benchmarkId":"parseBenchLlamaIndex","benchmarkName":"ParseBench: document parsing for AI agents (LlamaIndex leaderboard)","category":"multimodal","score":71.2,"unit":"percent","family":"parseBenchLlamaIndex","index":73.06121313960593,"aboveFit":false,"run":null,"source":"Boyang Zhang, Sebastián G. Acosta, Preston Carlson, Sacha Bron, Pierre-Loïc Doulcet, Daniel B. Ospina and Simon Suo (LlamaIndex)","sourceUrl":"https://arxiv.org/abs/2604.08538","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-simpleqa-verified:overall","benchmarkName":"SimpleQA Verified","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":69.69999999999999,"unit":"percent","family":"epoch-ai:epoch-simpleqa-verified:overall","index":81.98530530625422,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.4539683710535192,"costPerTest":null,"latency":null},{"benchmarkId":"cursorBench32","benchmarkName":"cursorBench32","category":"coding","score":69.2,"unit":"percent","family":"cursorBench32","index":72.69684609757846,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tiers-1-3-v2-private:overall","benchmarkName":"FrontierMath-Tiers-1-3-v2-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":68.42105263157895,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tiers-1-3-v2-private:overall","index":66.90823444681409,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.7582557383453006,"costPerTest":null,"latency":null},{"benchmarkId":"scarfBench","benchmarkName":"ScarfBench: Cross-Framework Application Migration in Enterprise Java","category":"coding","score":66,"unit":"percent","family":"scarfBench","index":71.36935369903283,"aboveFit":false,"run":null,"source":"Advait Pavuluri, Bridget McGinn, Ashita Saxena, George Safta, Srikanth Tamilselvam, Raju Pavuluri, Michele Merler, Baishakhi Ray, Rahul Krishna (IBM Research)","sourceUrl":"https://arxiv.org/abs/2605.06754","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:mortgage_tax:overall","benchmarkName":"MortgageTax","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":65.342,"unit":"percent","family":"vals-ai:mortgage_tax:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mortgage_tax","measuredAt":"2026-09-01","stderr":0.943,"costPerTest":0.046973,"latency":14.794},{"benchmarkId":"vals-ai:public-benefits-bench:overall","benchmarkName":"Public Benefits Bench v1.1","category":null,"domain":"Public benefits","eligible":false,"ranked":true,"score":65.291,"unit":"percent","family":"vals-ai:public-benefits-bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/public-benefits-bench","measuredAt":"2026-10-12","stderr":1.238,"costPerTest":1.011271,"latency":201.419},{"benchmarkId":"livebench:livebench-browser:overall","benchmarkName":"LiveBench Browser Use","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":64,"unit":"percent","family":"livebench-2-agentic","index":65.39466165727944,"aboveFit":false,"run":"high effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Browser","measuredAt":"2026-10-14","stderr":null,"costPerTest":1.213092,"latency":null},{"benchmarkId":"vals-ai:biomysterybench:overall","benchmarkName":"BioMysteryBench","category":null,"domain":"Science","eligible":false,"ranked":true,"score":62.222,"unit":"percent","family":"vals-ai:biomysterybench:overall","index":null,"aboveFit":false,"run":"Terminus 2","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/biomysterybench","measuredAt":"2026-11-09","stderr":1.697,"costPerTest":1.5539,"latency":326.755},{"benchmarkId":"hermesBench","benchmarkName":"Hermes Bench: 150 agent tasks graded on the files and state the agent leaves behind","category":"agentic","score":62.19,"unit":"percent","family":"hermesBench","index":null,"aboveFit":false,"run":null,"source":"Nous Research","sourceUrl":"https://portal.nousresearch.com/bench","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:fabv2:overall","benchmarkName":"Finance Agent (v2)","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":61.435,"unit":"percent","family":"vals-ai:fabv2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/fabv2","measuredAt":"2026-10-29","stderr":0.128,"costPerTest":2.000457,"latency":200.952},{"benchmarkId":"aaAutomationBench","benchmarkName":"Artificial Analysis AutomationBench","category":"agentic","score":59.9,"unit":"percent","family":"aaAutomationBench","index":70.28098033331072,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/automationbench-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"osWorld2","benchmarkName":"OSWorld 2.0","category":"agentic","score":59,"unit":"percent","family":"osWorld2","index":77.92670859928347,"aboveFit":false,"run":null,"source":"Mengqi Yuan, Zilong Zhou, Xinzhuang Xiong, Weiming Wu, Jiayang Sun, Jiamin Song, Kaiqian Cui, Bowen Wang, Haoyuan Wu, Yitong Li, Dunjie Lu, Haikong Lu, Qi Zhen, Xinyuan Wang, Jiaqi Deng, Yuhao Yang, Cheng Chen, Boyuan Zheng, Alex Su, Xiao Yu, Hao Zou, Saaket Agashe, Xing Han Lu, Manpreet Kaur, Zhengyang Qi, Vincent Sunn Chen, Frederic Sala, Dayiheng Liu, Junyang Lin, Zhou Yu, Yu Su, Siva Reddy, Xin Eric Wang, Peng Qi, Tianbao Xie, Tao Yu","sourceUrl":"https://arxiv.org/abs/2606.29537","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-chess-puzzles:overall","benchmarkName":"Chess Puzzles","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":57.99999999999999,"unit":"percent","family":"epoch-ai:epoch-chess-puzzles:overall","index":78.00755704384451,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":4.9604496374885825,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:skillsbench:overall","benchmarkName":"SkillsBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":57.985,"unit":"percent","family":"vals-ai:skillsbench:overall","index":68.30171888894095,"aboveFit":false,"run":"OpenHands","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/skillsbench","measuredAt":"2026-10-29","stderr":4.422,"costPerTest":2.336944,"latency":235.201},{"benchmarkId":"vals-ai:ioi:overall","benchmarkName":"IOI","category":"coding","domain":null,"eligible":true,"ranked":true,"score":56.944,"unit":"percent","family":"vals-ai:ioi:overall","index":67.49468736437052,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/ioi","measuredAt":"2026-10-29","stderr":2.257,"costPerTest":3.976052,"latency":860.465},{"benchmarkId":"aaSciCode","benchmarkName":"Artificial Analysis SciCode","category":"coding","score":56.6,"unit":"percent","family":"scicode-family","index":69.62871732958516,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/scicode","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"bioMysteryBenchHumanDifficult","benchmarkName":"BioMysteryBench Human Difficult","category":"knowledge","score":56.5,"unit":"percent","family":"bioMysteryBenchHumanDifficult","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hleVerified","benchmarkName":"HLE-Verified","category":"knowledge","score":54.9,"unit":"percent","family":"hleVerified","index":null,"aboveFit":false,"run":null,"source":"Weiqi Zhai, Zhihai Wang, Jinghang Wang, Boyu Yang, Xiaogang Li, Xiang Xu, Bohan Wang, Peng Wang, Xingzhe Wu, Anfeng Li, Qiyuan Feng, Yuhao Zhou, Shoulin Han, Wenjie Luo, Yiyuan Li, Yaxuan Wang, Ruixian Luo, Guojie Lin, Peiyao Xiao, Chengliang Xu, Ben Wang, Zeyu Wang, Zichao Chen, Jianan Ye, Yijie Hu, Jialong Chen, Zongwen Shen, Yuliang Xu, An Yang, Bowen Yu, Dayiheng Liu, Junyang Lin, Hu Wei, Que Shen, Bing Zhao","sourceUrl":"https://arxiv.org/abs/2602.13964","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-agentic-coding:overall","benchmarkName":"LiveBench Agentic Coding","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":54.242333333333335,"unit":"percent","family":"livebench:livebench-agentic-coding:overall","index":65.10188489119952,"aboveFit":false,"run":"high effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Agentic%20Coding","measuredAt":"2026-06-25","stderr":null,"costPerTest":1.5751833333333332,"latency":null},{"benchmarkId":"livebench:livebench-data-analysis:overall","benchmarkName":"LiveBench Data Analysis","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":54.00566666666666,"unit":"percent","family":"livebench","index":31.580497919978278,"aboveFit":false,"run":"high effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Data%20Analysis","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.2377133333333333,"latency":null},{"benchmarkId":"aaItbench","benchmarkName":"Artificial Analysis ITBench-AA","category":"agentic","score":52.5,"unit":"percent","family":"aaItbench","index":75.24689417642458,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/itbench-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaCweBench","benchmarkName":"Artificial Analysis CWE-Bench-AA","category":"agentic","score":52.5,"unit":"percent","family":"aaCweBench","index":65.51830280627749,"aboveFit":false,"run":null,"source":"Artificial Analysis; dataset by Collinear AI","sourceUrl":"https://artificialanalysis.ai/evaluations/cwe-bench-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"omniscienceAccuracy","benchmarkName":"Artificial Analysis Omniscience Accuracy","category":"knowledge","score":50.5,"unit":"percent","family":"omniscienceAccuracy","index":74.33287766163117,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:medcode:overall","benchmarkName":"MedCode","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":48.135,"unit":"percent","family":"vals-ai:medcode:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medcode","measuredAt":"2026-10-29","stderr":2.18,"costPerTest":0.0177,"latency":42.892},{"benchmarkId":"vals-ai:proof_bench:overall","benchmarkName":"ProofBench v1.1","category":"math","domain":null,"eligible":true,"ranked":true,"score":48,"unit":"percent","family":"vals-ai:proof_bench:overall","index":63.728366238272415,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/proof_bench","measuredAt":"2026-11-09","stderr":5.021,"costPerTest":0.602872,"latency":368.648},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":47.8,"unit":"percent","family":"hle","index":72.34403752573765,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-mystery-game-puzzles:overall","benchmarkName":"Mystery Game Puzzles","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":47,"unit":"percent","family":"epoch-ai:epoch-mystery-game-puzzles:overall","index":75.07849834721327,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":5.016135580465917,"costPerTest":null,"latency":null},{"benchmarkId":"gdpvalAaNormalized","benchmarkName":"GDPval-AA normalized","category":"agentic","score":46.8,"unit":"percent","family":"gdpvalAaNormalized","index":67.49468736437052,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaAnalystAgent","benchmarkName":"Artificial Analysis AnalystAgent","category":"agentic","score":46.3,"unit":"percent","family":"aaAnalystAgent","index":67.82972161858362,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/aa-analyst-agent","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaTau3Banking","benchmarkName":"Artificial Analysis Tau3-Banking","category":"agentic","score":44.9,"unit":"percent","family":"aaTau3Banking","index":70.06892968620139,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/tau3-banking","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:cyber:overall","benchmarkName":"CyberBench v1.1","category":null,"domain":"Other","eligible":false,"ranked":true,"score":43.75,"unit":"percent","family":"vals-ai:cyber:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/cyber","measuredAt":"2026-11-05","stderr":2.21,"costPerTest":1.963581,"latency":588.848},{"benchmarkId":"cursorBench40","benchmarkName":"cursorBench40","category":"coding","score":39.6,"unit":"percent","family":"cursorBench40","index":66.64970843831662,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:legal_research:overall","benchmarkName":"Legal Research Bench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":38.942,"unit":"percent","family":"vals-ai:legal_research:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_research","measuredAt":"2026-11-09","stderr":3.389,"costPerTest":1.630498,"latency":324.318},{"benchmarkId":"epoch-ai:epoch-mirrorcode:overall","benchmarkName":"MirrorCode","category":"coding","domain":null,"eligible":true,"ranked":true,"score":38.88888888888889,"unit":"percent","family":"epoch-ai:epoch-mirrorcode:overall","index":66.64452405295084,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":11.91533921640401,"costPerTest":null,"latency":null},{"benchmarkId":"hermesIndex","benchmarkName":"Hermes Index: agentic capability and cost per task inside Hermes Agent","category":"agentic","score":38.12,"unit":"percent","family":"hermesIndex","index":66.88698046792014,"aboveFit":false,"run":null,"source":"Nous Research","sourceUrl":"https://portal.nousresearch.com/bench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:code-migration:overall","benchmarkName":"Code Migration","category":"coding","domain":null,"eligible":true,"ranked":true,"score":36.546,"unit":"percent","family":"vals-ai:code-migration:overall","index":64.60383790904653,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/code-migration","measuredAt":"2026-10-29","stderr":4.184,"costPerTest":18.491664,"latency":9443.316},{"benchmarkId":"vals-ai:mysterymechanism:overall","benchmarkName":"MysteryMechanism","category":null,"domain":"Science","eligible":false,"ranked":true,"score":36.486,"unit":"percent","family":"vals-ai:mysterymechanism:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mysterymechanism","measuredAt":"2026-11-11","stderr":3.238,"costPerTest":1.731041,"latency":322.237},{"benchmarkId":"mka","benchmarkName":"Multimodal Knowledge Agent Benchmark","category":"agentic","score":35.3,"unit":"percent","family":"mka","index":65.2677296972091,"aboveFit":false,"run":null,"source":"Google DeepMind, in partnership with Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mka","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:tax_agent_bench:overall","benchmarkName":"Tax Agent Bench","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":32.124,"unit":"percent","family":"vals-ai:tax_agent_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench","measuredAt":"2026-11-09","stderr":3.37,"costPerTest":0.857214,"latency":178.329},{"benchmarkId":"epoch-ai:epoch-furniture-assembly:overall","benchmarkName":"Furniture Assembly","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":31.666666666666664,"unit":"percent","family":"epoch-ai:epoch-furniture-assembly:overall","index":61.42578287414143,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":5.93832076297605,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-long-horizon:overall","benchmarkName":"LiveBench Long Agentic","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":30.1,"unit":"percent","family":"livebench-2-agentic","index":40.60253753947596,"aboveFit":false,"run":"high effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Long%20Agentic","measuredAt":"2026-10-14","stderr":null,"costPerTest":7.039016666666666,"latency":null},{"benchmarkId":"aaCyberIndex","benchmarkName":"Artificial Analysis Cyber Index","category":"agentic","score":25.1,"unit":"percent","family":"aaCyberIndex","index":53.01512431697944,"aboveFit":false,"run":null,"source":"Artificial Analysis with the Cyber Index Alliance (Collinear AI, IBM, NVIDIA, Pasta Lab, Vercel)","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-cyber-index","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"apprenticeBench","benchmarkName":"ApprenticeBench: end-to-end computer use, continual learning, and long-horizon agency on a real accounts-payable job","category":"agentic","score":24,"unit":"percent","family":"apprenticeBench","index":70.37534816259333,"aboveFit":false,"run":null,"source":"NeoCognition","sourceUrl":"https://neocognition.io/blog/apprentice-bench/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tier-4-v2-private:overall","benchmarkName":"FrontierMath-Tier-4-v2-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":21.951219512195124,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tier-4-v2-private:overall","index":57.11184193415877,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":6.5445892024384085,"costPerTest":null,"latency":null},{"benchmarkId":"aaGdpPdf","benchmarkName":"Artificial Analysis GDP.pdf","category":"agentic","score":21,"unit":"percent","family":"aaGdpPdf","index":67.47527358877707,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdp-pdf","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaTerminalBench4","benchmarkName":"Artificial Analysis Terminal-Bench v4.0","category":"agentic","score":19.7,"unit":"percent","family":"aaTerminalBench4","index":60.25712376162086,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/terminalbench-v4-0","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:terminal-bench-4:overall","benchmarkName":"Terminal-Bench 4.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":19.697,"unit":"percent","family":"terminal-bench-4","index":63.86890998891293,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-4","measuredAt":"2026-11-09","stderr":2.314,"costPerTest":8.64749,"latency":6427.575},{"benchmarkId":"frontierSweV2","benchmarkName":"FrontierSWE v2","category":"coding","score":19.6,"unit":"percent","family":"frontierSweV2","index":62.66480227205283,"aboveFit":false,"run":null,"source":"Proximal","sourceUrl":"https://www.frontierswe.com/blog/v2","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"terminal-bench:terminal-bench-4-0-0:overall","benchmarkName":"Terminal-Bench 4.0.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":19.09,"unit":"percent","family":"terminal-bench-4","index":63.58068293964169,"aboveFit":false,"run":"high effort · mini-SWE-agent","source":"Terminal-Bench","sourceUrl":"https://www.tbench.ai/leaderboard","measuredAt":"2026-09-29","stderr":1.7142857142857142,"costPerTest":5.541727272727273,"latency":1995.1},{"benchmarkId":"vals-ai:time_horizon_index:overall","benchmarkName":"Time Horizon Index: KSP","category":null,"domain":"Other","eligible":false,"ranked":true,"score":18.833,"unit":"percent","family":"vals-ai:time_horizon_index:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/time_horizon_index","measuredAt":"2026-10-07","stderr":0,"costPerTest":543.140818,"latency":0},{"benchmarkId":"vals-ai:vcb-1-100:overall","benchmarkName":"Vibe Code Bench 1-100","category":"coding","domain":null,"eligible":true,"ranked":true,"score":18.772,"unit":"percent","family":"vals-ai:vcb-1-100:overall","index":67.15005740345865,"aboveFit":false,"run":"OpenHands","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vcb-1-100","measuredAt":"2026-10-29","stderr":3.313,"costPerTest":18.906982,"latency":3931.352},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":18.3,"unit":"percent","family":"critpt","index":72.27578070836255,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"terminal-bench:terminal-bench-4-1-0:overall","benchmarkName":"Terminal-Bench 4.1.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":18.18,"unit":"percent","family":"terminal-bench:terminal-bench-4-1-0:overall","index":61.457329448229636,"aboveFit":false,"run":"high effort · mini-SWE-agent","source":"Terminal-Bench","sourceUrl":"https://www.tbench.ai/leaderboard","measuredAt":"2026-09-29","stderr":1.7142857142857142,"costPerTest":5.541727272727273,"latency":1995.1},{"benchmarkId":"bug-hunt-bench","benchmarkName":"Bug Hunt Bench","category":"coding","score":18,"unit":"percent","family":"bug-hunt-bench","index":62.31185384089135,"aboveFit":false,"run":null,"source":"Pawel Huryn","sourceUrl":"https://bughunt.productcompass.pm/method","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaDeepsecBench","benchmarkName":"Artificial Analysis DeepsecBench-AA","category":"agentic","score":14.5,"unit":"percent","family":"aaDeepsecBench","index":61.43804532818574,"aboveFit":false,"run":null,"source":"Artificial Analysis; dataset by Vercel","sourceUrl":"https://artificialanalysis.ai/evaluations/deepsecbench-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:hlab:overall","benchmarkName":"HLAB-Vals-1.1","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":12.083,"unit":"percent","family":"vals-ai:hlab:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/hlab","measuredAt":"2026-11-09","stderr":2.579,"costPerTest":3.656207,"latency":1161.715},{"benchmarkId":"vals-ai:srebench:overall","benchmarkName":"SRE Bench","category":null,"domain":"Other","eligible":false,"ranked":true,"score":11.069,"unit":"percent","family":"vals-ai:srebench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/srebench","measuredAt":"2026-11-05","stderr":1.942,"costPerTest":27.459572,"latency":8853.856},{"benchmarkId":"aaMlcr","benchmarkName":"Medical Long Context Reasoning (MLCR-AA)","category":"reasoning","score":10.6,"unit":"percent","family":"aaMlcr","index":56.1155856352738,"aboveFit":false,"run":null,"source":"Wisedocs and Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mlcr-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"arc-prize:v3-semi-private:overall","benchmarkName":"ARC-AGI-3 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":10.36627922392312,"unit":"percent","family":"arc-prize:v3-semi-private:overall","index":69.70983320913604,"aboveFit":false,"run":"high effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:terminal-bench-science:overall","benchmarkName":"Terminal-Bench-Science","category":null,"domain":"Science","eligible":false,"ranked":true,"score":8.571,"unit":"percent","family":"vals-ai:terminal-bench-science:overall","index":null,"aboveFit":false,"run":"Mini-SWE-agent","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-science","measuredAt":"2026-11-09","stderr":3.37,"costPerTest":5.643008,"latency":3311.435},{"benchmarkId":"aaCyberGymE2e","benchmarkName":"Artificial Analysis CyberGym-E2E-AA","category":"agentic","score":8.4,"unit":"percent","family":"aaCyberGymE2e","index":18.783110049094937,"aboveFit":false,"run":null,"source":"Artificial Analysis; dataset by Berkeley RDI","sourceUrl":"https://artificialanalysis.ai/evaluations/cybergym-e2e-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:cua_bench:overall","benchmarkName":"CUA-bench","category":null,"domain":"Other","eligible":false,"ranked":true,"score":4.166667,"unit":"percent","family":"vals-ai:cua_bench:overall","index":null,"aboveFit":false,"run":"google-computer-use","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/cua_bench","measuredAt":"2026-10-13","stderr":0,"costPerTest":280.032879,"latency":36.55},{"benchmarkId":"mazeBench","benchmarkName":"MazeBench: visual spatial reasoning in a 3D open world","category":"agentic","score":4,"unit":"percent","family":"mazeBench","index":69.25860316466222,"aboveFit":false,"run":null,"source":"Jonathan Pappas and David Pappas","sourceUrl":"https://mazebench.com/blog?post=maze-bench-results","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lmarena:agent-task-outcome:overall","benchmarkName":"Agent Arena task outcome","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":1.3139767226551635,"unit":"index","family":"agent-arena","index":67.61109617053492,"aboveFit":false,"run":"high effort","source":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset","measuredAt":"2026-11-11","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:programbench:overall","benchmarkName":"ProgramBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":1,"unit":"percent","family":"vals-ai:programbench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/programbench","measuredAt":"2026-11-09","stderr":0.705,"costPerTest":10.928509,"latency":2799.268},{"benchmarkId":"epoch-ai:epoch-frontiermath-erdos:overall","benchmarkName":"FrontierMath-Erdos","category":"math","domain":null,"eligible":true,"ranked":true,"score":0,"unit":"percent","family":"epoch-ai:epoch-frontiermath-erdos:overall","index":null,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0,"costPerTest":null,"latency":null},{"benchmarkId":"lmarena:agent-bash-recovery:overall","benchmarkName":"Agent Arena command recovery","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":-0.6867809731099461,"unit":"index","family":"agent-arena","index":65.96615968317562,"aboveFit":false,"run":"high effort","source":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset","measuredAt":"2026-10-08","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lmarena:agent-steerability:overall","benchmarkName":"Agent Arena steerability","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":-1.2611466543311245,"unit":"index","family":"agent-arena","index":65.49394104914138,"aboveFit":false,"run":"high effort","source":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset","measuredAt":"2026-11-11","stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":0.8250000000000001,"outputPerMillion":4.125,"batch":{"inputPerMillion":0.375,"outputPerMillion":1.875},"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":68,"timeToFirstToken":2.9665,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":{"elo":1500.4121268761392,"eloUpper":1504.642932073527,"eloLower":1496.1813216787514,"votes":61143,"rank":9,"rankUpper":null,"rankLower":null,"arenaName":"Gemini 3.8 Flash-high","organization":"google","updatedAt":"2026-11-09"},"quality":{"coveragePercent":95,"benchmarkCount":66,"categoryCount":7,"confidence":"high","state":"available"},"hosts":[{"host":"google-ai-studio","name":"Google AI Studio","kind":"maker","url":"https://ai.google.dev/gemini-api/docs/models","inputPerMillion":0.75,"outputPerMillion":3.75,"cacheReadPerMillion":0.075,"contextTokens":1048576,"maxOutputTokens":65536,"deprecated":false},{"host":"google-vertex","name":"Google Vertex AI","kind":"maker","url":"https://cloud.google.com/vertex-ai/generative-ai/docs/models","inputPerMillion":0.75,"outputPerMillion":3.75,"cacheReadPerMillion":0.075,"contextTokens":1048576,"maxOutputTokens":65536,"deprecated":false},{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":0.75,"outputPerMillion":3.75,"cacheReadPerMillion":0.075,"contextTokens":1048576,"maxOutputTokens":65536,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:arc-prize-foundation","name":"ARC Prize Foundation","url":"https://arcprize.org","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:livebench","name":"LiveBench","url":"https://livebench.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:terminal-bench","name":"Terminal-Bench","url":"https://www.tbench.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:lmarena","name":"LMArena","url":"https://huggingface.co","license":"CC BY 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"grok-4-6","providerId":"xai","slug":"grok-4-6","name":"Grok 4.6","provider":"xAI","description":"Grok 4.6 is a reasoning model from xAI. 54 benchmarks count toward its index, in 6 categories.","logoUrl":null,"releaseDate":"2026-08-12","contextWindowTokens":500000,"modalities":["text","image","file"],"openWeights":false,"details":{"apiId":"x-ai/grok-4.6","maxOutputTokens":450000,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.5,"weightsUrl":null},"reasoning":true,"overallScore":67.79160591487398,"overallScoreRange":{"low":64.09114762927376,"high":71.4920642004742},"badges":[],"capabilityScores":{"agentic":68.42952206719399,"coding":67.72831703181058,"reasoning":66.07498902768762,"multimodal":null,"knowledge":65.65000074929974,"multilingual":null,"instruction":72.00415848048893,"math":65.34527304837836},"capabilityCounts":{"math":5,"coding":16,"knowledge":6,"reasoning":10,"agentic":16,"instruction":1},"benchmarks":[{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":99.16666666666667,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":66.6136670784291,"aboveFit":false,"run":"xhigh effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0.4700633956814717,"costPerTest":null,"latency":null},{"benchmarkId":"arc-prize:v1-public-eval:overall","benchmarkName":"ARC-AGI-1 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":96.25,"unit":"percent","family":"arc-prize:v1-public-eval:overall","index":null,"aboveFit":false,"run":"xhigh effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.23996201499999992,"latency":null},{"benchmarkId":"vals-ai:swebench:overall","benchmarkName":"SWE-bench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":95.6,"unit":"percent","family":"swe-bench-verified","index":71.73843196559258,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/swebench","measuredAt":"2026-09-01","stderr":0.918,"costPerTest":0.784684,"latency":604.494},{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":94.9,"unit":"percent","family":"gpqa-diamond","index":64.24878758773713,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:gpqa:overall","benchmarkName":"GPQA Diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":94.697,"unit":"percent","family":"gpqa-diamond","index":64.06260153472842,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","measuredAt":"2026-09-01","stderr":1.125,"costPerTest":0.045365,"latency":209.056},{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":93.18181818181817,"unit":"percent","family":"gpqa-diamond","index":62.672918173556766,"aboveFit":false,"run":"xhigh effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.5204299687218694,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-math:overall","benchmarkName":"LiveBench Mathematics","category":"math","domain":null,"eligible":true,"ranked":true,"score":92.56799999999998,"unit":"percent","family":"livebench:livebench-math:overall","index":68.78863181282424,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Mathematics","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.06367836411609498,"latency":null},{"benchmarkId":"livebench:livebench-reasoning:overall","benchmarkName":"LiveBench Reasoning","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":91.2,"unit":"percent","family":"livebench","index":79.76330178398537,"aboveFit":false,"run":"xhigh effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Reasoning","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.08725733663366336,"latency":null},{"benchmarkId":"vals-ai:mmlu_pro:overall","benchmarkName":"MMLU Pro","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":89.4,"unit":"percent","family":"mmlu-pro","index":57.11402752202025,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","measuredAt":"2026-09-01","stderr":0.305,"costPerTest":0.015391,"latency":54.975},{"benchmarkId":"livebench:livebench-coding-2:overall","benchmarkName":"LiveBench Coding 2.0","category":"coding","domain":null,"eligible":true,"ranked":true,"score":89.1,"unit":"percent","family":"livebench:livebench-coding-2:overall","index":72.70928315073732,"aboveFit":false,"run":"xhigh effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Coding","measuredAt":"2026-10-14","stderr":null,"costPerTest":1.8583793103448276,"latency":null},{"benchmarkId":"aaTerminalBench21","benchmarkName":"Artificial Analysis Terminal-Bench v2.1","category":"agentic","score":88.4,"unit":"percent","family":"aaTerminalBench21","index":71.05522338919815,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/terminalbench-v2-1","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:lcb:overall","benchmarkName":"LiveCodeBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":88.224,"unit":"percent","family":"livecodebench","index":62.54685669103644,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","measuredAt":"2026-09-01","stderr":0.941,"costPerTest":0.039923,"latency":118.4},{"benchmarkId":"vulcanBench","benchmarkName":"VulcanBench v3","category":"coding","score":87,"unit":"percent","family":"vulcanBench","index":68.4175418120868,"aboveFit":false,"run":null,"source":"VulcanBench contributors","sourceUrl":"https://github.com/morganlinton/VulcanBench/tree/main","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"arc-prize:v1-semi-private:overall","benchmarkName":"ARC-AGI-1 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":87,"unit":"percent","family":"arc-prize:v1-semi-private:overall","index":66.98439981896328,"aboveFit":false,"run":"xhigh effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.348182,"latency":null},{"benchmarkId":"vals-ai:medscribe:overall","benchmarkName":"MedScribe","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":86.534,"unit":"percent","family":"vals-ai:medscribe:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medscribe","measuredAt":"2026-10-29","stderr":1.956,"costPerTest":0.03719,"latency":70.944},{"benchmarkId":"vals-ai:legal_bench:overall","benchmarkName":"LegalBench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":86.307,"unit":"percent","family":"vals-ai:legal_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","measuredAt":"2026-10-13","stderr":0.418,"costPerTest":0.01457,"latency":27.42},{"benchmarkId":"livebench:livebench-research:overall","benchmarkName":"LiveBench Research","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":84.5,"unit":"percent","family":"livebench-2-agentic","index":80.38694910314882,"aboveFit":false,"run":"xhigh effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Research","measuredAt":"2026-10-14","stderr":null,"costPerTest":2.3254636363636365,"latency":null},{"benchmarkId":"livebench:livebench-language:overall","benchmarkName":"LiveBench Language","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":83.697,"unit":"percent","family":"livebench:livebench-language:overall","index":69.20411434853425,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Language","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.03499066666666667,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":80.3,"unit":"percent","family":"lcr","index":62.39031566369921,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-design:overall","benchmarkName":"LiveBench Design","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":79.8,"unit":"percent","family":"livebench-2-agentic","index":76.94969295702266,"aboveFit":false,"run":"xhigh effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Design","measuredAt":"2026-10-14","stderr":null,"costPerTest":1.266572,"latency":null},{"benchmarkId":"vals-ai:terminal-bench-2-1:overall","benchmarkName":"Terminal-Bench 2.1","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":78.277,"unit":"percent","family":"terminal-bench-2-1","index":66.92184457220628,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2-1","measuredAt":"2026-10-01","stderr":2.085,"costPerTest":0.453642,"latency":642.908},{"benchmarkId":"livebench:livebench-coding:overall","benchmarkName":"LiveBench Coding","category":"coding","domain":null,"eligible":true,"ranked":true,"score":76.77600000000001,"unit":"percent","family":"livebench:livebench-coding:overall","index":62.80269157979403,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Coding","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.020515384615384616,"latency":null},{"benchmarkId":"vals-ai:vibe-code:overall","benchmarkName":"Vibe Code Bench v1.1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":76.239,"unit":"percent","family":"vals-ai:vibe-code:overall","index":69.75775159548617,"aboveFit":false,"run":"OpenHands","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","measuredAt":"2026-11-09","stderr":3.824,"costPerTest":4.875513,"latency":1530.145},{"benchmarkId":"livebench:livebench-data-analysis:overall","benchmarkName":"LiveBench Data Analysis","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":75.3,"unit":"percent","family":"livebench","index":59.16590124525511,"aboveFit":false,"run":"xhigh effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Data%20Analysis","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.12299009333333333,"latency":null},{"benchmarkId":"vals-ai:biomysterybench:overall","benchmarkName":"BioMysteryBench","category":null,"domain":"Science","eligible":false,"ranked":true,"score":72.222,"unit":"percent","family":"vals-ai:biomysterybench:overall","index":null,"aboveFit":false,"run":"Terminus 2","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/biomysterybench","measuredAt":"2026-11-09","stderr":0,"costPerTest":1.226606,"latency":858.019},{"benchmarkId":"livebench:livebench-instruction-following:overall","benchmarkName":"LiveBench Instruction Following","category":"instruction","domain":null,"eligible":true,"ranked":true,"score":71.8665,"unit":"percent","family":"livebench:livebench-instruction-following:overall","index":72.00415848048893,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=IF","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.0444595,"latency":null},{"benchmarkId":"vals-ai:tax_eval_v2:overall","benchmarkName":"TaxEval v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":71.096,"unit":"percent","family":"vals-ai:tax_eval_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","measuredAt":"2026-09-01","stderr":0.899,"costPerTest":0.017298,"latency":47.57},{"benchmarkId":"cursorBench32","benchmarkName":"cursorBench32","category":"coding","score":70.8,"unit":"percent","family":"cursorBench32","index":74.01174160656439,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"arc-prize:v2-public-eval:overall","benchmarkName":"ARC-AGI-2 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":68.33333333333333,"unit":"percent","family":"arc-prize:v2-public-eval:overall","index":null,"aboveFit":false,"run":"xhigh effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.7545896499999999,"latency":null},{"benchmarkId":"vals-ai:cyber:overall","benchmarkName":"CyberBench v1.1","category":null,"domain":"Other","eligible":false,"ranked":true,"score":67.678,"unit":"percent","family":"vals-ai:cyber:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/cyber","measuredAt":"2026-11-05","stderr":5.866,"costPerTest":4.06856,"latency":2212.209},{"benchmarkId":"arc-prize:v2-semi-private:overall","benchmarkName":"ARC-AGI-2 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":67.08333333333333,"unit":"percent","family":"arc-prize:v2-semi-private:overall","index":69.98880159272635,"aboveFit":false,"run":"xhigh effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.7574904499999999,"latency":null},{"benchmarkId":"vals-ai:public-benefits-bench:overall","benchmarkName":"Public Benefits Bench v1.1","category":null,"domain":"Public benefits","eligible":false,"ranked":true,"score":66.847,"unit":"percent","family":"vals-ai:public-benefits-bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/public-benefits-bench","measuredAt":"2026-10-12","stderr":1.225,"costPerTest":0.91559,"latency":1552.915},{"benchmarkId":"vals-ai:corp_fin_v2:overall","benchmarkName":"CorpFin v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":66.161,"unit":"percent","family":"vals-ai:corp_fin_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","measuredAt":"2026-08-12","stderr":0.932,"costPerTest":0.165532,"latency":24.256},{"benchmarkId":"epoch-ai:epoch-frontiermath-tiers-1-3-v2-private:overall","benchmarkName":"FrontierMath-Tiers-1-3-v2-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":65.96491228070175,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tiers-1-3-v2-private:overall","index":65.61544076994005,"aboveFit":false,"run":"xhigh effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.8116467881852296,"costPerTest":null,"latency":null},{"benchmarkId":"deepSwe","benchmarkName":"DeepSWE","category":"agentic","score":65.9,"unit":"percent","family":"deepSwe","index":68.98418096806778,"aboveFit":false,"run":null,"source":"Datacurve AI","sourceUrl":"https://deepswe.datacurve.ai/blog","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:mortgage_tax:overall","benchmarkName":"MortgageTax","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":64.189,"unit":"percent","family":"vals-ai:mortgage_tax:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mortgage_tax","measuredAt":"2026-09-01","stderr":0.952,"costPerTest":0.013712,"latency":8.548},{"benchmarkId":"livebench:livebench-browser:overall","benchmarkName":"LiveBench Browser Use","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":64,"unit":"percent","family":"livebench-2-agentic","index":65.39466165727944,"aboveFit":false,"run":"xhigh effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Browser","measuredAt":"2026-10-14","stderr":null,"costPerTest":2.921248,"latency":null},{"benchmarkId":"vals-ai:emb:overall","benchmarkName":"EMB","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":62.727,"unit":"percent","family":"vals-ai:emb:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/emb","measuredAt":"2026-10-20","stderr":3.085,"costPerTest":3.056382,"latency":3178.533},{"benchmarkId":"frontierCode11Extended","benchmarkName":"FrontierCode 1.1 Extended","category":"coding","score":61.3,"unit":"percent","family":"frontierCode11Extended","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"apexAgents","benchmarkName":"APEX-Agents","category":"agentic","score":57.5,"unit":"percent","family":"apexAgents","index":74.29769942999104,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-agentic-coding:overall","benchmarkName":"LiveBench Agentic Coding","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":57.02,"unit":"percent","family":"livebench:livebench-agentic-coding:overall","index":67.96644521620146,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Agentic%20Coding","measuredAt":"2026-06-25","stderr":null,"costPerTest":1.787576388888889,"latency":null},{"benchmarkId":"cweBenchV1","benchmarkName":"CWE-bench v1","category":"agentic","score":57,"unit":"percent","family":"cweBenchV1","index":70.23250574767468,"aboveFit":false,"run":null,"source":"Collinear AI","sourceUrl":"https://cwe-bench.com/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaSciCode","benchmarkName":"Artificial Analysis SciCode","category":"coding","score":56.5,"unit":"percent","family":"scicode-family","index":69.49834821494369,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/scicode","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gdpvalAaNormalized","benchmarkName":"GDPval-AA normalized","category":"agentic","score":55.9,"unit":"percent","family":"gdpvalAaNormalized","index":74.00097589576619,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:skillsbench:overall","benchmarkName":"SkillsBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":55.766,"unit":"percent","family":"vals-ai:skillsbench:overall","index":66.65017894896235,"aboveFit":false,"run":"OpenHands","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/skillsbench","measuredAt":"2026-10-29","stderr":4.844,"costPerTest":0.921477,"latency":808.117},{"benchmarkId":"vals-ai:fabv2:overall","benchmarkName":"Finance Agent (v2)","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":53.682,"unit":"percent","family":"vals-ai:fabv2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/fabv2","measuredAt":"2026-10-29","stderr":0.67,"costPerTest":1.657732,"latency":1000.937},{"benchmarkId":"vals-ai:proof_bench:overall","benchmarkName":"ProofBench v1.1","category":"math","domain":null,"eligible":true,"ranked":true,"score":51,"unit":"percent","family":"vals-ai:proof_bench:overall","index":64.76410034124713,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/proof_bench","measuredAt":"2026-11-09","stderr":5.024,"costPerTest":0.824475,"latency":861.898},{"benchmarkId":"aaTau3Banking","benchmarkName":"Artificial Analysis Tau3-Banking","category":"agentic","score":50.7,"unit":"percent","family":"aaTau3Banking","index":77.2456656428254,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/tau3-banking","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-simpleqa-verified:overall","benchmarkName":"SimpleQA Verified","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":48.9,"unit":"percent","family":"epoch-ai:epoch-simpleqa-verified:overall","index":64.65706911712957,"aboveFit":false,"run":"xhigh effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.5815471195292574,"costPerTest":null,"latency":null},{"benchmarkId":"aaEnterpriseOpsGym","benchmarkName":"Artificial Analysis EnterpriseOps-Gym","category":"agentic","score":48.3,"unit":"percent","family":"aaEnterpriseOpsGym","index":null,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/enterprise-ops-gym-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"omniscienceAccuracy","benchmarkName":"Artificial Analysis Omniscience Accuracy","category":"knowledge","score":48.2,"unit":"percent","family":"omniscienceAccuracy","index":71.43428889829424,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:legal_research:overall","benchmarkName":"Legal Research Bench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":48.077,"unit":"percent","family":"vals-ai:legal_research:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_research","measuredAt":"2026-11-09","stderr":3.473,"costPerTest":1.530204,"latency":1504.077},{"benchmarkId":"vals-ai:ioi:overall","benchmarkName":"IOI","category":"coding","domain":null,"eligible":true,"ranked":true,"score":47.611,"unit":"percent","family":"vals-ai:ioi:overall","index":63.99087009771402,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/ioi","measuredAt":"2026-10-29","stderr":2.091,"costPerTest":8.183386,"latency":14233.553},{"benchmarkId":"vals-ai:medcode:overall","benchmarkName":"MedCode","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":44.712,"unit":"percent","family":"vals-ai:medcode:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medcode","measuredAt":"2026-10-29","stderr":2.256,"costPerTest":0.050335,"latency":127.284},{"benchmarkId":"vals-ai:code-migration:overall","benchmarkName":"Code Migration","category":"coding","domain":null,"eligible":true,"ranked":true,"score":44.572,"unit":"percent","family":"vals-ai:code-migration:overall","index":68.86757090109404,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/code-migration","measuredAt":"2026-10-29","stderr":4.479,"costPerTest":14.575491,"latency":5429.239},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":42.9,"unit":"percent","family":"hle","index":67.42790307509165,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"cursorBench40","benchmarkName":"cursorBench40","category":"coding","score":41.4,"unit":"percent","family":"cursorBench40","index":68.09828316602241,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaAnalystAgent","benchmarkName":"Artificial Analysis AnalystAgent","category":"agentic","score":41.3,"unit":"percent","family":"aaAnalystAgent","index":62.17923610169161,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/aa-analyst-agent","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-mirrorcode:overall","benchmarkName":"MirrorCode","category":"coding","domain":null,"eligible":true,"ranked":true,"score":41.11111111111111,"unit":"percent","family":"epoch-ai:epoch-mirrorcode:overall","index":67.62045997253124,"aboveFit":false,"run":"xhigh effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":11.01546131612538,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-long-horizon:overall","benchmarkName":"LiveBench Long Agentic","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":40.7,"unit":"percent","family":"livebench-2-agentic","index":48.3546471456328,"aboveFit":false,"run":"xhigh effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Long%20Agentic","measuredAt":"2026-10-14","stderr":null,"costPerTest":9.050531034482757,"latency":null},{"benchmarkId":"epoch-ai:epoch-furniture-assembly:overall","benchmarkName":"Furniture Assembly","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":40,"unit":"percent","family":"epoch-ai:epoch-furniture-assembly:overall","index":65.31058433073342,"aboveFit":false,"run":"xhigh effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":6.266221885886569,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-mystery-game-puzzles:overall","benchmarkName":"Mystery Game Puzzles","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":34,"unit":"percent","family":"epoch-ai:epoch-mystery-game-puzzles:overall","index":65.01110469004604,"aboveFit":false,"run":"xhigh effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":4.760952285695233,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:tax_agent_bench:overall","benchmarkName":"Tax Agent Bench","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":33.659,"unit":"percent","family":"vals-ai:tax_agent_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench","measuredAt":"2026-11-09","stderr":3.409,"costPerTest":0.978328,"latency":657.844},{"benchmarkId":"epoch-ai:epoch-chess-puzzles:overall","benchmarkName":"Chess Puzzles","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":33,"unit":"percent","family":"epoch-ai:epoch-chess-puzzles:overall","index":59.03217716509211,"aboveFit":false,"run":"xhigh effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":4.725815626252608,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tier-4-v2-private:overall","benchmarkName":"FrontierMath-Tier-4-v2-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":31.70731707317073,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tier-4-v2-private:overall","index":60.94452523945129,"aboveFit":false,"run":"xhigh effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":7.3576112824382225,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:mysterymechanism:overall","benchmarkName":"MysteryMechanism","category":null,"domain":"Science","eligible":false,"ranked":true,"score":30.631,"unit":"percent","family":"vals-ai:mysterymechanism:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mysterymechanism","measuredAt":"2026-11-11","stderr":3.101,"costPerTest":1.309658,"latency":1674.514},{"benchmarkId":"epoch-ai:epoch-ebr-bench:overall","benchmarkName":"EBR-bench","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":30.476190476190474,"unit":"percent","family":"epoch-ai:epoch-ebr-bench:overall","index":66.41313700476985,"aboveFit":false,"run":"xhigh effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.618736353521519,"costPerTest":null,"latency":null},{"benchmarkId":"bug-hunt-bench","benchmarkName":"Bug Hunt Bench","category":"coding","score":27,"unit":"percent","family":"bug-hunt-bench","index":68.11334880122467,"aboveFit":false,"run":null,"source":"Pawel Huryn","sourceUrl":"https://bughunt.productcompass.pm/method","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"frontierBench","benchmarkName":"Terminal-Bench 3.0","category":"agentic","score":26.49,"unit":"percent","family":"frontierBench","index":71.55854023386148,"aboveFit":false,"run":null,"source":"Ryan Marten, Alex Shaw, Andy Konwinski, Harbor, and the Laude Institute","sourceUrl":"https://www.frontierbench.ai/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"frontierSweV2","benchmarkName":"FrontierSWE v2","category":"coding","score":25.3,"unit":"percent","family":"frontierSweV2","index":65.41367081120174,"aboveFit":false,"run":null,"source":"Proximal","sourceUrl":"https://www.frontierswe.com/blog/v2","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"terminal-bench:terminal-bench-4-0-0:overall","benchmarkName":"Terminal-Bench 4.0.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":20.3,"unit":"percent","family":"terminal-bench-4","index":64.1552376836585,"aboveFit":false,"run":"high effort · Grok Build","source":"Terminal-Bench","sourceUrl":"https://www.tbench.ai/leaderboard","measuredAt":"2026-09-29","stderr":1.5765306122448979,"costPerTest":10.883575757575757,"latency":2351.5},{"benchmarkId":"terminal-bench:terminal-bench-4-1-0:overall","benchmarkName":"Terminal-Bench 4.1.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":19.7,"unit":"percent","family":"terminal-bench:terminal-bench-4-1-0:overall","index":62.16551312828901,"aboveFit":false,"run":"high effort · Grok Build","source":"Terminal-Bench","sourceUrl":"https://www.tbench.ai/leaderboard","measuredAt":"2026-09-29","stderr":1.5765306122448979,"costPerTest":10.883575757575757,"latency":2351.5},{"benchmarkId":"vals-ai:terminal-bench-4:overall","benchmarkName":"Terminal-Bench 4.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":17.172,"unit":"percent","family":"terminal-bench-4","index":62.66994244458032,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-4","measuredAt":"2026-11-09","stderr":1.336,"costPerTest":5.062449,"latency":1765.468},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":17.1,"unit":"percent","family":"critpt","index":69.95272072832388,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:vcb-1-100:overall","benchmarkName":"Vibe Code Bench 1-100","category":"coding","domain":null,"eligible":true,"ranked":true,"score":14.75,"unit":"percent","family":"vals-ai:vcb-1-100:overall","index":63.41604319397734,"aboveFit":false,"run":"OpenHands","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vcb-1-100","measuredAt":"2026-10-29","stderr":3.17,"costPerTest":26.271157,"latency":7851.405},{"benchmarkId":"apprenticeBench","benchmarkName":"ApprenticeBench: end-to-end computer use, continual learning, and long-horizon agency on a real accounts-payable job","category":"agentic","score":13,"unit":"percent","family":"apprenticeBench","index":64.79891337827728,"aboveFit":false,"run":null,"source":"NeoCognition","sourceUrl":"https://neocognition.io/blog/apprentice-bench/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:terminal-bench-science:overall","benchmarkName":"Terminal-Bench-Science","category":null,"domain":"Science","eligible":false,"ranked":true,"score":11.429,"unit":"percent","family":"vals-ai:terminal-bench-science:overall","index":null,"aboveFit":false,"run":"Mini-SWE-agent","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-science","measuredAt":"2026-11-09","stderr":3.83,"costPerTest":4.815933,"latency":4696.051},{"benchmarkId":"vals-ai:time_horizon_index:overall","benchmarkName":"Time Horizon Index: KSP","category":null,"domain":"Other","eligible":false,"ranked":true,"score":5.833,"unit":"percent","family":"vals-ai:time_horizon_index:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/time_horizon_index","measuredAt":"2026-10-07","stderr":0,"costPerTest":225.11666,"latency":0},{"benchmarkId":"lmarena:agent-bash-recovery:overall","benchmarkName":"Agent Arena command recovery","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":5.680396400430032,"unit":"index","family":"agent-arena","index":71.20097767507608,"aboveFit":false,"run":"xhigh effort","source":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset","measuredAt":"2026-10-08","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"arc-prize:v3-semi-private:overall","benchmarkName":"ARC-AGI-3 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":2.11,"unit":"percent","family":"arc-prize:v3-semi-private:overall","index":66.20204776790175,"aboveFit":false,"run":"xhigh effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mazeBench","benchmarkName":"MazeBench: visual spatial reasoning in a 3D open world","category":"agentic","score":1,"unit":"percent","family":"mazeBench","index":63.79865130242642,"aboveFit":false,"run":null,"source":"Jonathan Pappas and David Pappas","sourceUrl":"https://mazebench.com/blog?post=maze-bench-results","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:programbench:overall","benchmarkName":"ProgramBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":1,"unit":"percent","family":"vals-ai:programbench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/programbench","measuredAt":"2026-11-09","stderr":0.705,"costPerTest":16.759019,"latency":16044.462},{"benchmarkId":"lmarena:agent-steerability:overall","benchmarkName":"Agent Arena steerability","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":-1.7619639754849925,"unit":"index","family":"agent-arena","index":65.08219069735654,"aboveFit":false,"run":"xhigh effort","source":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset","measuredAt":"2026-11-11","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lmarena:agent-task-outcome:overall","benchmarkName":"Agent Arena task outcome","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":-2.591439149778685,"unit":"index","family":"agent-arena","index":64.40023206616274,"aboveFit":false,"run":"xhigh effort","source":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset","measuredAt":"2026-11-11","stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":2,"outputPerMillion":6,"batch":{"inputPerMillion":1.5999999999999999,"outputPerMillion":4.8},"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":56,"timeToFirstToken":2.827,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":{"elo":1429.8825897955767,"eloUpper":1434.5398430234795,"eloLower":1425.2253365676734,"votes":34616,"rank":96,"rankUpper":null,"rankLower":null,"arenaName":"Grok 4.6-high","organization":"xai","updatedAt":"2026-11-09"},"quality":{"coveragePercent":85,"benchmarkCount":54,"categoryCount":6,"confidence":"medium","state":"available"},"hosts":[{"host":"amazon-bedrock","name":"Amazon Bedrock","kind":"cloud","url":"https://docs.aws.amazon.com/bedrock/latest/userguide/models-supported.html","inputPerMillion":2,"outputPerMillion":6,"cacheReadPerMillion":0.5,"contextTokens":500000,"maxOutputTokens":500000,"deprecated":false},{"host":"google-vertex","name":"Google Vertex AI","kind":"cloud","url":"https://cloud.google.com/vertex-ai/generative-ai/docs/models","inputPerMillion":2,"outputPerMillion":6,"cacheReadPerMillion":0.5,"contextTokens":524288,"maxOutputTokens":500000,"deprecated":false},{"host":"azure","name":"Microsoft Azure","kind":"cloud","url":"https://learn.microsoft.com/en-us/azure/ai-services/openai/concepts/models","inputPerMillion":2,"outputPerMillion":6,"cacheReadPerMillion":0.5,"contextTokens":200000,"maxOutputTokens":128000,"deprecated":false},{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":2,"outputPerMillion":6,"cacheReadPerMillion":0.5,"contextTokens":500000,"maxOutputTokens":450000,"deprecated":false},{"host":"oracle-cloud","name":"Oracle Cloud","kind":"cloud","url":"https://docs.oracle.com/en-us/iaas/Content/generative-ai/pretrained-models.htm","inputPerMillion":2,"outputPerMillion":6,"cacheReadPerMillion":0.5,"contextTokens":500000,"maxOutputTokens":500000,"deprecated":false},{"host":"xai","name":"xAI","kind":"maker","url":"https://docs.x.ai/docs/models","inputPerMillion":2,"outputPerMillion":6,"cacheReadPerMillion":0.5,"contextTokens":500000,"maxOutputTokens":500000,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:livebench","name":"LiveBench","url":"https://livebench.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:arc-prize-foundation","name":"ARC Prize Foundation","url":"https://arcprize.org","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:terminal-bench","name":"Terminal-Bench","url":"https://www.tbench.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:lmarena","name":"LMArena","url":"https://huggingface.co","license":"CC BY 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"claude-opus-4-8","providerId":"anthropic","slug":"claude-opus-4-8","name":"Claude Opus 4.8","provider":"Anthropic","description":"Claude Opus 4.8 is a reasoning model from Anthropic. 67 benchmarks count toward its index, in 8 categories.","logoUrl":null,"releaseDate":"2026-05-28","contextWindowTokens":1000000,"modalities":["text","image","file"],"openWeights":false,"details":{"apiId":"anthropic/claude-opus-4.8","maxOutputTokens":128000,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.5,"weightsUrl":null},"reasoning":true,"overallScore":67.67452624202542,"overallScoreRange":{"low":65.69904632678397,"high":69.65000615726687},"badges":[],"capabilityScores":{"agentic":66.45951783341128,"coding":68.32038855849575,"reasoning":67.98532403367878,"multimodal":66.46481699509023,"knowledge":66.42950865902863,"multilingual":60.263599494561234,"instruction":59.74613295669795,"math":71.24061715858926},"capabilityCounts":{"math":6,"agentic":21,"knowledge":6,"reasoning":10,"multimodal":7,"coding":14,"multilingual":1,"instruction":2},"benchmarks":[{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":98.33333333333333,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":66.12883870698487,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.4101901870444153,"costPerTest":null,"latency":null},{"benchmarkId":"usamo2026","benchmarkName":"United States of America Mathematical Olympiad 2026","category":"math","score":96.7,"unit":"percent","family":"usamo2026","index":null,"aboveFit":false,"run":null,"source":"Mathematical Association of America","sourceUrl":"https://www.maa.org/math-competitions/usamo","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-browser:overall","benchmarkName":"LiveBench Browser Use","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":96,"unit":"percent","family":"livebench-2-agentic","index":88.79725669473407,"aboveFit":true,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Browser","measuredAt":"2026-10-14","stderr":null,"costPerTest":8.871739999999999,"latency":null},{"benchmarkId":"tau2Bench","benchmarkName":"τ²-Bench Tool-Agent-User Evaluation","category":"agentic","score":94.4,"unit":"percent","family":"tau2-bench","index":65.43074191073848,"aboveFit":false,"run":null,"source":"Victor Barres, Honghua Dong, Soham Ray, Xujie Si, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2506.07982","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-math:overall","benchmarkName":"LiveBench Mathematics","category":"math","domain":null,"eligible":true,"ranked":true,"score":94.31649999999999,"unit":"percent","family":"livebench:livebench-math:overall","index":71.22554999749781,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Mathematics","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.5115841688654352,"latency":null},{"benchmarkId":"gpqa","benchmarkName":"Graduate-Level Google-Proof Q&A","category":"knowledge","score":93.6,"unit":"percent","family":"gpqa-diamond","index":63.056463110341404,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gpqaDiamond","benchmarkName":"GPQA Diamond","category":"knowledge","score":93.6,"unit":"percent","family":"gpqa-diamond","index":63.056463110341404,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"deepSearchQa","benchmarkName":"DeepSearchQA","category":"agentic","score":93.1,"unit":"percent","family":"deepSearchQa","index":68.11375040731687,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"arc-prize:v1-semi-private:overall","benchmarkName":"ARC-AGI-1 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":92.5,"unit":"percent","family":"arc-prize:v1-semi-private:overall","index":69.51294838532246,"aboveFit":false,"run":"max effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":2.332,"latency":null},{"benchmarkId":"vals-ai:gpqa:overall","benchmarkName":"GPQA Diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":92.424,"unit":"percent","family":"gpqa-diamond","index":61.97786804463575,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","measuredAt":"2026-09-01","stderr":1.984,"costPerTest":0.339261,"latency":133.876},{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":92,"unit":"percent","family":"gpqa-diamond","index":61.58898683046975,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":91.03535353535354,"unit":"percent","family":"gpqa-diamond","index":60.70423945213804,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.9157125794612027,"costPerTest":null,"latency":null},{"benchmarkId":"charxiv","benchmarkName":"CharXiv Reasoning","category":"multimodal","score":89.9,"unit":"percent","family":"charxiv","index":66.53520948984384,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:mmlu_pro:overall","benchmarkName":"MMLU Pro","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":89.585,"unit":"percent","family":"mmlu-pro","index":57.34710537808946,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","measuredAt":"2026-09-01","stderr":0.303,"costPerTest":0.060716,"latency":23.44},{"benchmarkId":"livebench:livebench-reasoning:overall","benchmarkName":"LiveBench Reasoning","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":89.19225,"unit":"percent","family":"livebench","index":77.16239417822212,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Reasoning","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.605730693069307,"latency":null},{"benchmarkId":"sweVerified","benchmarkName":"Software Engineering Benchmark Verified","category":"coding","score":88.6,"unit":"percent","family":"swe-bench-verified","index":66.57552892110675,"aboveFit":false,"run":null,"source":"Carlos E. Jimenez, John Yang, Alexander Wettig, Shunyu Yao, Kexin Pei, Ofir Press, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2310.06770","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:swebench:overall","benchmarkName":"SWE-bench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":88.6,"unit":"percent","family":"swe-bench-verified","index":66.57552892110675,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/swebench","measuredAt":"2026-09-01","stderr":1.423,"costPerTest":1.923224,"latency":566.945},{"benchmarkId":"screenSpotPro","benchmarkName":"ScreenSpot Pro","category":"multimodal","score":87.9,"unit":"percent","family":"screenSpotPro","index":67.39182877653785,"aboveFit":false,"run":null,"source":"Kaixin Li, Ziyang Meng, Hongzhan Lin, Ziyang Luo, Yuchen Tian, Jing Ma, Zhiyong Huang, Tat-Seng Chua","sourceUrl":"https://arxiv.org/abs/2504.07981","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:lcb:overall","benchmarkName":"LiveCodeBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":87.819,"unit":"percent","family":"livecodebench","index":62.21081149329033,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","measuredAt":"2026-09-01","stderr":0.948,"costPerTest":0.327373,"latency":159.595},{"benchmarkId":"include","benchmarkName":"INCLUDE","category":"multilingual","score":87.6,"unit":"percent","family":"include","index":60.263599494561234,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"extractBenchLlamaIndex","benchmarkName":"ExtractBench: schema-guided enterprise document extraction (LlamaIndex leaderboard)","category":"multimodal","score":86.84,"unit":"percent","family":"extractBenchLlamaIndex","index":67.2794829882113,"aboveFit":false,"run":null,"source":"Boyang Zhang, Adrian Lyjak, Eli Stewart, Zhaoqi Li and Simon Suo (LlamaIndex)","sourceUrl":"https://arxiv.org/abs/2607.29677","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:mmmu:overall","benchmarkName":"MMMU Pro","category":"multimodal","domain":null,"eligible":true,"ranked":true,"score":86.59,"unit":"percent","family":"mmmu-pro","index":67.84210306450419,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmmu","measuredAt":"2026-09-01","stderr":0.819,"costPerTest":0.120663,"latency":49.044},{"benchmarkId":"vals-ai:medscribe:overall","benchmarkName":"MedScribe","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":85.755,"unit":"percent","family":"vals-ai:medscribe:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medscribe","measuredAt":"2026-10-29","stderr":1.928,"costPerTest":0.259121,"latency":82.364},{"benchmarkId":"aaTerminalBench21","benchmarkName":"Artificial Analysis Terminal-Bench v2.1","category":"agentic","score":84.6,"unit":"percent","family":"aaTerminalBench21","index":67.20045463795731,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/terminalbench-v2-1","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"browseComp","benchmarkName":"BrowseComp","category":"agentic","score":84.3,"unit":"percent","family":"browseComp","index":66.50766369634027,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:legal_bench:overall","benchmarkName":"LegalBench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":83.568,"unit":"percent","family":"vals-ai:legal_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","measuredAt":"2026-10-13","stderr":0.474,"costPerTest":0.009106,"latency":3.106},{"benchmarkId":"osWorldVerified","benchmarkName":"OSWorld-Verified","category":"agentic","score":83.4,"unit":"percent","family":"osWorldVerified","index":67.80624948629216,"aboveFit":false,"run":null,"source":"Tianbao Xie, Danyang Zhang, Jixuan Chen, Xiaochuan Li, Siheng Zhao, Ruisheng Cao, Toh Jing Hua, Zhoujun Cheng, Dongchan Shin, Fangyu Lei, Yitao Liu, Yiheng Xu, Shuyan Zhou, Silvio Savarese, Caiming Xiong, Victor Zhong, Tao Yu","sourceUrl":"https://os-world.github.io/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:vibe-code:overall","benchmarkName":"Vibe Code Bench v1.1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":82.725,"unit":"percent","family":"vals-ai:vibe-code:overall","index":72.09800229818882,"aboveFit":false,"run":"OpenHands","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","measuredAt":"2026-11-09","stderr":3.081,"costPerTest":26.879989,"latency":4549.256},{"benchmarkId":"mcpAtlas","benchmarkName":"MCP Atlas","category":"agentic","score":82.2,"unit":"percent","family":"mcpAtlas","index":66.41540130870948,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-coding:overall","benchmarkName":"LiveBench Coding","category":"coding","domain":null,"eligible":true,"ranked":true,"score":81.828,"unit":"percent","family":"livebench:livebench-coding:overall","index":69.907761722011,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Coding","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.12269572649572649,"latency":null},{"benchmarkId":"livebench:livebench-research:overall","benchmarkName":"LiveBench Research","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":81.8,"unit":"percent","family":"livebench-2-agentic","index":78.41235514686358,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Research","measuredAt":"2026-10-14","stderr":null,"costPerTest":6.235313636363636,"latency":null},{"benchmarkId":"charxivNoTools","benchmarkName":"CharXiv Reasoning without tools","category":"multimodal","score":80.5,"unit":"percent","family":"charxivNoTools","index":59.950924309819165,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tiers-1-3-v2-private:overall","benchmarkName":"FrontierMath-Tiers-1-3-v2-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":80,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tiers-1-3-v2-private:overall","index":73.00283320922033,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.3735633163877066,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-language:overall","benchmarkName":"LiveBench Language","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":79.65933333333334,"unit":"percent","family":"livebench:livebench-language:overall","index":64.63828196866429,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Language","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.48583800000000005,"latency":null},{"benchmarkId":"livebench:livebench-design:overall","benchmarkName":"LiveBench Design","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":78.8,"unit":"percent","family":"livebench-2-agentic","index":76.21836186210221,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Design","measuredAt":"2026-10-14","stderr":null,"costPerTest":6.450344,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":77.7,"unit":"percent","family":"lcr","index":60.6065062971765,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-coding-2:overall","benchmarkName":"LiveBench Coding 2.0","category":"coding","domain":null,"eligible":true,"ranked":true,"score":77.3,"unit":"percent","family":"livebench:livebench-coding-2:overall","index":64.16224725377171,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Coding","measuredAt":"2026-10-14","stderr":null,"costPerTest":4.1516586206896555,"latency":null},{"benchmarkId":"vals-ai:tax_eval_v2:overall","benchmarkName":"TaxEval v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":75.634,"unit":"percent","family":"vals-ai:tax_eval_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","measuredAt":"2026-09-01","stderr":0.838,"costPerTest":0.159139,"latency":70.33},{"benchmarkId":"terminalBench21","benchmarkName":"Terminal-Bench 2.1","category":"agentic","score":74.6,"unit":"percent","family":"terminal-bench-2-1","index":64.77133139703915,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gertLabs","benchmarkName":"Gert Labs Composite Game Benchmark","category":"agentic","score":72.97,"unit":"percent","family":"gertLabs","index":74.63179079964154,"aboveFit":false,"run":null,"source":"Gert Labs","sourceUrl":"https://gertlabs.com/rankings","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"arc-prize:v2-semi-private:overall","benchmarkName":"ARC-AGI-2 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":72.08,"unit":"percent","family":"arc-prize:v2-semi-private:overall","index":72.28294464860387,"aboveFit":false,"run":"high effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":2.7425,"latency":null},{"benchmarkId":"livebench:livebench-instruction-following:overall","benchmarkName":"LiveBench Instruction Following","category":"instruction","domain":null,"eligible":true,"ranked":true,"score":72.0335,"unit":"percent","family":"livebench:livebench-instruction-following:overall","index":72.3420762789359,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=IF","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.471148,"latency":null},{"benchmarkId":"vals-ai:terminal-bench-2-1:overall","benchmarkName":"Terminal-Bench 2.1","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":71.91,"unit":"percent","family":"terminal-bench-2-1","index":63.198070466606836,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2-1","measuredAt":"2026-10-01","stderr":0.649,"costPerTest":2.409772,"latency":929.902},{"benchmarkId":"vals-ai:terminal-bench-2:overall","benchmarkName":"Terminal-Bench 2.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":70.037,"unit":"percent","family":"vals-ai:terminal-bench-2:overall","index":68.7213305249151,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2","measuredAt":"2026-06-04","stderr":1.35,"costPerTest":0.883864,"latency":472.445},{"benchmarkId":"vals-ai:mortgage_tax:overall","benchmarkName":"MortgageTax","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":69.912,"unit":"percent","family":"vals-ai:mortgage_tax:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mortgage_tax","measuredAt":"2026-09-01","stderr":0.893,"costPerTest":0.105627,"latency":20.814},{"benchmarkId":"vals-ai:emb:overall","benchmarkName":"EMB","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":69.373,"unit":"percent","family":"vals-ai:emb:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/emb","measuredAt":"2026-10-20","stderr":2.606,"costPerTest":12.05844,"latency":2632.285},{"benchmarkId":"swePro","benchmarkName":"SWE-bench Pro","category":"coding","score":69.2,"unit":"percent","family":"swePro","index":67.77079437212194,"aboveFit":false,"run":null,"source":"Xiang Deng, Jeff Da, Edwin Pan, Yannis Yiming He, Charles Ide, Kanak Garg, Niklas Lauffer, Andrew Park, Nitin Pasari, Chetan Rane, Karmini Sampath, Maya Krishnan, Srivatsa Kundurthy, Sean Hendryx, Zifan Wang, Chen Bo Calvin Zhang, Noah Jacobson, Bing Liu, Brad Kenstler","sourceUrl":"https://arxiv.org/abs/2509.16941","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:public-benefits-bench:overall","benchmarkName":"Public Benefits Bench v1.1","category":null,"domain":"Public benefits","eligible":false,"ranked":true,"score":68.133,"unit":"percent","family":"vals-ai:public-benefits-bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/public-benefits-bench","measuredAt":"2026-10-12","stderr":1.212,"costPerTest":1.889795,"latency":767.85},{"benchmarkId":"vals-ai:corp_fin_v2:overall","benchmarkName":"CorpFin v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":66.706,"unit":"percent","family":"vals-ai:corp_fin_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","measuredAt":"2026-08-12","stderr":0.928,"costPerTest":0.855534,"latency":20.226},{"benchmarkId":"officeQaPro","benchmarkName":"OfficeQA Pro","category":"multimodal","score":66.2,"unit":"percent","family":"officeQaPro","index":75.65342717321258,"aboveFit":false,"run":null,"source":"OfficeQA Pro authors","sourceUrl":"https://arxiv.org/abs/2603.08655","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-data-analysis:overall","benchmarkName":"LiveBench Data Analysis","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":66.03333333333333,"unit":"percent","family":"livebench","index":47.16154621429702,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Data%20Analysis","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.6455813333333335,"latency":null},{"benchmarkId":"epoch-ai:epoch-mirrorcode:overall","benchmarkName":"MirrorCode","category":"coding","domain":null,"eligible":true,"ranked":true,"score":65.55555555555556,"unit":"percent","family":"epoch-ai:epoch-mirrorcode:overall","index":78.35575508791557,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":10.846044638639055,"costPerTest":null,"latency":null},{"benchmarkId":"cursorBench32","benchmarkName":"cursorBench32","category":"coding","score":62.3,"unit":"percent","family":"cursorBench32","index":67.02635921507661,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"parseBenchLlamaIndex","benchmarkName":"ParseBench: document parsing for AI agents (LlamaIndex leaderboard)","category":"multimodal","score":62.27,"unit":"percent","family":"parseBenchLlamaIndex","index":60.600743163502734,"aboveFit":false,"run":null,"source":"Boyang Zhang, Sebastián G. Acosta, Preston Carlson, Sacha Bron, Pierre-Loïc Doulcet, Daniel B. Ospina and Simon Suo (LlamaIndex)","sourceUrl":"https://arxiv.org/abs/2604.08538","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaIfBench","benchmarkName":"Artificial Analysis IFBench","category":"instruction","score":62.2,"unit":"percent","family":"ifbench-family","index":47.15018963445999,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:public-benefits-bench-v1:overall","benchmarkName":"Public Benefits Bench v1","category":null,"domain":"Public benefits","eligible":false,"ranked":true,"score":62.111,"unit":"percent","family":"vals-ai:public-benefits-bench-v1:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/public-benefits-bench-v1","measuredAt":"2026-06-09","stderr":1.262,"costPerTest":1.316771,"latency":597.691},{"benchmarkId":"toolathlon","benchmarkName":"Toolathlon","category":"agentic","score":59.9,"unit":"percent","family":"toolathlon","index":70.63621010667275,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:skillsbench:overall","benchmarkName":"SkillsBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":59.227,"unit":"percent","family":"vals-ai:skillsbench:overall","index":69.22610492114168,"aboveFit":false,"run":"OpenHands","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/skillsbench","measuredAt":"2026-10-29","stderr":4.5,"costPerTest":4.766601,"latency":930.403},{"benchmarkId":"cursorBench31","benchmarkName":"cursorBench31","category":"coding","score":58.4,"unit":"percent","family":"cursorBench31","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"terminalBenchHard","benchmarkName":"Terminal-Bench Hard","category":"agentic","score":58.3,"unit":"percent","family":"terminalBenchHard","index":66.94287513725891,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/terminalbench-hard","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hle","benchmarkName":"Humanity's Last Exam","category":"knowledge","score":57.9,"unit":"percent","family":"hle","index":82.47729425053856,"aboveFit":false,"run":null,"source":"Center for AI Safety, Scale AI, and thousands of expert contributors","sourceUrl":"https://lastexam.ai/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tier-4-v2-private:overall","benchmarkName":"FrontierMath-Tier-4-v2-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":56.09756097560976,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tier-4-v2-private:overall","index":70.5262335026826,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":7.846686801046543,"costPerTest":null,"latency":null},{"benchmarkId":"aaSciCode","benchmarkName":"Artificial Analysis SciCode","category":"coding","score":54.4,"unit":"percent","family":"scicode-family","index":66.76059680747275,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/scicode","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:fabv2:overall","benchmarkName":"Finance Agent (v2)","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":53.918,"unit":"percent","family":"vals-ai:fabv2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/fabv2","measuredAt":"2026-10-29","stderr":0.159,"costPerTest":4.218538,"latency":547.397},{"benchmarkId":"vals-ai:medcode:overall","benchmarkName":"MedCode","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":53.217,"unit":"percent","family":"vals-ai:medcode:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medcode","measuredAt":"2026-10-29","stderr":2.165,"costPerTest":0.350925,"latency":105.915},{"benchmarkId":"epoch-ai:epoch-simpleqa-verified:overall","benchmarkName":"SimpleQA Verified","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":53,"unit":"percent","family":"epoch-ai:epoch-simpleqa-verified:overall","index":68.0727310582551,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.5790799515836722,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-chess-puzzles:overall","benchmarkName":"Chess Puzzles","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":52,"unit":"percent","family":"epoch-ai:epoch-chess-puzzles:overall","index":73.45346587294395,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":5.021167315686783,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-agentic-coding:overall","benchmarkName":"LiveBench Agentic Coding","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":50.504999999999995,"unit":"percent","family":"livebench:livebench-agentic-coding:overall","index":61.24763667760034,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Agentic%20Coding","measuredAt":"2026-06-25","stderr":null,"costPerTest":4.955130555555556,"latency":null},{"benchmarkId":"hleNoTools","benchmarkName":"Humanity's Last Exam without tools","category":"knowledge","score":49.8,"unit":"percent","family":"hle","index":74.35062301579724,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"omniscienceAccuracy","benchmarkName":"Artificial Analysis Omniscience Accuracy","category":"knowledge","score":48.8,"unit":"percent","family":"omniscienceAccuracy","index":72.19044248872996,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":48.7,"unit":"percent","family":"hle","index":73.24700099626446,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-long-horizon:overall","benchmarkName":"LiveBench Long Agentic","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":48.3,"unit":"percent","family":"livebench-2-agentic","index":53.91276346702827,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Long%20Agentic","measuredAt":"2026-10-14","stderr":null,"costPerTest":51.96964444444445,"latency":null},{"benchmarkId":"gdpvalAaNormalized","benchmarkName":"GDPval-AA normalized","category":"agentic","score":47.7,"unit":"percent","family":"gdpvalAaNormalized","index":68.13816644989318,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:code-migration:overall","benchmarkName":"Code Migration","category":"coding","domain":null,"eligible":true,"ranked":true,"score":47.25,"unit":"percent","family":"vals-ai:code-migration:overall","index":70.29023187202641,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/code-migration","measuredAt":"2026-10-29","stderr":4.176,"costPerTest":30.514184,"latency":4498.693},{"benchmarkId":"epoch-ai:epoch-frontiermath-2025-02-28-private:overall","benchmarkName":"FrontierMath-2025-02-28-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":47.241379310344826,"unit":"percent","family":"epoch-ai:epoch-frontiermath-2025-02-28-private:overall","index":71.61780605137685,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.9366965952163047,"costPerTest":null,"latency":null},{"benchmarkId":"frontierCode","benchmarkName":"FrontierCode 1.1 Main","category":"coding","score":46.5,"unit":"percent","family":"frontierCode","index":70.85185555621923,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaAnalystAgent","benchmarkName":"Artificial Analysis AnalystAgent","category":"agentic","score":45,"unit":"percent","family":"aaAnalystAgent","index":66.3605953841917,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/aa-analyst-agent","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaEnterpriseOpsGym","benchmarkName":"Artificial Analysis EnterpriseOps-Gym","category":"agentic","score":44,"unit":"percent","family":"aaEnterpriseOpsGym","index":null,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/enterprise-ops-gym-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:legal_research:overall","benchmarkName":"Legal Research Bench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":43.75,"unit":"percent","family":"vals-ai:legal_research:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_research","measuredAt":"2026-11-09","stderr":3.448,"costPerTest":2.820942,"latency":714.036},{"benchmarkId":"epoch-ai:epoch-furniture-assembly:overall","benchmarkName":"Furniture Assembly","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":42.5,"unit":"percent","family":"epoch-ai:epoch-furniture-assembly:overall","index":66.47602476771101,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":6.380695638890652,"costPerTest":null,"latency":null},{"benchmarkId":"tau2-bench:banking-knowledge:overall","benchmarkName":"τ²-bench Banking","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":39.69072164948454,"unit":"percent","family":"tau2-bench","index":29.12713582470158,"aboveFit":false,"run":"max effort · Sierra","source":"Sierra Research","sourceUrl":"https://github.com/sierra-research/tau2-bench","measuredAt":"2026-10-20","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-oeis-open-lite:overall","benchmarkName":"OEIS Open Lite","category":"math","domain":null,"eligible":true,"ranked":true,"score":39,"unit":"percent","family":"epoch-ai:epoch-oeis-open-lite:overall","index":null,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":4.902071300001973,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-mystery-game-puzzles:overall","benchmarkName":"Mystery Game Puzzles","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":36,"unit":"percent","family":"epoch-ai:epoch-mystery-game-puzzles:overall","index":66.55993448345637,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":4.824181513244218,"costPerTest":null,"latency":null},{"benchmarkId":"postTrainBench11","benchmarkName":"PostTrainBench v1.1","category":"coding","score":32.9,"unit":"percent","family":"postTrainBench11","index":68.32381168051438,"aboveFit":false,"run":null,"source":"PostTrainBench team","sourceUrl":"https://posttrainbench.com/?version=v1.1","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-private:overall","benchmarkName":"FrontierMath-Tier-4-2025-07-01-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":31.25,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-private:overall","index":74.94244148377315,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":6.761023307918171,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:tax_agent_bench:overall","benchmarkName":"Tax Agent Bench","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":29.999,"unit":"percent","family":"vals-ai:tax_agent_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench","measuredAt":"2026-11-09","stderr":3.303,"costPerTest":1.68964,"latency":479.2},{"benchmarkId":"epoch-ai:epoch-oeis-open:overall","benchmarkName":"OEIS Open","category":"math","domain":null,"eligible":true,"ranked":true,"score":29.878048780487802,"unit":"percent","family":"epoch-ai:epoch-oeis-open:overall","index":null,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.065676772328729,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-ebr-bench:overall","benchmarkName":"EBR-bench","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":28.57142857142857,"unit":"percent","family":"epoch-ai:epoch-ebr-bench:overall","index":65.53898960423028,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.4197256999998662,"costPerTest":null,"latency":null},{"benchmarkId":"terminal-bench:terminal-bench-4-0-0:overall","benchmarkName":"Terminal-Bench 4.0.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":23.64,"unit":"percent","family":"terminal-bench-4","index":65.74119871259748,"aboveFit":false,"run":"max effort · Claude Code","source":"Terminal-Bench","sourceUrl":"https://www.tbench.ai/leaderboard","measuredAt":"2026-09-29","stderr":1.816326530612245,"costPerTest":19.64018181818182,"latency":5163},{"benchmarkId":"terminal-bench:terminal-bench-4-1-0:overall","benchmarkName":"Terminal-Bench 4.1.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":23.64,"unit":"percent","family":"terminal-bench:terminal-bench-4-1-0:overall","index":64.00119977265344,"aboveFit":false,"run":"max effort · Claude Code","source":"Terminal-Bench","sourceUrl":"https://www.tbench.ai/leaderboard","measuredAt":"2026-09-29","stderr":1.8316326530612244,"costPerTest":19.64018181818182,"latency":5163},{"benchmarkId":"vals-ai:terminal-bench-4:overall","benchmarkName":"Terminal-Bench 4.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":23.232,"unit":"percent","family":"terminal-bench-4","index":65.54746455097859,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-4","measuredAt":"2026-11-09","stderr":1.336,"costPerTest":17.140717,"latency":4283.164},{"benchmarkId":"researchClawBench","benchmarkName":"ResearchClawBench","category":"agentic","score":21.1,"unit":"percent","family":"researchClawBench","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"frontierBench","benchmarkName":"Terminal-Bench 3.0","category":"agentic","score":21.08,"unit":"percent","family":"frontierBench","index":67.85445482285712,"aboveFit":false,"run":null,"source":"Ryan Marten, Alex Shaw, Andy Konwinski, Harbor, and the Laude Institute","sourceUrl":"https://www.frontierbench.ai/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":20.9,"unit":"percent","family":"critpt","index":77.30907733177966,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"osWorld2","benchmarkName":"OSWorld 2.0","category":"agentic","score":20.6,"unit":"percent","family":"osWorld2","index":62.400373594925966,"aboveFit":false,"run":null,"source":"Mengqi Yuan, Zilong Zhou, Xinzhuang Xiong, Weiming Wu, Jiayang Sun, Jiamin Song, Kaiqian Cui, Bowen Wang, Haoyuan Wu, Yitong Li, Dunjie Lu, Haikong Lu, Qi Zhen, Xinyuan Wang, Jiaqi Deng, Yuhao Yang, Cheng Chen, Boyuan Zheng, Alex Su, Xiao Yu, Hao Zou, Saaket Agashe, Xing Han Lu, Manpreet Kaur, Zhengyang Qi, Vincent Sunn Chen, Frederic Sala, Dayiheng Liu, Junyang Lin, Zhou Yu, Yu Su, Siva Reddy, Xin Eric Wang, Peng Qi, Tianbao Xie, Tao Yu","sourceUrl":"https://arxiv.org/abs/2606.29537","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"frontierCode2","benchmarkName":"FrontierCode 2.0","category":"coding","score":14.7,"unit":"percent","family":"frontierCode2","index":62.92557861808323,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:time_horizon_index:overall","benchmarkName":"Time Horizon Index: KSP","category":null,"domain":"Other","eligible":false,"ranked":true,"score":11.833,"unit":"percent","family":"vals-ai:time_horizon_index:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/time_horizon_index","measuredAt":"2026-10-07","stderr":0,"costPerTest":1499.319504,"latency":0},{"benchmarkId":"lmarena:agent-bash-recovery:overall","benchmarkName":"Agent Arena command recovery","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":7.22358959596161,"unit":"index","family":"agent-arena","index":72.4697244102492,"aboveFit":false,"run":"high effort","source":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset","measuredAt":"2026-10-08","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:terminal-bench-science:overall","benchmarkName":"Terminal-Bench-Science","category":null,"domain":"Science","eligible":false,"ranked":true,"score":4.286,"unit":"percent","family":"vals-ai:terminal-bench-science:overall","index":null,"aboveFit":false,"run":"Mini-SWE-agent","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-science","measuredAt":"2026-11-09","stderr":2.438,"costPerTest":23.141733,"latency":6906.538},{"benchmarkId":"arc-prize:v3-semi-private:overall","benchmarkName":"ARC-AGI-3 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":1.52,"unit":"percent","family":"arc-prize:v3-semi-private:overall","index":65.9513787493042,"aboveFit":false,"run":"high effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:programbench:overall","benchmarkName":"ProgramBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":1,"unit":"percent","family":"vals-ai:programbench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/programbench","measuredAt":"2026-11-09","stderr":0.705,"costPerTest":31.267424,"latency":5523.099},{"benchmarkId":"mazeBench","benchmarkName":"MazeBench: visual spatial reasoning in a 3D open world","category":"agentic","score":0,"unit":"percent","family":"mazeBench","index":61.97866734834783,"aboveFit":false,"run":null,"source":"Jonathan Pappas and David Pappas","sourceUrl":"https://mazebench.com/blog?post=maze-bench-results","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":5,"outputPerMillion":25,"batch":{"inputPerMillion":2.5,"outputPerMillion":12.5},"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":69,"timeToFirstToken":0.819,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":{"elo":1453.7424108187697,"eloUpper":1457.6019224375354,"eloLower":1449.8828992000044,"votes":64170,"rank":48,"rankUpper":null,"rankLower":null,"arenaName":"Claude Opus 4.8","organization":"anthropic","updatedAt":"2026-11-09"},"quality":{"coveragePercent":100,"benchmarkCount":67,"categoryCount":8,"confidence":"high","state":"available"},"hosts":[{"host":"amazon-bedrock","name":"Amazon Bedrock","kind":"cloud","url":"https://docs.aws.amazon.com/bedrock/latest/userguide/models-supported.html","inputPerMillion":5,"outputPerMillion":25,"cacheReadPerMillion":0.5,"contextTokens":1000000,"maxOutputTokens":128000,"deprecated":false},{"host":"digitalocean","name":"DigitalOcean","kind":"cloud","url":"https://docs.digitalocean.com/products/gradient-ai-platform/details/models/","inputPerMillion":5,"outputPerMillion":25,"cacheReadPerMillion":0.5,"contextTokens":1000000,"maxOutputTokens":128000,"deprecated":false},{"host":"gmi-cloud","name":"GMI Cloud","kind":"host","url":"https://docs.gmicloud.ai/inference-engine/api-reference/llm-api-reference","inputPerMillion":5,"outputPerMillion":25,"cacheReadPerMillion":0.5,"contextTokens":1000000,"maxOutputTokens":128000,"deprecated":false},{"host":"google-vertex","name":"Google Vertex AI","kind":"cloud","url":"https://cloud.google.com/vertex-ai/generative-ai/docs/models","inputPerMillion":5,"outputPerMillion":25,"cacheReadPerMillion":0.5,"contextTokens":1000000,"maxOutputTokens":128000,"deprecated":false},{"host":"azure","name":"Microsoft Azure","kind":"cloud","url":"https://learn.microsoft.com/en-us/azure/ai-services/openai/concepts/models","inputPerMillion":5,"outputPerMillion":25,"cacheReadPerMillion":0.5,"contextTokens":1000000,"maxOutputTokens":128000,"deprecated":false},{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":5,"outputPerMillion":25,"cacheReadPerMillion":0.5,"contextTokens":1000000,"maxOutputTokens":128000,"deprecated":false},{"host":"sap-ai-core","name":"SAP AI Core","kind":"cloud","url":"https://help.sap.com/docs/sap-ai-core","inputPerMillion":5,"outputPerMillion":25,"cacheReadPerMillion":0.5,"contextTokens":1000000,"maxOutputTokens":128000,"deprecated":false},{"host":"snowflake","name":"Snowflake Cortex","kind":"cloud","url":"https://docs.snowflake.com/en/user-guide/snowflake-cortex/cortex-rest-api","inputPerMillion":5,"outputPerMillion":25,"cacheReadPerMillion":0.5,"contextTokens":1000000,"maxOutputTokens":128000,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:livebench","name":"LiveBench","url":"https://livebench.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:arc-prize-foundation","name":"ARC Prize Foundation","url":"https://arcprize.org","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:sierra-research","name":"Sierra Research","url":"https://github.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:terminal-bench","name":"Terminal-Bench","url":"https://www.tbench.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:lmarena","name":"LMArena","url":"https://huggingface.co","license":"CC BY 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"ling-3-1-flash","providerId":"inclusionai","slug":"ling-3-1-flash","name":"Ling 3.1 Flash","provider":"InclusionAI","description":"Ling 3.1 Flash is a reasoning model from InclusionAI in the Ling 3.1 family. 15 benchmarks count toward its index, in 4 categories.","logoUrl":null,"releaseDate":"2026-09-29","contextWindowTokens":1000000,"modalities":["text"],"openWeights":true,"details":{"apiId":"inclusionai/ling-3.1-flash","maxOutputTokens":262144,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":false,"cacheReadPerMillion":0.009,"weightsUrl":"https://huggingface.co/inclusionAI/Ling-3.1-flash"},"reasoning":true,"overallScore":67.20803735717801,"overallScoreRange":{"low":60.03296130996642,"high":74.3831134043896},"badges":[],"capabilityScores":{"agentic":66.64445855749642,"coding":66.8742716221619,"reasoning":67.96887439806669,"multimodal":null,"knowledge":70.35820047231141,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"agentic":9,"reasoning":2,"coding":2,"knowledge":2},"benchmarks":[{"benchmarkId":"browseComp","benchmarkName":"BrowseComp","category":"agentic","score":91.67,"unit":"percent","family":"browseComp","index":72.3301096665553,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"cyberGym","benchmarkName":"CyberGym","category":"agentic","score":87.9,"unit":"percent","family":"cyberGym","index":71.59023671144567,"aboveFit":false,"run":null,"source":"Zhun Wang, Tianneng Shi, Jingxuan He, Matthew Cai, Jialin Zhang, Dawn Song","sourceUrl":"https://www.cybergym.io/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"draco","benchmarkName":"Data Research and Analysis with Complex Operations","category":"agentic","score":85.49,"unit":"percent","family":"draco","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"wideResearch","benchmarkName":"WideResearch","category":"agentic","score":83.32,"unit":"percent","family":"wideResearch","index":65.42045085906301,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mcpAtlas","benchmarkName":"MCP Atlas","category":"agentic","score":83,"unit":"percent","family":"mcpAtlas","index":66.9541177855669,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":83,"unit":"percent","family":"lcr","index":64.2427330827805,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"terminalBench21","benchmarkName":"Terminal-Bench 2.1","category":"agentic","score":81.18,"unit":"percent","family":"terminal-bench-2-1","index":68.61967969527134,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"swePro","benchmarkName":"SWE-bench Pro","category":"coding","score":68.74,"unit":"percent","family":"swePro","index":67.37905378077546,"aboveFit":false,"run":null,"source":"Xiang Deng, Jeff Da, Edwin Pan, Yannis Yiming He, Charles Ide, Kanak Garg, Niklas Lauffer, Andrew Park, Nitin Pasari, Chetan Rane, Karmini Sampath, Maya Krishnan, Srivatsa Kundurthy, Sean Hendryx, Zifan Wang, Chen Bo Calvin Zhang, Noah Jacobson, Bing Liu, Brad Kenstler","sourceUrl":"https://arxiv.org/abs/2509.16941","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"healthBenchProfessional","benchmarkName":"HealthBench Professional","category":"knowledge","score":65.35,"unit":"percent","family":"healthBenchProfessional","index":77.78324936726463,"aboveFit":false,"run":null,"source":"Rebecca Soskin Hicks, Mikhail Trofimov, Dominick Lim, Rahul K. Arora, Foivos Tsimpourlas, Preston Bowman, Michael Sharman, Chi Tong, Kavin Karthik, Arnav Dugar, Akshay Jagadeesh, Khaled Saab, Johannes Heidecke, Ashley Alexander, Nate Gross, Karan Singhal","sourceUrl":"https://arxiv.org/abs/2604.27470","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"sweAtlasCodebaseQna","benchmarkName":"SWE-Atlas Codebase QnA","category":"coding","score":64.51,"unit":"percent","family":"sweAtlasCodebaseQna","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"deepSwe","benchmarkName":"DeepSWE","category":"agentic","score":59.7,"unit":"percent","family":"deepSwe","index":64.47695404730526,"aboveFit":false,"run":null,"source":"Datacurve AI","sourceUrl":"https://deepswe.datacurve.ai/blog","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gdpvalAaNormalized","benchmarkName":"GDPval-AA normalized","category":"agentic","score":55.9,"unit":"percent","family":"gdpvalAaNormalized","index":74.00097589576619,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaSciCode","benchmarkName":"Artificial Analysis SciCode","category":"coding","score":54.1,"unit":"percent","family":"scicode-family","index":66.36948946354832,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/scicode","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"automationBench","benchmarkName":"AutomationBench","category":"agentic","score":52.5,"unit":"percent","family":"automationBench","index":81.95023432494372,"aboveFit":true,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"tau3Bench","benchmarkName":"τ³-Bench Tool-Agent-User Evaluation","category":"agentic","score":47.8,"unit":"percent","family":"tau3Bench","index":34.457368031550395,"aboveFit":false,"run":null,"source":"Sierra Research","sourceUrl":"https://github.com/sierra-research/tau2-bench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":39.4,"unit":"percent","family":"hle","index":63.91637846748738,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hle","benchmarkName":"Humanity's Last Exam","category":"knowledge","score":37.44,"unit":"percent","family":"hle","index":61.94992468722897,"aboveFit":false,"run":null,"source":"Center for AI Safety, Scale AI, and thousands of expert contributors","sourceUrl":"https://lastexam.ai/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":18,"unit":"percent","family":"critpt","index":71.69501571335289,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":0.045,"outputPerMillion":0.135,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":81,"timeToFirstToken":2.211,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":70,"benchmarkCount":15,"categoryCount":4,"confidence":"low","state":"available"},"hosts":[{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":0.045,"outputPerMillion":0.135,"cacheReadPerMillion":0.009,"contextTokens":1000000,"maxOutputTokens":262144,"deprecated":false},{"host":"novita-ai","name":"Novita AI","kind":"host","url":"https://novita.ai/docs/guides/introduction","inputPerMillion":0.3,"outputPerMillion":0.9,"cacheReadPerMillion":0.06,"contextTokens":1000000,"maxOutputTokens":262144,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"kimi-3","providerId":"moonshot-ai","slug":"kimi-k3","name":"Kimi K3","provider":"Moonshot AI","description":"Kimi K3 is a reasoning model from Moonshot AI. 73 benchmarks count toward its index, in 7 categories.","logoUrl":null,"releaseDate":"2026-07-16","contextWindowTokens":1048576,"modalities":["text","image","video"],"openWeights":true,"details":{"apiId":"moonshotai/kimi-k3","maxOutputTokens":943718,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.182,"weightsUrl":"https://huggingface.co/moonshotai/Kimi-K3"},"reasoning":true,"overallScore":66.92745237878269,"overallScoreRange":{"low":64.53489702862126,"high":69.32000772894412},"badges":[],"capabilityScores":{"agentic":65.94805277995842,"coding":66.20833584139869,"reasoning":67.71575404750686,"multimodal":65.60269888044884,"knowledge":66.29134658676847,"multilingual":null,"instruction":70.98484057050291,"math":66.56773875143807},"capabilityCounts":{"math":5,"agentic":27,"reasoning":8,"multimodal":7,"knowledge":6,"coding":19,"instruction":1},"benchmarks":[{"benchmarkId":"mathVisionPython","benchmarkName":"MathVision with Python","category":"multimodal","score":97.8,"unit":"percent","family":"mathVisionPython","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":97.22222222222221,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":65.48240087839254,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.1158359139033183,"costPerTest":null,"latency":null},{"benchmarkId":"deepSearchQa","benchmarkName":"DeepSearchQA","category":"agentic","score":95,"unit":"percent","family":"deepSearchQa","index":69.44790025841915,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"arc-prize:v1-public-eval:overall","benchmarkName":"ARC-AGI-1 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":94.875,"unit":"percent","family":"arc-prize:v1-public-eval:overall","index":null,"aboveFit":false,"run":"max effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.5450828250000002,"latency":null},{"benchmarkId":"aaHarveyLab","benchmarkName":"Artificial Analysis Harvey LAB-AA v1.0","category":"agentic","score":94.6,"unit":"percent","family":"aaHarveyLab","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"arc-prize:v1-semi-private:overall","benchmarkName":"ARC-AGI-1 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":94.5,"unit":"percent","family":"arc-prize:v1-semi-private:overall","index":70.43242059127125,"aboveFit":false,"run":"max effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.7704815999999995,"latency":null},{"benchmarkId":"mathVision","benchmarkName":"MathVision","category":"multimodal","score":94.3,"unit":"percent","family":"mathVision","index":64.98600682818528,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"cyberHarmfulRefusal","benchmarkName":"Refusal of harmful cyber requests (JailbreakBench, StrongREJECT, AgentHarm)","category":"agentic","score":94,"unit":"percent","family":"cyberHarmfulRefusal","index":null,"aboveFit":false,"run":null,"source":"Mistral AI (composite of JailbreakBench, StrongREJECT and AgentHarm)","sourceUrl":"https://mistral.ai/news/mistral-large-4-oss/","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gpqa","benchmarkName":"Graduate-Level Google-Proof Q&A","category":"knowledge","score":93.5,"unit":"percent","family":"gpqa-diamond","index":62.96474584284943,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gpqaDiamond","benchmarkName":"GPQA Diamond","category":"knowledge","score":93.5,"unit":"percent","family":"gpqa-diamond","index":62.96474584284943,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":93.5,"unit":"percent","family":"gpqa-diamond","index":62.96474584284943,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:swebench:overall","benchmarkName":"SWE-bench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":93.4,"unit":"percent","family":"swe-bench-verified","index":70.11580529446847,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/swebench","measuredAt":"2026-09-01","stderr":1.111,"costPerTest":0.759959,"latency":619.492},{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":93.11868686868688,"unit":"percent","family":"gpqa-diamond","index":62.615015858220936,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.493722536419486,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:gpqa:overall","benchmarkName":"GPQA Diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":92.929,"unit":"percent","family":"gpqa-diamond","index":62.441040245470234,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","measuredAt":"2026-09-01","stderr":1.309,"costPerTest":0.088491,"latency":122.425},{"benchmarkId":"charxiv","benchmarkName":"CharXiv Reasoning","category":"multimodal","score":91.3,"unit":"percent","family":"charxiv","index":68.12910728563736,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"browseComp","benchmarkName":"BrowseComp","category":"agentic","score":91.2,"unit":"percent","family":"browseComp","index":71.95880035773561,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"omniDocBench","benchmarkName":"OmniDocBench","category":"multimodal","score":91.1,"unit":"percent","family":"omniDocBench","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-reasoning:overall","benchmarkName":"LiveBench Reasoning","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":90.673,"unit":"percent","family":"livebench","index":79.08060806801612,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Reasoning","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.19509603960396038,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":88.7,"unit":"percent","family":"lcr","index":68.15339207861877,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"terminalBench21","benchmarkName":"Terminal-Bench 2.1","category":"agentic","score":88.3,"unit":"percent","family":"terminal-bench-2-1","index":72.78384989031893,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:mmmu:overall","benchmarkName":"MMMU Pro","category":"multimodal","domain":null,"eligible":true,"ranked":true,"score":88.15,"unit":"percent","family":"mmmu-pro","index":69.87230619809009,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmmu","measuredAt":"2026-09-01","stderr":0.777,"costPerTest":0.059115,"latency":106.516},{"benchmarkId":"vals-ai:medscribe:overall","benchmarkName":"MedScribe","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":88.048,"unit":"percent","family":"vals-ai:medscribe:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medscribe","measuredAt":"2026-10-29","stderr":1.976,"costPerTest":0.110262,"latency":38.241},{"benchmarkId":"b3AgentSecurity","benchmarkName":"b3: Backbone Breaker Benchmark","category":"agentic","score":88,"unit":"percent","family":"b3AgentSecurity","index":null,"aboveFit":false,"run":null,"source":"Lakera with the UK AI Security Institute","sourceUrl":"https://www.lakera.ai/blog/the-backbone-breaker-benchmark","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:mmlu_pro:overall","benchmarkName":"MMLU Pro","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":87.973,"unit":"percent","family":"mmlu-pro","index":55.31617832952955,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","measuredAt":"2026-09-01","stderr":0.321,"costPerTest":0.021327,"latency":38.718},{"benchmarkId":"vals-ai:lcb:overall","benchmarkName":"LiveCodeBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":87.185,"unit":"percent","family":"livecodebench","index":61.684755554102594,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","measuredAt":"2026-09-01","stderr":0.971,"costPerTest":0.103719,"latency":200.263},{"benchmarkId":"vals-ai:proof_bench:overall","benchmarkName":"ProofBench v1.1","category":"math","domain":null,"eligible":true,"ranked":true,"score":87,"unit":"percent","family":"vals-ai:proof_bench:overall","index":77.19290957694369,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/proof_bench","measuredAt":"2026-11-09","stderr":3.38,"costPerTest":1.646802,"latency":2070.511},{"benchmarkId":"vals-ai:sage:overall","benchmarkName":"SAGE","category":null,"domain":"Education","eligible":false,"ranked":true,"score":86.773,"unit":"percent","family":"vals-ai:sage:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/sage","measuredAt":"2026-11-05","stderr":1.85,"costPerTest":0.227261,"latency":145.052},{"benchmarkId":"vals-ai:legal_bench:overall","benchmarkName":"LegalBench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":86.212,"unit":"percent","family":"vals-ai:legal_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","measuredAt":"2026-10-13","stderr":0.412,"costPerTest":0.007921,"latency":7.106},{"benchmarkId":"babyVisionPython","benchmarkName":"BabyVision with Python","category":"multimodal","score":85.7,"unit":"percent","family":"babyVisionPython","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-language:overall","benchmarkName":"LiveBench Language","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":85.528,"unit":"percent","family":"livebench:livebench-language:overall","index":71.27462679488328,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Language","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.14656000000000002,"latency":null},{"benchmarkId":"aaTerminalBench21","benchmarkName":"Artificial Analysis Terminal-Bench v2.1","category":"agentic","score":85,"unit":"percent","family":"aaTerminalBench21","index":67.60621976966686,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/terminalbench-v2-1","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:vibe-code:overall","benchmarkName":"Vibe Code Bench v1.1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":84.968,"unit":"percent","family":"vals-ai:vibe-code:overall","index":72.90731193836181,"aboveFit":false,"run":"OpenHands","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","measuredAt":"2026-11-09","stderr":2.746,"costPerTest":10.007576,"latency":999.021},{"benchmarkId":"charxivNoTools","benchmarkName":"CharXiv Reasoning without tools","category":"multimodal","score":84.8,"unit":"percent","family":"charxivNoTools","index":65.62757687074334,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-math:overall","benchmarkName":"LiveBench Mathematics","category":"math","domain":null,"eligible":true,"ranked":true,"score":84.43675,"unit":"percent","family":"livebench:livebench-math:overall","index":57.455951807604464,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Mathematics","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.21042321899736147,"latency":null},{"benchmarkId":"mcpAtlas","benchmarkName":"MCP Atlas","category":"agentic","score":84.2,"unit":"percent","family":"mcpAtlas","index":67.76219250085303,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmmuProPython","benchmarkName":"MMMU-Pro with Python","category":"multimodal","score":83.4,"unit":"percent","family":"mmmuProPython","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"extractBenchLlamaIndex","benchmarkName":"ExtractBench: schema-guided enterprise document extraction (LlamaIndex leaderboard)","category":"multimodal","score":83.17,"unit":"percent","family":"extractBenchLlamaIndex","index":62.38515595193897,"aboveFit":false,"run":null,"source":"Boyang Zhang, Adrian Lyjak, Eli Stewart, Zhaoqi Li and Simon Suo (LlamaIndex)","sourceUrl":"https://arxiv.org/abs/2607.29677","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmmuPro","benchmarkName":"Massive Multi-discipline Multimodal Understanding Pro","category":"multimodal","score":81.6,"unit":"percent","family":"mmmu-pro","index":61.348055861559544,"aboveFit":false,"run":null,"source":"MMMU-Pro authors","sourceUrl":"https://arxiv.org/abs/2409.02813","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-coding:overall","benchmarkName":"LiveBench Coding","category":"coding","domain":null,"eligible":true,"ranked":true,"score":81.4455,"unit":"percent","family":"livebench:livebench-coding:overall","index":69.36981846599397,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Coding","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.05036752136752137,"latency":null},{"benchmarkId":"frontierSwe","benchmarkName":"FrontierSWE","category":"coding","score":81.2,"unit":"percent","family":"frontierSwe","index":null,"aboveFit":false,"run":null,"source":"Evan Chu, Rajan Agarwal, Abishek Thangamuthu, Brendan Graham, Justus Mattern","sourceUrl":"https://www.frontierswe.com/blog","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:terminal-bench-2-1:overall","benchmarkName":"Terminal-Bench 2.1","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":80.899,"unit":"percent","family":"terminal-bench-2-1","index":68.45533533785442,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2-1","measuredAt":"2026-10-01","stderr":0.649,"costPerTest":0.341034,"latency":608.575},{"benchmarkId":"aaMmmuPro","benchmarkName":"Artificial Analysis MMMU-Pro","category":"multimodal","score":80.5,"unit":"percent","family":"mmmu-pro","index":59.91650236992847,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmmu-pro","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-research:overall","benchmarkName":"LiveBench Research","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":79,"unit":"percent","family":"livebench-2-agentic","index":76.36462808108631,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Research","measuredAt":"2026-10-14","stderr":null,"costPerTest":2.4049363636363634,"latency":null},{"benchmarkId":"livebench:livebench-data-analysis:overall","benchmarkName":"LiveBench Data Analysis","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":78.73366666666668,"unit":"percent","family":"livebench","index":63.61398977669035,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Data%20Analysis","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.35235533333333335,"latency":null},{"benchmarkId":"programBench","benchmarkName":"ProgramBench: Can Language Models Rebuild Programs From Scratch?","category":"coding","score":77.8,"unit":"percent","family":"programBench","index":89.80584139004777,"aboveFit":false,"run":null,"source":"John Yang, Kilian Lieret, Jeffrey Ma, Parth Thakkar, Dmitrii Pedchenko, Sten Sootla, Emily McMilin, Pengcheng Yin, Rui Hou, Gabriel Synnaeve, Diyi Yang, Ofir Press","sourceUrl":"https://programbench.com/static/paper.pdf","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"finch","benchmarkName":"Finch (FinWorkBench): Finance & Accounting across Spreadsheet-Centric Enterprise Workflows","category":"agentic","score":77.3,"unit":"percent","family":"finch","index":null,"aboveFit":false,"run":null,"source":"Finch authors (ACL 2026 Findings)","sourceUrl":"https://arxiv.org/abs/2512.13168","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-design:overall","benchmarkName":"LiveBench Design","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":77.2,"unit":"percent","family":"livebench-2-agentic","index":75.04823211022948,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Design","measuredAt":"2026-10-14","stderr":null,"costPerTest":1.00074,"latency":null},{"benchmarkId":"vals-ai:tax_eval_v2:overall","benchmarkName":"TaxEval v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":75.716,"unit":"percent","family":"vals-ai:tax_eval_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","measuredAt":"2026-09-01","stderr":0.841,"costPerTest":0.078156,"latency":144.708},{"benchmarkId":"vals-ai:cyber:overall","benchmarkName":"CyberBench v1.1","category":null,"domain":"Other","eligible":false,"ranked":true,"score":75.238,"unit":"percent","family":"vals-ai:cyber:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/cyber","measuredAt":"2026-11-05","stderr":5.562,"costPerTest":2.12561,"latency":1844.34},{"benchmarkId":"vulcanBench","benchmarkName":"VulcanBench v3","category":"coding","score":73.7,"unit":"percent","family":"vulcanBench","index":50.61409202724054,"aboveFit":false,"run":null,"source":"VulcanBench contributors","sourceUrl":"https://github.com/morganlinton/VulcanBench/tree/main","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"deckBench","benchmarkName":"DECK-Bench (Internal)","category":"agentic","score":73.5,"unit":"percent","family":"deckBench","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"toolathlonVerified","benchmarkName":"Toolathlon-Verified","category":"agentic","score":73.2,"unit":"percent","family":"toolathlonVerified","index":68.05090342535203,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"kimiCodeBenchV2","benchmarkName":"Kimi Code Bench v2","category":"coding","score":72.9,"unit":"percent","family":"kimiCodeBenchV2","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:biomysterybench:overall","benchmarkName":"BioMysteryBench","category":null,"domain":"Science","eligible":false,"ranked":true,"score":72.593,"unit":"percent","family":"vals-ai:biomysterybench:overall","index":null,"aboveFit":false,"run":"Terminus 2","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/biomysterybench","measuredAt":"2026-11-09","stderr":1.614,"costPerTest":1.060934,"latency":295.415},{"benchmarkId":"epoch-ai:epoch-frontiermath-tiers-1-3-v2-private:overall","benchmarkName":"FrontierMath-Tiers-1-3-v2-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":72.1830985915493,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tiers-1-3-v2-private:overall","index":68.88839377582893,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.6636607935112644,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:corp_fin_v2:overall","benchmarkName":"CorpFin v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":71.562,"unit":"percent","family":"vals-ai:corp_fin_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","measuredAt":"2026-08-12","stderr":0.888,"costPerTest":0.063819,"latency":37.99},{"benchmarkId":"hermesBench","benchmarkName":"Hermes Bench: 150 agent tasks graded on the files and state the agent leaves behind","category":"agentic","score":71.52,"unit":"percent","family":"hermesBench","index":null,"aboveFit":false,"run":null,"source":"Nous Research","sourceUrl":"https://portal.nousresearch.com/bench","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-instruction-following:overall","benchmarkName":"LiveBench Instruction Following","category":"instruction","domain":null,"eligible":true,"ranked":true,"score":71.36275,"unit":"percent","family":"livebench:livebench-instruction-following:overall","index":70.98484057050291,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=IF","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.117773,"latency":null},{"benchmarkId":"vals-ai:public-benefits-bench:overall","benchmarkName":"Public Benefits Bench v1.1","category":null,"domain":"Public benefits","eligible":false,"ranked":true,"score":68.2,"unit":"percent","family":"vals-ai:public-benefits-bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/public-benefits-bench","measuredAt":"2026-10-12","stderr":1.211,"costPerTest":0.913402,"latency":438.672},{"benchmarkId":"deepSwe","benchmarkName":"DeepSWE","category":"agentic","score":67.5,"unit":"percent","family":"deepSwe","index":70.1473363024581,"aboveFit":false,"run":null,"source":"Datacurve AI","sourceUrl":"https://deepswe.datacurve.ai/blog","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:emb:overall","benchmarkName":"EMB","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":66.677,"unit":"percent","family":"vals-ai:emb:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/emb","measuredAt":"2026-10-20","stderr":2.873,"costPerTest":3.074655,"latency":741.42},{"benchmarkId":"vals-ai:mortgage_tax:overall","benchmarkName":"MortgageTax","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":66.336,"unit":"percent","family":"vals-ai:mortgage_tax:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mortgage_tax","measuredAt":"2026-09-01","stderr":0.909,"costPerTest":0.038981,"latency":78.943},{"benchmarkId":"officeQaPro","benchmarkName":"OfficeQA Pro","category":"multimodal","score":63.3,"unit":"percent","family":"officeQaPro","index":72.88649436357049,"aboveFit":false,"run":null,"source":"OfficeQA Pro authors","sourceUrl":"https://arxiv.org/abs/2603.08655","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-agentic-coding:overall","benchmarkName":"LiveBench Agentic Coding","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":62.17166666666666,"unit":"percent","family":"livebench:livebench-agentic-coding:overall","index":73.27927130799375,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Agentic%20Coding","measuredAt":"2026-06-25","stderr":null,"costPerTest":1.760715277777778,"latency":null},{"benchmarkId":"arc-prize:v2-public-eval:overall","benchmarkName":"ARC-AGI-2 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":61.66666666666667,"unit":"percent","family":"arc-prize:v2-public-eval:overall","index":null,"aboveFit":false,"run":"max effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":1.656023325000001,"latency":null},{"benchmarkId":"scarfBench","benchmarkName":"ScarfBench: Cross-Framework Application Migration in Enterprise Java","category":"coding","score":61.3,"unit":"percent","family":"scarfBench","index":67.8501285823281,"aboveFit":false,"run":null,"source":"Advait Pavuluri, Bridget McGinn, Ashita Saxena, George Safta, Srikanth Tamilselvam, Raju Pavuluri, Michele Merler, Baishakhi Ray, Rahul Krishna (IBM Research)","sourceUrl":"https://arxiv.org/abs/2605.06754","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"cursorBench32","benchmarkName":"cursorBench32","category":"coding","score":60.8,"unit":"percent","family":"cursorBench32","index":65.7936446754023,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"arc-prize:v2-semi-private:overall","benchmarkName":"ARC-AGI-2 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":60.416666666666664,"unit":"percent","family":"arc-prize:v2-semi-private:overall","index":66.92790358621863,"aboveFit":false,"run":"max effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":1.5926940250000003,"latency":null},{"benchmarkId":"aaSciCode","benchmarkName":"Artificial Analysis SciCode","category":"coding","score":59.5,"unit":"percent","family":"scicode-family","index":73.40942165418788,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/scicode","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-coding-2:overall","benchmarkName":"LiveBench Coding 2.0","category":"coding","domain":null,"eligible":true,"ranked":true,"score":59.3,"unit":"percent","family":"livebench:livebench-coding-2:overall","index":51.12439588551907,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Coding","measuredAt":"2026-10-14","stderr":null,"costPerTest":2.0613862068965516,"latency":null},{"benchmarkId":"perceptionBench","benchmarkName":"PerceptionBench (Internal)","category":"multimodal","score":58.5,"unit":"percent","family":"perceptionBench","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaCyberGymE2e","benchmarkName":"Artificial Analysis CyberGym-E2E-AA","category":"agentic","score":58,"unit":"percent","family":"aaCyberGymE2e","index":95,"aboveFit":true,"run":null,"source":"Artificial Analysis; dataset by Berkeley RDI","sourceUrl":"https://artificialanalysis.ai/evaluations/cybergym-e2e-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"openHarmonyBench","benchmarkName":"OpenHarmony Bench v1.0","category":"coding","score":57.3,"unit":"percent","family":"openHarmonyBench","index":64.25562796270816,"aboveFit":false,"run":null,"source":"OpenHarmony Bench authors","sourceUrl":"https://arxiv.org/abs/2608.16022","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hle","benchmarkName":"Humanity's Last Exam","category":"knowledge","score":56,"unit":"percent","family":"hle","index":80.57103803498197,"aboveFit":false,"run":null,"source":"Center for AI Safety, Scale AI, and thousands of expert contributors","sourceUrl":"https://lastexam.ai/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-mirrorcode:overall","benchmarkName":"MirrorCode","category":"coding","domain":null,"eligible":true,"ranked":true,"score":54.44444444444444,"unit":"percent","family":"epoch-ai:epoch-mirrorcode:overall","index":73.4760754900136,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":11.439589045541112,"costPerTest":null,"latency":null},{"benchmarkId":"aaCweBench","benchmarkName":"Artificial Analysis CWE-Bench-AA","category":"agentic","score":53.3,"unit":"percent","family":"aaCweBench","index":66.19692837642404,"aboveFit":false,"run":null,"source":"Artificial Analysis; dataset by Collinear AI","sourceUrl":"https://artificialanalysis.ai/evaluations/cwe-bench-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:fabv2:overall","benchmarkName":"Finance Agent (v2)","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":53.114,"unit":"percent","family":"vals-ai:fabv2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/fabv2","measuredAt":"2026-10-29","stderr":0.374,"costPerTest":1.913224,"latency":285.753},{"benchmarkId":"jobBench","benchmarkName":"JobBench","category":"agentic","score":52.9,"unit":"percent","family":"jobBench","index":65.75253824548768,"aboveFit":false,"run":null,"source":"Yuetai Li, Yichen Feng, Zhangchen Xu, Zixian Ma, Kaiyuan Zheng, Fengqing Jiang, Xinghua Sun, Rulin Shao, Zichen Chen, Yue Huang, Xinyang Han, Brian Lee, Kayla Xu, Shenglai Zeng, Hang Hua, Xiangliang Zhang, Basel Alomair, Ranjay Krishna, Luke Zettlemoyer, Pang Wei Koh, Bhaskar Ramasubramanian, Luyao Niu, Xiang Yue, Radha Poovendran","sourceUrl":"https://arxiv.org/abs/2605.26329","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gdpvalAaNormalized","benchmarkName":"GDPval-AA normalized","category":"agentic","score":51.6,"unit":"percent","family":"gdpvalAaNormalized","index":70.92657582049132,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"worldVqaForceAnswer","benchmarkName":"WorldVQA ForceAnswer","category":"multimodal","score":51,"unit":"percent","family":"worldVqaForceAnswer","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-simpleqa-verified:overall","benchmarkName":"SimpleQA Verified","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":50.6,"unit":"percent","family":"epoch-ai:epoch-simpleqa-verified:overall","index":66.07331919027918,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.5818160898606837,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:medcode:overall","benchmarkName":"MedCode","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":49.357,"unit":"percent","family":"vals-ai:medcode:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medcode","measuredAt":"2026-10-29","stderr":2.202,"costPerTest":0.093857,"latency":36.994},{"benchmarkId":"vals-ai:ioi:overall","benchmarkName":"IOI","category":"coding","domain":null,"eligible":true,"ranked":true,"score":48.944,"unit":"percent","family":"vals-ai:ioi:overall","index":64.491308157979,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/ioi","measuredAt":"2026-10-29","stderr":9.82,"costPerTest":14.165034,"latency":15508.583},{"benchmarkId":"mlsBenchLite","benchmarkName":"MLS-Bench Lite","category":"coding","score":48.3,"unit":"percent","family":"mlsBenchLite","index":null,"aboveFit":false,"run":null,"source":"MLS-Bench","sourceUrl":"https://mls-bench.com/","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaItbench","benchmarkName":"Artificial Analysis ITBench-AA","category":"agentic","score":47.7,"unit":"percent","family":"aaItbench","index":68.67167810005965,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/itbench-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"omniscienceAccuracy","benchmarkName":"Artificial Analysis Omniscience Accuracy","category":"knowledge","score":47.6,"unit":"percent","family":"omniscienceAccuracy","index":70.67813530785851,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":46.9,"unit":"percent","family":"hle","index":71.44107405521083,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:legal_research:overall","benchmarkName":"Legal Research Bench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":46.154,"unit":"percent","family":"vals-ai:legal_research:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_research","measuredAt":"2026-11-09","stderr":3.465,"costPerTest":3.469601,"latency":760.122},{"benchmarkId":"aaTau3Banking","benchmarkName":"Artificial Analysis Tau3-Banking","category":"agentic","score":46,"unit":"percent","family":"aaTau3Banking","index":71.43003478142319,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/tau3-banking","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaEnterpriseOpsGym","benchmarkName":"Artificial Analysis EnterpriseOps-Gym","category":"agentic","score":45.3,"unit":"percent","family":"aaEnterpriseOpsGym","index":null,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/enterprise-ops-gym-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hleNoTools","benchmarkName":"Humanity's Last Exam without tools","category":"knowledge","score":43.5,"unit":"percent","family":"hle","index":68.02987872210953,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-long-horizon:overall","benchmarkName":"LiveBench Long Agentic","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":42.8,"unit":"percent","family":"livebench-2-agentic","index":49.89044244496576,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Long%20Agentic","measuredAt":"2026-10-14","stderr":null,"costPerTest":17.990494444444444,"latency":null},{"benchmarkId":"sweMarathon","benchmarkName":"SWE-Marathon","category":"coding","score":42,"unit":"percent","family":"sweMarathon","index":null,"aboveFit":false,"run":null,"source":"Abundant AI and BenchFlow","sourceUrl":"https://www.swe-marathon.org/","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"apexAgentsAa","benchmarkName":"APEX-Agents-AA","category":"agentic","score":41.3,"unit":"percent","family":"apexAgentsAa","index":70.45759939790219,"aboveFit":false,"run":null,"source":"Artificial Analysis / Mercor","sourceUrl":"https://artificialanalysis.ai/evaluations/apex-agents-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaCyberIndex","benchmarkName":"Artificial Analysis Cyber Index","category":"agentic","score":41.3,"unit":"percent","family":"aaCyberIndex","index":69.635280724121,"aboveFit":false,"run":null,"source":"Artificial Analysis with the Cyber Index Alliance (Collinear AI, IBM, NVIDIA, Pasta Lab, Vercel)","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-cyber-index","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"zeroBenchPython","benchmarkName":"ZeroBench_main with Python","category":"multimodal","score":41,"unit":"percent","family":"zeroBenchPython","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hermesIndex","benchmarkName":"Hermes Index: agentic capability and cost per task inside Hermes Agent","category":"agentic","score":39.16,"unit":"percent","family":"hermesIndex","index":67.64174303897227,"aboveFit":false,"run":null,"source":"Nous Research","sourceUrl":"https://portal.nousresearch.com/bench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tier-4-v2-private:overall","benchmarkName":"FrontierMath-Tier-4-v2-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":39.02439024390244,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tier-4-v2-private:overall","index":63.81903771842068,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":7.712872341874095,"costPerTest":null,"latency":null},{"benchmarkId":"aaAnalystAgent","benchmarkName":"Artificial Analysis AnalystAgent","category":"agentic","score":38.8,"unit":"percent","family":"aaAnalystAgent","index":59.353993343245605,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/aa-analyst-agent","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"apexAgents","benchmarkName":"APEX-Agents","category":"agentic","score":37.6,"unit":"percent","family":"apexAgents","index":63.04946979021916,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"tau2-bench:banking-knowledge:overall","benchmarkName":"τ²-bench Banking","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":37.11340206185567,"unit":"percent","family":"tau2-bench","index":27.416895845522188,"aboveFit":false,"run":"max effort · Sierra","source":"Sierra Research","sourceUrl":"https://github.com/sierra-research/tau2-bench","measuredAt":"2026-10-20","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"postTrainBench","benchmarkName":"PostTrain Bench","category":"coding","score":36.6,"unit":"percent","family":"postTrainBench","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-chess-puzzles:overall","benchmarkName":"Chess Puzzles","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":36,"unit":"percent","family":"epoch-ai:epoch-chess-puzzles:overall","index":61.309222750542396,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":4.824181513244217,"costPerTest":null,"latency":null},{"benchmarkId":"spreadsheetBench2","benchmarkName":"SpreadsheetBench 2","category":"agentic","score":34.8,"unit":"percent","family":"spreadsheetBench2","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-furniture-assembly:overall","benchmarkName":"Furniture Assembly","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":34.166666666666664,"unit":"percent","family":"epoch-ai:epoch-furniture-assembly:overall","index":62.59122331111903,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":6.116993402501947,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:tax_agent_bench:overall","benchmarkName":"Tax Agent Bench","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":33.368,"unit":"percent","family":"vals-ai:tax_agent_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench","measuredAt":"2026-11-09","stderr":3.386,"costPerTest":2.790127,"latency":2415.159},{"benchmarkId":"postTrainBench11","benchmarkName":"PostTrainBench v1.1","category":"coding","score":31.96,"unit":"percent","family":"postTrainBench11","index":67.56034339149709,"aboveFit":false,"run":null,"source":"PostTrainBench team","sourceUrl":"https://posttrainbench.com/?version=v1.1","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"automationBench","benchmarkName":"AutomationBench","category":"agentic","score":30.8,"unit":"percent","family":"automationBench","index":40.67501207994306,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-mystery-game-puzzles:overall","benchmarkName":"Mystery Game Puzzles","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":26,"unit":"percent","family":"epoch-ai:epoch-mystery-game-puzzles:overall","index":58.81578551640466,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":4.4084400227680804,"costPerTest":null,"latency":null},{"benchmarkId":"frontierSweV2","benchmarkName":"FrontierSWE v2","category":"coding","score":25.9,"unit":"percent","family":"frontierSweV2","index":65.70302539427004,"aboveFit":false,"run":null,"source":"Proximal","sourceUrl":"https://www.frontierswe.com/blog/v2","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":23.4,"unit":"percent","family":"critpt","index":82.14878562352686,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"zeroBench","benchmarkName":"ZeroBench","category":"multimodal","score":23,"unit":"percent","family":"zeroBench","index":63.856495001506914,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"bug-hunt-bench","benchmarkName":"Bug Hunt Bench","category":"coding","score":21,"unit":"percent","family":"bug-hunt-bench","index":64.24568549433579,"aboveFit":false,"run":null,"source":"Pawel Huryn","sourceUrl":"https://bughunt.productcompass.pm/method","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:vcb-1-100:overall","benchmarkName":"Vibe Code Bench 1-100","category":"coding","domain":null,"eligible":true,"ranked":true,"score":18.244,"unit":"percent","family":"vals-ai:vcb-1-100:overall","index":66.65986359376046,"aboveFit":false,"run":"OpenHands","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vcb-1-100","measuredAt":"2026-10-29","stderr":3.92,"costPerTest":9.39737,"latency":6487.278},{"benchmarkId":"apprenticeBench","benchmarkName":"ApprenticeBench: end-to-end computer use, continual learning, and long-horizon agency on a real accounts-payable job","category":"agentic","score":18,"unit":"percent","family":"apprenticeBench","index":67.33365646205729,"aboveFit":false,"run":null,"source":"NeoCognition","sourceUrl":"https://neocognition.io/blog/apprentice-bench/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"frontierCode2","benchmarkName":"FrontierCode 2.0","category":"coding","score":17.7,"unit":"percent","family":"frontierCode2","index":65.14966432122469,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:terminal-bench-4:overall","benchmarkName":"Terminal-Bench 4.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":17.172,"unit":"percent","family":"terminal-bench-4","index":62.66994244458032,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-4","measuredAt":"2026-11-09","stderr":0.505,"costPerTest":12.478176,"latency":11353.52},{"benchmarkId":"vals-ai:code-migration:overall","benchmarkName":"Code Migration","category":"coding","domain":null,"eligible":true,"ranked":true,"score":16.099,"unit":"percent","family":"vals-ai:code-migration:overall","index":53.74157171313385,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/code-migration","measuredAt":"2026-10-29","stderr":4.101,"costPerTest":13.871104,"latency":15450.785},{"benchmarkId":"livebench:livebench-browser:overall","benchmarkName":"LiveBench Browser Use","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":16,"unit":"percent","family":"livebench-2-agentic","index":30.29076910109751,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Browser","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.38201599999999997,"latency":null},{"benchmarkId":"vals-ai:hlab:overall","benchmarkName":"HLAB-Vals-1.1","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":12.917,"unit":"percent","family":"vals-ai:hlab:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/hlab","measuredAt":"2026-11-09","stderr":2.807,"costPerTest":4.144144,"latency":2556.302},{"benchmarkId":"aaDeepsecBench","benchmarkName":"Artificial Analysis DeepsecBench-AA","category":"agentic","score":12.5,"unit":"percent","family":"aaDeepsecBench","index":59.8995421906567,"aboveFit":false,"run":null,"source":"Artificial Analysis; dataset by Vercel","sourceUrl":"https://artificialanalysis.ai/evaluations/deepsecbench-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:time_horizon_index:overall","benchmarkName":"Time Horizon Index: KSP","category":null,"domain":"Other","eligible":false,"ranked":true,"score":10.5,"unit":"percent","family":"vals-ai:time_horizon_index:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/time_horizon_index","measuredAt":"2026-10-07","stderr":5.38,"costPerTest":216.426849,"latency":0},{"benchmarkId":"lmarena:agent-bash-recovery:overall","benchmarkName":"Agent Arena command recovery","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":4.379465536790916,"unit":"index","family":"agent-arena","index":70.13140855650597,"aboveFit":false,"run":"max effort","source":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset","measuredAt":"2026-10-08","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lmarena:agent-task-outcome:overall","benchmarkName":"Agent Arena task outcome","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":3.424563402912273,"unit":"index","family":"agent-arena","index":69.34632930118237,"aboveFit":false,"run":"max effort","source":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset","measuredAt":"2026-11-11","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:programbench:overall","benchmarkName":"ProgramBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":2,"unit":"percent","family":"vals-ai:programbench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/programbench","measuredAt":"2026-11-09","stderr":0.992,"costPerTest":70.475273,"latency":20494.929},{"benchmarkId":"vals-ai:terminal-bench-science:overall","benchmarkName":"Terminal-Bench-Science","category":null,"domain":"Science","eligible":false,"ranked":true,"score":1.429,"unit":"percent","family":"vals-ai:terminal-bench-science:overall","index":null,"aboveFit":false,"run":"Mini-SWE-agent","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-science","measuredAt":"2026-11-09","stderr":1.429,"costPerTest":21.301409,"latency":17349.404},{"benchmarkId":"mazeBench","benchmarkName":"MazeBench: visual spatial reasoning in a 3D open world","category":"agentic","score":1,"unit":"percent","family":"mazeBench","index":63.79865130242642,"aboveFit":false,"run":null,"source":"Jonathan Pappas and David Pappas","sourceUrl":"https://mazebench.com/blog?post=maze-bench-results","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-erdos:overall","benchmarkName":"FrontierMath-Erdos","category":"math","domain":null,"eligible":true,"ranked":true,"score":0,"unit":"percent","family":"epoch-ai:epoch-frontiermath-erdos:overall","index":null,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0,"costPerTest":null,"latency":null},{"benchmarkId":"lmarena:agent-steerability:overall","benchmarkName":"Agent Arena steerability","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":-0.7892769302309643,"unit":"index","family":"agent-arena","index":65.88189193799487,"aboveFit":false,"run":"max effort","source":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset","measuredAt":"2026-11-11","stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":0.192,"outputPerMillion":16,"batch":{"inputPerMillion":2.2800000000000002,"outputPerMillion":11.399999999999999},"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":44,"timeToFirstToken":3.265,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":{"elo":1474.3298939995852,"eloUpper":1478.6237663948605,"eloLower":1470.0360216043096,"votes":40867,"rank":27,"rankUpper":null,"rankLower":null,"arenaName":"Kimi K3-max","organization":"moonshot","updatedAt":"2026-11-09"},"quality":{"coveragePercent":95,"benchmarkCount":73,"categoryCount":7,"confidence":"high","state":"available"},"hosts":[{"host":"chutes","name":"Chutes","kind":"host","url":"https://llm.chutes.ai/v1/models","inputPerMillion":1.85,"outputPerMillion":9.5,"cacheReadPerMillion":0.185,"contextTokens":1048576,"maxOutputTokens":65535,"deprecated":false},{"host":"fireworks-ai","name":"Fireworks AI","kind":"host","url":"https://fireworks.ai/docs/","inputPerMillion":2.1,"outputPerMillion":10.5,"cacheReadPerMillion":0.21,"contextTokens":1048576,"maxOutputTokens":131072,"deprecated":false},{"host":"siliconflow","name":"SiliconFlow","kind":"host","url":"https://cloud.siliconflow.com/models","inputPerMillion":2.7,"outputPerMillion":13.5,"cacheReadPerMillion":0.27,"contextTokens":1048576,"maxOutputTokens":262000,"deprecated":false},{"host":"deepinfra","name":"Deep Infra","kind":"host","url":"https://deepinfra.com/models","inputPerMillion":2.85,"outputPerMillion":14.25,"cacheReadPerMillion":0.285,"contextTokens":1048576,"maxOutputTokens":131072,"deprecated":false},{"host":"alibaba-cloud","name":"Alibaba Cloud","kind":"host","url":"https://www.alibabacloud.com/help/en/model-studio/models","inputPerMillion":3,"outputPerMillion":15,"cacheReadPerMillion":0.3,"contextTokens":1048576,"maxOutputTokens":1048576,"deprecated":false},{"host":"amazon-bedrock","name":"Amazon Bedrock","kind":"cloud","url":"https://docs.aws.amazon.com/bedrock/latest/userguide/models-supported.html","inputPerMillion":3,"outputPerMillion":15,"cacheReadPerMillion":0.3,"contextTokens":1048576,"maxOutputTokens":128000,"deprecated":false},{"host":"arcee-ai","name":"Arcee AI","kind":"host","url":"https://docs.arcee.ai","inputPerMillion":3,"outputPerMillion":15,"cacheReadPerMillion":0.3,"contextTokens":1048576,"maxOutputTokens":131072,"deprecated":false},{"host":"baseten","name":"Baseten","kind":"host","url":"https://docs.baseten.co/inference/model-apis/overview","inputPerMillion":3,"outputPerMillion":15,"cacheReadPerMillion":0.3,"contextTokens":1048576,"maxOutputTokens":262144,"deprecated":false},{"host":"digitalocean","name":"DigitalOcean","kind":"cloud","url":"https://docs.digitalocean.com/products/gradient-ai-platform/details/models/","inputPerMillion":3,"outputPerMillion":15,"cacheReadPerMillion":0.3,"contextTokens":1048576,"maxOutputTokens":131072,"deprecated":false},{"host":"modal","name":"Modal","kind":"host","url":"https://modal.com/docs/guide/endpoints","inputPerMillion":3,"outputPerMillion":15,"cacheReadPerMillion":0.3,"contextTokens":1048576,"maxOutputTokens":131072,"deprecated":false},{"host":"moonshot-ai","name":"Moonshot AI","kind":"maker","url":"https://platform.moonshot.ai/docs/api/chat","inputPerMillion":3,"outputPerMillion":15,"cacheReadPerMillion":0.3,"contextTokens":1048576,"maxOutputTokens":1048576,"deprecated":false},{"host":"nebius","name":"Nebius","kind":"host","url":"https://docs.tokenfactory.nebius.com/","inputPerMillion":3,"outputPerMillion":15,"cacheReadPerMillion":0.3,"contextTokens":1024000,"maxOutputTokens":8000,"deprecated":false},{"host":"novita-ai","name":"Novita AI","kind":"host","url":"https://novita.ai/docs/guides/introduction","inputPerMillion":3,"outputPerMillion":15,"cacheReadPerMillion":0.3,"contextTokens":1048576,"maxOutputTokens":1048576,"deprecated":false},{"host":"together-ai","name":"Together AI","kind":"host","url":"https://docs.together.ai/docs/serverless-models","inputPerMillion":3,"outputPerMillion":15,"cacheReadPerMillion":0.3,"contextTokens":1048576,"maxOutputTokens":131072,"deprecated":false},{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":0.192,"outputPerMillion":16,"cacheReadPerMillion":0.182,"contextTokens":1048576,"maxOutputTokens":943718,"deprecated":false},{"host":"berget-ai","name":"Berget AI","kind":"host","url":"https://api.berget.ai","inputPerMillion":3.3645,"outputPerMillion":16.8225,"cacheReadPerMillion":1.00935,"contextTokens":327680,"maxOutputTokens":32768,"deprecated":false},{"host":"tinfoil","name":"Tinfoil","kind":"host","url":"https://docs.tinfoil.sh","inputPerMillion":4,"outputPerMillion":20,"cacheReadPerMillion":0.6,"contextTokens":262144,"maxOutputTokens":131072,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:arc-prize-foundation","name":"ARC Prize Foundation","url":"https://arcprize.org","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:livebench","name":"LiveBench","url":"https://livebench.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:sierra-research","name":"Sierra Research","url":"https://github.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:lmarena","name":"LMArena","url":"https://huggingface.co","license":"CC BY 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"hy4-preview","providerId":"tencent","slug":"hy4-preview","name":"Hy4 preview","provider":"Tencent","description":"Hy4 preview is a reasoning model from Tencent in the Hy4 family. 27 benchmarks count toward its index, in 6 categories.","logoUrl":null,"releaseDate":"2026-08-28","contextWindowTokens":1048576,"modalities":["text"],"openWeights":true,"details":{"apiId":"tencent/hy4-preview","maxOutputTokens":64000,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.041999999999999996,"weightsUrl":"https://huggingface.co/tencent/Hy4-preview"},"reasoning":true,"overallScore":66.40083126600396,"overallScoreRange":{"low":62.28891331507188,"high":70.51274921693604},"badges":[],"capabilityScores":{"agentic":63.48264549671332,"coding":65.46331284299893,"reasoning":69.5655440649841,"multimodal":75.65342717321258,"knowledge":67.9067222753655,"multilingual":null,"instruction":null,"math":73.04997316504483},"capabilityCounts":{"knowledge":2,"agentic":16,"coding":6,"math":1,"multimodal":1,"reasoning":1},"benchmarks":[{"benchmarkId":"gpqa","benchmarkName":"Graduate-Level Google-Proof Q&A","category":"knowledge","score":92.3,"unit":"percent","family":"gpqa-diamond","index":61.86413863294568,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gpqaDiamond","benchmarkName":"GPQA Diamond","category":"knowledge","score":92.3,"unit":"percent","family":"gpqa-diamond","index":61.86413863294568,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"terminalBench21","benchmarkName":"Terminal-Bench 2.1","category":"agentic","score":85.4,"unit":"percent","family":"terminal-bench-2-1","index":71.0877693333473,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"wideResearch","benchmarkName":"WideResearch","category":"agentic","score":83.9,"unit":"percent","family":"wideResearch","index":65.95762007225004,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:legal_bench:overall","benchmarkName":"LegalBench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":83.757,"unit":"percent","family":"vals-ai:legal_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","measuredAt":"2026-10-13","stderr":0.412,"costPerTest":0.012913,"latency":90.556},{"benchmarkId":"mcpAtlas","benchmarkName":"MCP Atlas","category":"agentic","score":83.7,"unit":"percent","family":"mcpAtlas","index":67.42549470281715,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:medscribe:overall","benchmarkName":"MedScribe","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":83.6,"unit":"percent","family":"vals-ai:medscribe:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medscribe","measuredAt":"2026-10-29","stderr":2.065,"costPerTest":0.053243,"latency":349.623},{"benchmarkId":"bankerToolBench","benchmarkName":"BankerToolBench","category":"agentic","score":78.6,"unit":"percent","family":"bankerToolBench","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"cyberGym","benchmarkName":"CyberGym","category":"agentic","score":78.4,"unit":"percent","family":"cyberGym","index":65.71716753622584,"aboveFit":false,"run":null,"source":"Zhun Wang, Tianneng Shi, Jingxuan He, Matthew Cai, Jialin Zhang, Dawn Song","sourceUrl":"https://www.cybergym.io/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:vibe-code:overall","benchmarkName":"Vibe Code Bench v1.1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":77.478,"unit":"percent","family":"vals-ai:vibe-code:overall","index":70.20480226163613,"aboveFit":false,"run":"OpenHands","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","measuredAt":"2026-11-09","stderr":4.035,"costPerTest":2.181516,"latency":2493.104},{"benchmarkId":"draco","benchmarkName":"Data Research and Analysis with Complex Operations","category":"agentic","score":77.2,"unit":"percent","family":"draco","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:proof_bench:overall","benchmarkName":"ProofBench v1.1","category":"math","domain":null,"eligible":true,"ranked":true,"score":75,"unit":"percent","family":"vals-ai:proof_bench:overall","index":73.04997316504483,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/proof_bench","measuredAt":"2026-11-09","stderr":4.352,"costPerTest":0.335717,"latency":1373.837},{"benchmarkId":"apex","benchmarkName":"Apex","category":"math","score":74.2,"unit":"percent","family":"apex","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"toolathlonVerified","benchmarkName":"Toolathlon-Verified","category":"agentic","score":74.1,"unit":"percent","family":"toolathlonVerified","index":68.74724142498236,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:biomysterybench:overall","benchmarkName":"BioMysteryBench","category":null,"domain":"Science","eligible":false,"ranked":true,"score":69.259,"unit":"percent","family":"vals-ai:biomysterybench:overall","index":null,"aboveFit":false,"run":"Terminus 2","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/biomysterybench","measuredAt":"2026-11-09","stderr":2.593,"costPerTest":0.361814,"latency":1410.317},{"benchmarkId":"vals-ai:public-benefits-bench:overall","benchmarkName":"Public Benefits Bench v1.1","category":null,"domain":"Public benefits","eligible":false,"ranked":true,"score":68.606,"unit":"percent","family":"vals-ai:public-benefits-bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/public-benefits-bench","measuredAt":"2026-10-12","stderr":1.207,"costPerTest":0.252532,"latency":3755.023},{"benchmarkId":"officeQaPro","benchmarkName":"OfficeQA Pro","category":"multimodal","score":66.2,"unit":"percent","family":"officeQaPro","index":75.65342717321258,"aboveFit":false,"run":null,"source":"OfficeQA Pro authors","sourceUrl":"https://arxiv.org/abs/2603.08655","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"swePro","benchmarkName":"SWE-bench Pro","category":"coding","score":65.7,"unit":"percent","family":"swePro","index":64.79015943796401,"aboveFit":false,"run":null,"source":"Xiang Deng, Jeff Da, Edwin Pan, Yannis Yiming He, Charles Ide, Kanak Garg, Niklas Lauffer, Andrew Park, Nitin Pasari, Chetan Rane, Karmini Sampath, Maya Krishnan, Srivatsa Kundurthy, Sean Hendryx, Zifan Wang, Chen Bo Calvin Zhang, Noah Jacobson, Bing Liu, Brad Kenstler","sourceUrl":"https://arxiv.org/abs/2509.16941","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:cyber:overall","benchmarkName":"CyberBench v1.1","category":null,"domain":"Other","eligible":false,"ranked":true,"score":65.357,"unit":"percent","family":"vals-ai:cyber:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/cyber","measuredAt":"2026-11-05","stderr":5.55,"costPerTest":0.457872,"latency":2202.787},{"benchmarkId":"deepSwe","benchmarkName":"DeepSWE","category":"agentic","score":64.3,"unit":"percent","family":"deepSwe","index":67.82102563367744,"aboveFit":false,"run":null,"source":"Datacurve AI","sourceUrl":"https://deepswe.datacurve.ai/blog","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"jobBench","benchmarkName":"JobBench","category":"agentic","score":61.7,"unit":"percent","family":"jobBench","index":70.6198118251528,"aboveFit":false,"run":null,"source":"Yuetai Li, Yichen Feng, Zhangchen Xu, Zixian Ma, Kaiyuan Zheng, Fengqing Jiang, Xinghua Sun, Rulin Shao, Zichen Chen, Yue Huang, Xinyang Han, Brian Lee, Kayla Xu, Shenglai Zeng, Hang Hua, Xiangliang Zhang, Basel Alomair, Ranjay Krishna, Luke Zettlemoyer, Pang Wei Koh, Bhaskar Ramasubramanian, Luyao Niu, Xiang Yue, Radha Poovendran","sourceUrl":"https://arxiv.org/abs/2605.26329","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:ioi:overall","benchmarkName":"IOI","category":"coding","domain":null,"eligible":true,"ranked":true,"score":59.333,"unit":"percent","family":"vals-ai:ioi:overall","index":68.39157147987919,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/ioi","measuredAt":"2026-10-29","stderr":4.596,"costPerTest":1.354793,"latency":3633.295},{"benchmarkId":"nl2Repo","benchmarkName":"NL2Repo","category":"coding","score":58.9,"unit":"percent","family":"nl2Repo","index":68.30961268303165,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hermesBench","benchmarkName":"Hermes Bench: 150 agent tasks graded on the files and state the agent leaves behind","category":"agentic","score":58.6,"unit":"percent","family":"hermesBench","index":null,"aboveFit":false,"run":null,"source":"Nous Research","sourceUrl":"https://portal.nousresearch.com/bench","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:emb:overall","benchmarkName":"EMB","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":58.076,"unit":"percent","family":"vals-ai:emb:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/emb","measuredAt":"2026-10-20","stderr":3.136,"costPerTest":0.787396,"latency":1839.41},{"benchmarkId":"hleWithTools","benchmarkName":"Humanity's Last Exam with tools","category":"agentic","score":55.4,"unit":"percent","family":"hleWithTools","index":64.64274817178813,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hle","benchmarkName":"Humanity's Last Exam","category":"knowledge","score":55.4,"unit":"percent","family":"hle","index":79.96906238796409,"aboveFit":false,"run":null,"source":"Center for AI Safety, Scale AI, and thousands of expert contributors","sourceUrl":"https://lastexam.ai/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:fabv2:overall","benchmarkName":"Finance Agent (v2)","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":55.063,"unit":"percent","family":"vals-ai:fabv2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/fabv2","measuredAt":"2026-10-29","stderr":0.308,"costPerTest":0.5903,"latency":1424.632},{"benchmarkId":"vals-ai:terminal-bench-2-1:overall","benchmarkName":"Terminal-Bench 2.1","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":55.056,"unit":"percent","family":"terminal-bench-2-1","index":53.34091815377927,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2-1","measuredAt":"2026-10-01","stderr":3.433,"costPerTest":0.140288,"latency":1109.129},{"benchmarkId":"cweBenchV1","benchmarkName":"CWE-bench v1","category":"agentic","score":53,"unit":"percent","family":"cweBenchV1","index":66.87336574754552,"aboveFit":false,"run":null,"source":"Collinear AI","sourceUrl":"https://cwe-bench.com/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:code-migration:overall","benchmarkName":"Code Migration","category":"coding","domain":null,"eligible":true,"ranked":true,"score":47.434,"unit":"percent","family":"vals-ai:code-migration:overall","index":70.38798004926747,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/code-migration","measuredAt":"2026-10-29","stderr":4.272,"costPerTest":3.405133,"latency":7994.848},{"benchmarkId":"cweBenchV2","benchmarkName":"CWE-bench v2","category":"agentic","score":47.3,"unit":"percent","family":"cweBenchV2","index":63.819546499453985,"aboveFit":false,"run":null,"source":"Collinear AI","sourceUrl":"https://cwe-bench.com/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:legal_research:overall","benchmarkName":"Legal Research Bench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":45.192,"unit":"percent","family":"vals-ai:legal_research:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_research","measuredAt":"2026-11-09","stderr":3.459,"costPerTest":0.846372,"latency":3839.511},{"benchmarkId":"hleNoTools","benchmarkName":"Humanity's Last Exam without tools","category":"knowledge","score":43.4,"unit":"percent","family":"hle","index":67.92954944760655,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:medcode:overall","benchmarkName":"MedCode","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":43.247,"unit":"percent","family":"vals-ai:medcode:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medcode","measuredAt":"2026-10-29","stderr":2.134,"costPerTest":0.062053,"latency":399.284},{"benchmarkId":"apexAgents","benchmarkName":"APEX-Agents","category":"agentic","score":37.1,"unit":"percent","family":"apexAgents","index":62.766850955049016,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"postTrainBench","benchmarkName":"PostTrain Bench","category":"coding","score":35.6,"unit":"percent","family":"postTrainBench","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:tax_agent_bench:overall","benchmarkName":"Tax Agent Bench","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":32.617,"unit":"percent","family":"vals-ai:tax_agent_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench","measuredAt":"2026-11-09","stderr":3.381,"costPerTest":0.585277,"latency":2464.028},{"benchmarkId":"automationBench","benchmarkName":"AutomationBench","category":"agentic","score":32.1,"unit":"percent","family":"automationBench","index":43.147721246694246,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"sweMarathon","benchmarkName":"SWE-Marathon","category":"coding","score":31.9,"unit":"percent","family":"sweMarathon","index":null,"aboveFit":false,"run":null,"source":"Abundant AI and BenchFlow","sourceUrl":"https://www.swe-marathon.org/","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hermesIndex","benchmarkName":"Hermes Index: agentic capability and cost per task inside Hermes Agent","category":"agentic","score":26.13,"unit":"percent","family":"hermesIndex","index":58.185438903578856,"aboveFit":false,"run":null,"source":"Nous Research","sourceUrl":"https://portal.nousresearch.com/bench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"agentsLastExam","benchmarkName":"Agents' Last Exam","category":"agentic","score":22.8,"unit":"percent","family":"agentsLastExam","index":59.986282722628154,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"programBench","benchmarkName":"ProgramBench: Can Language Models Rebuild Programs From Scratch?","category":"coding","score":17.5,"unit":"percent","family":"programBench","index":50.69575114621521,"aboveFit":false,"run":null,"source":"John Yang, Kilian Lieret, Jeffrey Ma, Parth Thakkar, Dmitrii Pedchenko, Sten Sootla, Emily McMilin, Pengcheng Yin, Rui Hou, Gabriel Synnaeve, Diyi Yang, Ofir Press","sourceUrl":"https://programbench.com/static/paper.pdf","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":16.9,"unit":"percent","family":"critpt","index":69.5655440649841,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:hlab:overall","benchmarkName":"HLAB-Vals-1.1","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":8.75,"unit":"percent","family":"vals-ai:hlab:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/hlab","measuredAt":"2026-11-09","stderr":2.187,"costPerTest":0.98378,"latency":1989.335},{"benchmarkId":"vals-ai:terminal-bench-4:overall","benchmarkName":"Terminal-Bench 4.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":8.081,"unit":"percent","family":"terminal-bench-4","index":58.353184446351506,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-4","measuredAt":"2026-11-09","stderr":1.336,"costPerTest":3.07266,"latency":7798.686},{"benchmarkId":"lmarena:agent-bash-recovery:overall","benchmarkName":"Agent Arena command recovery","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":6.505790470733808,"unit":"index","family":"agent-arena","index":71.87958099899005,"aboveFit":false,"run":null,"source":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset","measuredAt":"2026-10-08","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lmarena:agent-task-outcome:overall","benchmarkName":"Agent Arena task outcome","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":3.5439506267307297,"unit":"index","family":"agent-arena","index":69.44448431565473,"aboveFit":false,"run":null,"source":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset","measuredAt":"2026-11-11","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"stemBench120","benchmarkName":"STEMBench120: six-domain scientific research progress","category":"reasoning","score":2.586,"unit":"percent","family":"stemBench120","index":null,"aboveFit":false,"run":null,"source":"Ulam","sourceUrl":"https://www.ulam.ai/stembench","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:srebench:overall","benchmarkName":"SRE Bench","category":null,"domain":"Other","eligible":false,"ranked":true,"score":2.29,"unit":"percent","family":"vals-ai:srebench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/srebench","measuredAt":"2026-11-05","stderr":0.926,"costPerTest":4.902269,"latency":5996.674},{"benchmarkId":"vals-ai:terminal-bench-science:overall","benchmarkName":"Terminal-Bench-Science","category":null,"domain":"Science","eligible":false,"ranked":true,"score":1.429,"unit":"percent","family":"vals-ai:terminal-bench-science:overall","index":null,"aboveFit":false,"run":"Terminus 2","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-science","measuredAt":"2026-11-09","stderr":1.429,"costPerTest":2.143682,"latency":13679.187},{"benchmarkId":"lmarena:agent-steerability:overall","benchmarkName":"Agent Arena steerability","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":0.22731433360730913,"unit":"index","family":"agent-arena","index":66.7176893292321,"aboveFit":false,"run":null,"source":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset","measuredAt":"2026-11-11","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:programbench:overall","benchmarkName":"ProgramBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":0,"unit":"percent","family":"vals-ai:programbench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/programbench","measuredAt":"2026-11-09","stderr":0,"costPerTest":12.809118,"latency":7801.049}],"price":{"inputPerMillion":0.834,"outputPerMillion":2.501,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":67,"timeToFirstToken":1.304,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":90,"benchmarkCount":27,"categoryCount":6,"confidence":"medium","state":"available"},"hosts":[{"host":"deepinfra","name":"Deep Infra","kind":"host","url":"https://deepinfra.com/models","inputPerMillion":0.834,"outputPerMillion":2.501,"cacheReadPerMillion":0.042,"contextTokens":1048576,"maxOutputTokens":64000,"deprecated":false},{"host":"novita-ai","name":"Novita AI","kind":"host","url":"https://novita.ai/docs/guides/introduction","inputPerMillion":0.834,"outputPerMillion":2.501,"cacheReadPerMillion":0.042,"contextTokens":1000000,"maxOutputTokens":64000,"deprecated":false},{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":0.834,"outputPerMillion":2.501,"cacheReadPerMillion":0.042,"contextTokens":1048576,"maxOutputTokens":64000,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:lmarena","name":"LMArena","url":"https://huggingface.co","license":"CC BY 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"inkling-2-small","providerId":"thinking-machines-lab","slug":"inkling-2-small","name":"Inkling 2 Small","provider":"Thinking Machines Lab","description":"Inkling 2 Small is a reasoning model from Thinking Machines Lab in the Inkling family. 33 benchmarks count toward its index, in 6 categories.","logoUrl":null,"releaseDate":"2026-11-05","contextWindowTokens":1000000,"modalities":["text"],"openWeights":false,"details":null,"reasoning":true,"overallScore":66.35512760452673,"overallScoreRange":{"low":62.540291907723095,"high":70.16996330133037},"badges":[],"capabilityScores":{"agentic":67.60505982176097,"coding":66.58323600858957,"reasoning":68.41483229904966,"multimodal":63.614019680661656,"knowledge":61.47487220078462,"multilingual":null,"instruction":null,"math":60.44335521441966},"capabilityCounts":{"reasoning":5,"math":1,"knowledge":3,"multimodal":6,"agentic":14,"coding":4},"benchmarks":[{"benchmarkId":"arc-prize:v1-public-eval:overall","benchmarkName":"ARC-AGI-1 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":97.375,"unit":"percent","family":"arc-prize:v1-public-eval:overall","index":null,"aboveFit":false,"run":"xhigh effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.06200609450000002,"latency":null},{"benchmarkId":"arc-prize:v1-semi-private:overall","benchmarkName":"ARC-AGI-1 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":96.5,"unit":"percent","family":"arc-prize:v1-semi-private:overall","index":71.35189279722005,"aboveFit":false,"run":"xhigh effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.08188083600000001,"latency":null},{"benchmarkId":"hmmtFeb2026","benchmarkName":"Harvard-MIT Mathematics Tournament February 2026","category":"math","score":95.8,"unit":"percent","family":"hmmtFeb2026","index":60.44335521441966,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"babyVisionPython","benchmarkName":"BabyVision with Python","category":"multimodal","score":93.7,"unit":"percent","family":"babyVisionPython","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gpqa","benchmarkName":"Graduate-Level Google-Proof Q&A","category":"knowledge","score":93,"unit":"percent","family":"gpqa-diamond","index":62.50615950538954,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gpqaDiamond","benchmarkName":"GPQA Diamond","category":"knowledge","score":93,"unit":"percent","family":"gpqa-diamond","index":62.50615950538954,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mathVision","benchmarkName":"MathVision","category":"multimodal","score":92.8,"unit":"percent","family":"mathVision","index":63.55700085390556,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"deepSearchQa","benchmarkName":"DeepSearchQA","category":"agentic","score":90.9,"unit":"percent","family":"deepSearchQa","index":66.56894531656688,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"browseComp","benchmarkName":"BrowseComp","category":"agentic","score":90.2,"unit":"percent","family":"browseComp","index":71.16878055173629,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:sage:overall","benchmarkName":"SAGE","category":null,"domain":"Education","eligible":false,"ranked":true,"score":88.57,"unit":"percent","family":"vals-ai:sage:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/sage","measuredAt":"2026-11-05","stderr":1.754,"costPerTest":0.017884,"latency":40.985},{"benchmarkId":"charxiv","benchmarkName":"CharXiv Reasoning","category":"multimodal","score":87.7,"unit":"percent","family":"charxiv","index":64.03051295359684,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"charxivNoTools","benchmarkName":"CharXiv Reasoning without tools","category":"multimodal","score":85.6,"unit":"percent","family":"charxivNoTools","index":66.6836982774269,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"terminalBench21","benchmarkName":"Terminal-Bench 2.1","category":"agentic","score":85.1,"unit":"percent","family":"terminal-bench-2-1","index":70.9123127240054,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mcpAtlas","benchmarkName":"MCP Atlas","category":"agentic","score":85,"unit":"percent","family":"mcpAtlas","index":68.30090897771046,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"arc-prize:v2-public-eval:overall","benchmarkName":"ARC-AGI-2 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":84.16666666666667,"unit":"percent","family":"arc-prize:v2-public-eval:overall","index":null,"aboveFit":false,"run":"xhigh effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.26916689666666666,"latency":null},{"benchmarkId":"screenSpotPro","benchmarkName":"ScreenSpot Pro","category":"multimodal","score":83.8,"unit":"percent","family":"screenSpotPro","index":63.8136876968553,"aboveFit":false,"run":null,"source":"Kaixin Li, Ziyang Meng, Hongzhan Lin, Ziyang Luo, Yuchen Tian, Jing Ma, Zhiyong Huang, Tat-Seng Chua","sourceUrl":"https://arxiv.org/abs/2504.07981","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":83.3,"unit":"percent","family":"lcr","index":64.4485572404562,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"sweVerified","benchmarkName":"Software Engineering Benchmark Verified","category":"coding","score":82.2,"unit":"percent","family":"swe-bench-verified","index":61.855160423291125,"aboveFit":false,"run":null,"source":"Carlos E. Jimenez, John Yang, Alexander Wettig, Shunyu Yao, Kexin Pei, Ofir Press, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2310.06770","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"arc-prize:v2-semi-private:overall","benchmarkName":"ARC-AGI-2 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":81.66666666666667,"unit":"percent","family":"arc-prize:v2-semi-private:overall","index":76.68451598196197,"aboveFit":false,"run":"xhigh effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.27194635,"latency":null},{"benchmarkId":"mmmuPro","benchmarkName":"Massive Multi-discipline Multimodal Understanding Pro","category":"multimodal","score":81.6,"unit":"percent","family":"mmmu-pro","index":61.348055861559544,"aboveFit":false,"run":null,"source":"MMMU-Pro authors","sourceUrl":"https://arxiv.org/abs/2409.02813","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaMmmuPro","benchmarkName":"Artificial Analysis MMMU-Pro","category":"multimodal","score":81.6,"unit":"percent","family":"mmmu-pro","index":61.348055861559544,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmmu-pro","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:vibe-code:overall","benchmarkName":"Vibe Code Bench v1.1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":81.546,"unit":"percent","family":"vals-ai:vibe-code:overall","index":71.67260057470958,"aboveFit":false,"run":"OpenHands","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","measuredAt":"2026-11-09","stderr":3.454,"costPerTest":1.901278,"latency":1537.412},{"benchmarkId":"aaTerminalBench21","benchmarkName":"Artificial Analysis Terminal-Bench v2.1","category":"agentic","score":81.3,"unit":"percent","family":"aaTerminalBench21","index":63.852892301353414,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/terminalbench-v2-1","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"chartographyWithTools","benchmarkName":"Chartography with image and code tools","category":"multimodal","score":80.4,"unit":"percent","family":"chartographyWithTools","index":null,"aboveFit":false,"run":null,"source":"Surge AI and Anthropic","sourceUrl":"https://surgehq.ai/blog/chartography","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:cyber:overall","benchmarkName":"CyberBench v1.1","category":null,"domain":"Other","eligible":false,"ranked":true,"score":72.738,"unit":"percent","family":"vals-ai:cyber:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/cyber","measuredAt":"2026-11-05","stderr":5.67,"costPerTest":1.270201,"latency":1455.16},{"benchmarkId":"deepSwe","benchmarkName":"DeepSWE","category":"agentic","score":72.1,"unit":"percent","family":"deepSwe","index":73.4914078888303,"aboveFit":false,"run":null,"source":"Datacurve AI","sourceUrl":"https://deepswe.datacurve.ai/blog","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"longBenchV2","benchmarkName":"LongBench v2","category":"reasoning","score":70.6,"unit":"percent","family":"longBenchV2","index":64.04490550200595,"aboveFit":false,"run":null,"source":"LongBench v2 authors","sourceUrl":"https://arxiv.org/abs/2412.15204","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"jobBench","benchmarkName":"JobBench","category":"agentic","score":70.5,"unit":"percent","family":"jobBench","index":75.48708540481792,"aboveFit":false,"run":null,"source":"Yuetai Li, Yichen Feng, Zhangchen Xu, Zixian Ma, Kaiyuan Zheng, Fengqing Jiang, Xinghua Sun, Rulin Shao, Zichen Chen, Yue Huang, Xinyang Han, Brian Lee, Kayla Xu, Shenglai Zeng, Hang Hua, Xiangliang Zhang, Basel Alomair, Ranjay Krishna, Luke Zettlemoyer, Pang Wei Koh, Bhaskar Ramasubramanian, Luyao Niu, Xiang Yue, Radha Poovendran","sourceUrl":"https://arxiv.org/abs/2605.26329","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:biomysterybench:overall","benchmarkName":"BioMysteryBench","category":null,"domain":"Science","eligible":false,"ranked":true,"score":70,"unit":"percent","family":"vals-ai:biomysterybench:overall","index":null,"aboveFit":false,"run":"Terminus 2","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/biomysterybench","measuredAt":"2026-11-09","stderr":1.283,"costPerTest":0.268628,"latency":468.444},{"benchmarkId":"babyVision","benchmarkName":"BabyVision","category":"multimodal","score":68.6,"unit":"percent","family":"babyVision","index":62.25116244062579,"aboveFit":false,"run":null,"source":"Meta AI","sourceUrl":"https://ai.meta.com/static-resource/muse-spark-1-1-evaluation-report","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"perceptionBench","benchmarkName":"PerceptionBench (Internal)","category":"multimodal","score":64.7,"unit":"percent","family":"perceptionBench","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"apex","benchmarkName":"Apex","category":"math","score":63.5,"unit":"percent","family":"apex","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hle","benchmarkName":"Humanity's Last Exam","category":"knowledge","score":53.2,"unit":"percent","family":"hle","index":77.76181834889854,"aboveFit":false,"run":null,"source":"Center for AI Safety, Scale AI, and thousands of expert contributors","sourceUrl":"https://lastexam.ai/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaSciCode","benchmarkName":"Artificial Analysis SciCode","category":"coding","score":52.3,"unit":"percent","family":"scicode-family","index":64.0228454000018,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/scicode","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaAnalystAgent","benchmarkName":"Artificial Analysis AnalystAgent","category":"agentic","score":51.2,"unit":"percent","family":"aaAnalystAgent","index":73.36719742513782,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/aa-analyst-agent","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaEnterpriseOpsGym","benchmarkName":"Artificial Analysis EnterpriseOps-Gym","category":"agentic","score":46.8,"unit":"percent","family":"aaEnterpriseOpsGym","index":null,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/enterprise-ops-gym-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:legal_research:overall","benchmarkName":"Legal Research Bench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":46.635,"unit":"percent","family":"vals-ai:legal_research:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_research","measuredAt":"2026-11-09","stderr":3.467,"costPerTest":0.288586,"latency":735.191},{"benchmarkId":"aaAutomationBench","benchmarkName":"Artificial Analysis AutomationBench","category":"agentic","score":45.6,"unit":"percent","family":"aaAutomationBench","index":57.230000323491396,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/automationbench-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hleNoTools","benchmarkName":"Humanity's Last Exam without tools","category":"knowledge","score":44.1,"unit":"percent","family":"hle","index":68.63185436912741,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":44.1,"unit":"percent","family":"hle","index":68.63185436912741,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaTau3Banking","benchmarkName":"Artificial Analysis Tau3-Banking","category":"agentic","score":40.4,"unit":"percent","family":"aaTau3Banking","index":64.50077247847587,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/tau3-banking","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:tax_agent_bench:overall","benchmarkName":"Tax Agent Bench","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":38.909,"unit":"percent","family":"vals-ai:tax_agent_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench","measuredAt":"2026-11-09","stderr":3.269,"costPerTest":0.144342,"latency":414.041},{"benchmarkId":"sweMarathon","benchmarkName":"SWE-Marathon","category":"coding","score":38.8,"unit":"percent","family":"sweMarathon","index":null,"aboveFit":false,"run":null,"source":"Abundant AI and BenchFlow","sourceUrl":"https://www.swe-marathon.org/","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"chartography","benchmarkName":"Chartography without tools","category":"multimodal","score":36.7,"unit":"percent","family":"chartography","index":null,"aboveFit":false,"run":null,"source":"Surge AI and Anthropic","sourceUrl":"https://surgehq.ai/blog/chartography","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"omniscienceAccuracy","benchmarkName":"Artificial Analysis Omniscience Accuracy","category":"knowledge","score":33.8,"unit":"percent","family":"omniscienceAccuracy","index":53.28660272783691,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"postTrainBench12","benchmarkName":"PostTrainBench v1.2","category":"coding","score":31.9,"unit":"percent","family":"postTrainBench12","index":null,"aboveFit":false,"run":null,"source":"PostTrainBench authors","sourceUrl":"https://posttrainbench.com/","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:mysterymechanism:overall","benchmarkName":"MysteryMechanism","category":null,"domain":"Science","eligible":false,"ranked":true,"score":25.225,"unit":"percent","family":"vals-ai:mysterymechanism:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mysterymechanism","measuredAt":"2026-11-11","stderr":2.921,"costPerTest":0.160403,"latency":1016.822},{"benchmarkId":"aaGdpPdf","benchmarkName":"Artificial Analysis GDP.pdf","category":"agentic","score":25.2,"unit":"percent","family":"aaGdpPdf","index":72.32070588726522,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdp-pdf","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"frontierCode2","benchmarkName":"FrontierCode 2.0","category":"coding","score":22.6,"unit":"percent","family":"frontierCode2","index":68.78233763635576,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:hlab:overall","benchmarkName":"HLAB-Vals-1.1","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":21.667,"unit":"percent","family":"vals-ai:hlab:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/hlab","measuredAt":"2026-11-09","stderr":3.333,"costPerTest":1.234911,"latency":1290.908},{"benchmarkId":"aaMlcr","benchmarkName":"Medical Long Context Reasoning (MLCR-AA)","category":"reasoning","score":20.6,"unit":"percent","family":"aaMlcr","index":65.54428997360414,"aboveFit":false,"run":null,"source":"Wisedocs and Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mlcr-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaTerminalBench4","benchmarkName":"Artificial Analysis Terminal-Bench v4.0","category":"agentic","score":13.1,"unit":"percent","family":"aaTerminalBench4","index":57.068772911958746,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/terminalbench-v4-0","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:terminal-bench-4:overall","benchmarkName":"Terminal-Bench 4.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":12.626,"unit":"percent","family":"terminal-bench-4","index":60.5113260261502,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-4","measuredAt":"2026-11-09","stderr":3.072,"costPerTest":8.218043,"latency":6062.689},{"benchmarkId":"lmarena:agent-steerability:overall","benchmarkName":"Agent Arena steerability","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":9.283420759343358,"unit":"index","family":"agent-arena","index":74.16322854901532,"aboveFit":false,"run":null,"source":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset","measuredAt":"2026-11-11","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lmarena:agent-task-outcome:overall","benchmarkName":"Agent Arena task outcome","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":3.2663219574515456,"unit":"index","family":"agent-arena","index":69.21623002529215,"aboveFit":false,"run":null,"source":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset","measuredAt":"2026-11-11","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:terminal-bench-science:overall","benchmarkName":"Terminal-Bench-Science","category":null,"domain":"Science","eligible":false,"ranked":true,"score":2.857,"unit":"percent","family":"vals-ai:terminal-bench-science:overall","index":null,"aboveFit":false,"run":"Mini-SWE-agent","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-science","measuredAt":"2026-11-09","stderr":2.006,"costPerTest":5.546533,"latency":8862.875},{"benchmarkId":"vals-ai:srebench:overall","benchmarkName":"SRE Bench","category":null,"domain":"Other","eligible":false,"ranked":true,"score":1.908,"unit":"percent","family":"vals-ai:srebench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/srebench","measuredAt":"2026-11-05","stderr":0.847,"costPerTest":6.667906,"latency":8369.945},{"benchmarkId":"vals-ai:programbench:overall","benchmarkName":"ProgramBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":0.5,"unit":"percent","family":"vals-ai:programbench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/programbench","measuredAt":"2026-11-09","stderr":0.5,"costPerTest":8.458245,"latency":6359.9}],"price":{"inputPerMillion":0.3,"outputPerMillion":1,"batch":null,"freeTier":true,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":{"elo":1403.175477868264,"eloUpper":1414.9178803270775,"eloLower":1391.4330754094508,"votes":2668,"rank":153,"rankUpper":null,"rankLower":null,"arenaName":"Inkling 2 Small","organization":"thinky","updatedAt":"2026-11-09"},"quality":{"coveragePercent":90,"benchmarkCount":33,"categoryCount":6,"confidence":"medium","state":"available"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:arc-prize-foundation","name":"ARC Prize Foundation","url":"https://arcprize.org","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:lmarena","name":"LMArena","url":"https://huggingface.co","license":"CC BY 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"sakana-fugu-ultra","providerId":"sakana","slug":"sakana-fugu-ultra-sakana-fugu-ultra","name":"Sakana Fugu-Ultra","provider":"Sakana","description":"Sakana Fugu-Ultra is a reasoning model from Sakana in the Sakana Fugu family. 8 benchmarks count toward its index, in 5 categories.","logoUrl":null,"releaseDate":"2026-06-22","contextWindowTokens":1000000,"modalities":["text"],"openWeights":false,"details":null,"reasoning":true,"overallScore":66.33022038858476,"overallScoreRange":{"low":58.613262062691526,"high":74.047178714478},"badges":[],"capabilityScores":{"agentic":69.15774663058647,"coding":68.30303507990395,"reasoning":68.42474934724422,"multimodal":62.778164685473335,"knowledge":69.6751863787461,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"knowledge":2,"reasoning":1,"coding":3,"multimodal":1,"agentic":1},"benchmarks":[{"benchmarkId":"gpqa","benchmarkName":"Graduate-Level Google-Proof Q&A","category":"knowledge","score":95.5,"unit":"percent","family":"gpqa-diamond","index":64.799091192689,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gpqaDiamond","benchmarkName":"GPQA Diamond","category":"knowledge","score":95.5,"unit":"percent","family":"gpqa-diamond","index":64.799091192689,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mrcrv2","benchmarkName":"MRCRv2","category":"reasoning","score":93.6,"unit":"percent","family":"mrcrv2","index":68.42474934724422,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"liveCodeBenchV6","benchmarkName":"LiveCodeBench v6","category":"coding","score":93.2,"unit":"percent","family":"liveCodeBenchV6","index":60.939597215187874,"aboveFit":false,"run":null,"source":"LiveCodeBench maintainers","sourceUrl":"https://github.com/LiveCodeBench/LiveCodeBench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"liveCodeBenchPro","benchmarkName":"LiveCodeBench Pro","category":"coding","score":90.8,"unit":"percent","family":"liveCodeBenchPro","index":null,"aboveFit":false,"run":null,"source":"LiveCodeBench Pro authors","sourceUrl":"https://arxiv.org/abs/2506.11928","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"charxiv","benchmarkName":"CharXiv Reasoning","category":"multimodal","score":86.6,"unit":"percent","family":"charxiv","index":62.778164685473335,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"terminalBench21","benchmarkName":"Terminal-Bench 2.1","category":"agentic","score":82.1,"unit":"percent","family":"terminal-bench-2-1","index":69.15774663058647,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"swePro","benchmarkName":"SWE-bench Pro","category":"coding","score":73.7,"unit":"percent","family":"swePro","index":71.60303928746787,"aboveFit":false,"run":null,"source":"Xiang Deng, Jeff Da, Edwin Pan, Yannis Yiming He, Charles Ide, Kanak Garg, Niklas Lauffer, Andrew Park, Nitin Pasari, Chetan Rane, Karmini Sampath, Maya Krishnan, Srivatsa Kundurthy, Sean Hendryx, Zifan Wang, Chen Bo Calvin Zhang, Noah Jacobson, Bing Liu, Brad Kenstler","sourceUrl":"https://arxiv.org/abs/2509.16941","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"sciCode","benchmarkName":"Scientific Code Benchmark","category":"coding","score":58.7,"unit":"percent","family":"scicode-family","index":72.3664687370561,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hleNoTools","benchmarkName":"Humanity's Last Exam without tools","category":"knowledge","score":50,"unit":"percent","family":"hle","index":74.55128156480319,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":80,"benchmarkCount":8,"categoryCount":5,"confidence":"low","state":"available"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"claude-haiku-5-5","providerId":"anthropic","slug":"claude-haiku-5-5","name":"Claude Haiku 5.5","provider":"Anthropic","description":"Claude Haiku 5.5 is a reasoning model from Anthropic. 50 benchmarks count toward its index, in 7 categories.","logoUrl":null,"releaseDate":"2026-10-07","contextWindowTokens":1000000,"modalities":["text","image","file"],"openWeights":false,"details":{"apiId":"anthropic/claude-haiku-5.5","maxOutputTokens":128000,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.01,"weightsUrl":null},"reasoning":true,"overallScore":66.25266900040717,"overallScoreRange":{"low":63.46588567004987,"high":69.03945233076446},"badges":[],"capabilityScores":{"agentic":68.55833342480895,"coding":70.97773989782019,"reasoning":67.8157494180531,"multimodal":51.879809529611926,"knowledge":54.5324345611829,"multilingual":null,"instruction":37.18395515638591,"math":68.33744080777964},"capabilityCounts":{"math":5,"coding":13,"knowledge":5,"reasoning":11,"agentic":14,"multimodal":1,"instruction":1},"benchmarks":[{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":98.88888888888889,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":66.45205762128103,"aboveFit":false,"run":"xhigh effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0.5362818328685681,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-coding-2:overall","benchmarkName":"LiveBench Coding 2.0","category":"coding","domain":null,"eligible":true,"ranked":true,"score":91.4,"unit":"percent","family":"livebench:livebench-coding-2:overall","index":74.37523082556962,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Coding","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.8737862068965517,"latency":null},{"benchmarkId":"vals-ai:vibe-code:overall","benchmarkName":"Vibe Code Bench v1.1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":90.444,"unit":"percent","family":"vals-ai:vibe-code:overall","index":74.88313877277434,"aboveFit":false,"run":"OpenHands","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","measuredAt":"2026-11-09","stderr":1.725,"costPerTest":6.071856,"latency":2075.723},{"benchmarkId":"arc-prize:v1-public-eval:overall","benchmarkName":"ARC-AGI-1 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":90.125,"unit":"percent","family":"arc-prize:v1-public-eval:overall","index":null,"aboveFit":false,"run":"max effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.03800877500000003,"latency":null},{"benchmarkId":"aaHarveyLab","benchmarkName":"Artificial Analysis Harvey LAB-AA v1.0","category":"agentic","score":89.9,"unit":"percent","family":"aaHarveyLab","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:sage:overall","benchmarkName":"SAGE","category":null,"domain":"Education","eligible":false,"ranked":true,"score":89.829,"unit":"percent","family":"vals-ai:sage:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/sage","measuredAt":"2026-11-05","stderr":1.435,"costPerTest":0.016108,"latency":112.327},{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":89.64646464646465,"unit":"percent","family":"gpqa-diamond","index":59.43038851474945,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.9774484382483126,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:medscribe:overall","benchmarkName":"MedScribe","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":89.542,"unit":"percent","family":"vals-ai:medscribe:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medscribe","measuredAt":"2026-10-29","stderr":1.917,"costPerTest":0.043777,"latency":323.396},{"benchmarkId":"vals-ai:proof_bench:overall","benchmarkName":"ProofBench v1.1","category":"math","domain":null,"eligible":true,"ranked":true,"score":89,"unit":"percent","family":"vals-ai:proof_bench:overall","index":77.88339897892683,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/proof_bench","measuredAt":"2026-11-09","stderr":3.145,"costPerTest":0.173874,"latency":418.047},{"benchmarkId":"arc-prize:v1-semi-private:overall","benchmarkName":"ARC-AGI-1 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":89,"unit":"percent","family":"arc-prize:v1-semi-private:overall","index":67.90387202491206,"aboveFit":false,"run":"max effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.05132828800000002,"latency":null},{"benchmarkId":"livebench:livebench-math:overall","benchmarkName":"LiveBench Mathematics","category":"math","domain":null,"eligible":true,"ranked":true,"score":86.435,"unit":"percent","family":"livebench:livebench-math:overall","index":60.24095138582804,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Mathematics","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.01624881266490765,"latency":null},{"benchmarkId":"livebench:livebench-reasoning:overall","benchmarkName":"LiveBench Reasoning","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":83.51725,"unit":"percent","family":"livebench","index":69.81080625009041,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Reasoning","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.02059851485148515,"latency":null},{"benchmarkId":"livebench:livebench-design:overall","benchmarkName":"LiveBench Design","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":83.1,"unit":"percent","family":"livebench-2-agentic","index":79.36308557026018,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Design","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.6631279999999999,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":82.7,"unit":"percent","family":"lcr","index":64.0369089251048,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-research:overall","benchmarkName":"LiveBench Research","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":81.9,"unit":"percent","family":"livebench-2-agentic","index":78.48548825635564,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Research","measuredAt":"2026-10-14","stderr":null,"costPerTest":1.9399227272727273,"latency":null},{"benchmarkId":"livebench:livebench-browser:overall","benchmarkName":"LiveBench Browser Use","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":80,"unit":"percent","family":"livebench-2-agentic","index":77.09595917600676,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Browser","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.46001600000000004,"latency":null},{"benchmarkId":"epoch-ai:epoch-mirrorcode:overall","benchmarkName":"MirrorCode","category":"coding","domain":null,"eligible":true,"ranked":true,"score":78.88888888888889,"unit":"percent","family":"epoch-ai:epoch-mirrorcode:overall","index":84.21137060539795,"aboveFit":true,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":8.045286284110595,"costPerTest":null,"latency":null},{"benchmarkId":"perceptionBenchTools","benchmarkName":"PerceptionBench with tools","category":"multimodal","score":78.6,"unit":"percent","family":"perceptionBenchTools","index":null,"aboveFit":false,"run":null,"source":"Meta (reported run)","sourceUrl":"https://research.meta.ai/blog/muse-ember","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-coding:overall","benchmarkName":"LiveBench Coding","category":"coding","domain":null,"eligible":true,"ranked":true,"score":77.863,"unit":"percent","family":"livebench:livebench-coding:overall","index":64.33143489820056,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Coding","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.017903418803418805,"latency":null},{"benchmarkId":"vals-ai:cyber:overall","benchmarkName":"CyberBench v1.1","category":null,"domain":"Other","eligible":false,"ranked":true,"score":75.476,"unit":"percent","family":"vals-ai:cyber:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/cyber","measuredAt":"2026-11-05","stderr":5.205,"costPerTest":1.248102,"latency":1959.181},{"benchmarkId":"epoch-ai:epoch-frontiermath-tiers-1-3-v2-private:overall","benchmarkName":"FrontierMath-Tiers-1-3-v2-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":75.08771929824562,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tiers-1-3-v2-private:overall","index":70.41724585547223,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.566445410232946,"costPerTest":null,"latency":null},{"benchmarkId":"osWorld21Offline","benchmarkName":"OSWorld 2.1 offline subset (82 tasks)","category":"agentic","score":72.4,"unit":"percent","family":"osWorld21Offline","index":75.43265399452034,"aboveFit":false,"run":null,"source":"Mengqi Yuan, Zilong Zhou, Xinzhuang Xiong, Weiming Wu, Jiayang Sun, Jiamin Song, Kaiqian Cui, Bowen Wang, Haoyuan Wu, Yitong Li, Dunjie Lu, Haikong Lu, Qi Zhen, Xinyuan Wang, Jiaqi Deng, Yuhao Yang, Cheng Chen, Boyuan Zheng, Alex Su, Xiao Yu, Hao Zou, Saaket Agashe, Xing Han Lu, Manpreet Kaur, Zhengyang Qi, Vincent Sunn Chen, Frederic Sala, Dayiheng Liu, Junyang Lin, Zhou Yu, Yu Su, Siva Reddy, Xin Eric Wang, Peng Qi, Tianbao Xie, Tao Yu","sourceUrl":"https://github.com/xlang-ai/OSWorld-V2/releases","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hermesBench","benchmarkName":"Hermes Bench: 150 agent tasks graded on the files and state the agent leaves behind","category":"agentic","score":70.97,"unit":"percent","family":"hermesBench","index":null,"aboveFit":false,"run":null,"source":"Nous Research","sourceUrl":"https://portal.nousresearch.com/bench","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:biomysterybench:overall","benchmarkName":"BioMysteryBench","category":null,"domain":"Science","eligible":false,"ranked":true,"score":70.37,"unit":"percent","family":"vals-ai:biomysterybench:overall","index":null,"aboveFit":false,"run":"Terminus 2","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/biomysterybench","measuredAt":"2026-11-09","stderr":1.335,"costPerTest":0.512756,"latency":752.457},{"benchmarkId":"aaHealthBenchProfessionalRaw","benchmarkName":"Artificial Analysis HealthBench Professional raw score","category":"knowledge","score":64.8,"unit":"percent","family":"aaHealthBenchProfessionalRaw","index":null,"aboveFit":false,"run":null,"source":"OpenAI (benchmark); Artificial Analysis (evaluation)","sourceUrl":"https://artificialanalysis.ai/evaluations/healthbench-professional","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"arc-prize:v2-public-eval:overall","benchmarkName":"ARC-AGI-2 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":64.44444444444444,"unit":"percent","family":"arc-prize:v2-public-eval:overall","index":null,"aboveFit":false,"run":"xhigh effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.09271681,"latency":null},{"benchmarkId":"vals-ai:public-benefits-bench:overall","benchmarkName":"Public Benefits Bench v1.1","category":null,"domain":"Public benefits","eligible":false,"ranked":true,"score":63.194,"unit":"percent","family":"vals-ai:public-benefits-bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/public-benefits-bench","measuredAt":"2026-10-12","stderr":1.254,"costPerTest":0.525402,"latency":2192.484},{"benchmarkId":"livebench:livebench-language:overall","benchmarkName":"LiveBench Language","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":62.62800000000001,"unit":"percent","family":"livebench:livebench-language:overall","index":45.37908609505115,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Language","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.021951333333333333,"latency":null},{"benchmarkId":"arc-prize:v2-semi-private:overall","benchmarkName":"ARC-AGI-2 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":61.25000000000001,"unit":"percent","family":"arc-prize:v2-semi-private:overall","index":67.31051583703211,"aboveFit":false,"run":"xhigh effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.08498917333333329,"latency":null},{"benchmarkId":"aaHealthBench","benchmarkName":"Artificial Analysis HealthBench raw score","category":"knowledge","score":61.2,"unit":"percent","family":"aaHealthBench","index":null,"aboveFit":false,"run":null,"source":"OpenAI (benchmark); Artificial Analysis (evaluation)","sourceUrl":"https://artificialanalysis.ai/evaluations/healthbench","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:emb:overall","benchmarkName":"EMB","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":59.015,"unit":"percent","family":"vals-ai:emb:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/emb","measuredAt":"2026-10-20","stderr":4.17,"costPerTest":1.146892,"latency":1082.902},{"benchmarkId":"aaHealthBenchLengthAdjusted","benchmarkName":"Artificial Analysis HealthBench length-adjusted score","category":"knowledge","score":58.5,"unit":"percent","family":"aaHealthBenchLengthAdjusted","index":null,"aboveFit":false,"run":null,"source":"OpenAI (benchmark); Artificial Analysis (evaluation)","sourceUrl":"https://artificialanalysis.ai/evaluations/healthbench","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaHealthBenchProfessional","benchmarkName":"Artificial Analysis HealthBench Professional length-adjusted score","category":"knowledge","score":58.4,"unit":"percent","family":"aaHealthBenchProfessional","index":null,"aboveFit":false,"run":null,"source":"OpenAI (benchmark); Artificial Analysis (evaluation)","sourceUrl":"https://artificialanalysis.ai/evaluations/healthbench-professional","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-long-horizon:overall","benchmarkName":"LiveBench Long Agentic","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":57.6,"unit":"percent","family":"livebench-2-agentic","index":60.71414264978853,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Long%20Agentic","measuredAt":"2026-10-14","stderr":null,"costPerTest":9.761927586206896,"latency":null},{"benchmarkId":"hleWithTools","benchmarkName":"Humanity's Last Exam with tools","category":"agentic","score":57.4,"unit":"percent","family":"hleWithTools","index":66.34692379451198,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-agentic-coding:overall","benchmarkName":"LiveBench Agentic Coding","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":57.17166666666666,"unit":"percent","family":"livebench:livebench-agentic-coding:overall","index":68.12285646639657,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Agentic%20Coding","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.21128750000000002,"latency":null},{"benchmarkId":"agentsLastExamLinux","benchmarkName":"Agents' Last Exam, Linux subset","category":"agentic","score":57,"unit":"percent","family":"agentsLastExamLinux","index":null,"aboveFit":false,"run":null,"source":"Meta (reported run)","sourceUrl":"https://research.meta.ai/blog/muse-ember","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"parseBenchLlamaIndex","benchmarkName":"ParseBench: document parsing for AI agents (LlamaIndex leaderboard)","category":"multimodal","score":56.02,"unit":"percent","family":"parseBenchLlamaIndex","index":51.879809529611926,"aboveFit":false,"run":null,"source":"Boyang Zhang, Sebastián G. Acosta, Preston Carlson, Sacha Bron, Pierre-Loïc Doulcet, Daniel B. Ospina and Simon Suo (LlamaIndex)","sourceUrl":"https://arxiv.org/abs/2604.08538","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gdpvalAaNormalized","benchmarkName":"GDPval-AA normalized","category":"agentic","score":56,"unit":"percent","family":"gdpvalAaNormalized","index":74.07247357193538,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-chess-puzzles:overall","benchmarkName":"Chess Puzzles","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":55.00000000000001,"unit":"percent","family":"epoch-ai:epoch-chess-puzzles:overall","index":75.73051145839423,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":5,"costPerTest":null,"latency":null},{"benchmarkId":"aaSciCode","benchmarkName":"Artificial Analysis SciCode","category":"coding","score":55,"unit":"percent","family":"scicode-family","index":67.54281149532159,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/scicode","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-instruction-following:overall","benchmarkName":"LiveBench Instruction Following","category":"instruction","domain":null,"eligible":true,"ranked":true,"score":54.65825,"unit":"percent","family":"livebench:livebench-instruction-following:overall","index":37.18395515638591,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=IF","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.042943499999999996,"latency":null},{"benchmarkId":"vals-ai:fabv2:overall","benchmarkName":"Finance Agent (v2)","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":54.138,"unit":"percent","family":"vals-ai:fabv2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/fabv2","measuredAt":"2026-10-29","stderr":2.132,"costPerTest":1.636603,"latency":1849},{"benchmarkId":"aaAnalystAgent","benchmarkName":"Artificial Analysis AnalystAgent","category":"agentic","score":53.8,"unit":"percent","family":"aaAnalystAgent","index":76.30544989392165,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/aa-analyst-agent","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-data-analysis:overall","benchmarkName":"LiveBench Data Analysis","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":52.44166666666666,"unit":"percent","family":"livebench","index":29.554439150005063,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Data%20Analysis","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.03038933333333333,"latency":null},{"benchmarkId":"vals-ai:medcode:overall","benchmarkName":"MedCode","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":51.542,"unit":"percent","family":"vals-ai:medcode:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medcode","measuredAt":"2026-10-29","stderr":2.166,"costPerTest":0.031937,"latency":226.464},{"benchmarkId":"cursorBench40","benchmarkName":"cursorBench40","category":"coding","score":48.4,"unit":"percent","family":"cursorBench40","index":73.73162932932276,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-furniture-assembly:overall","benchmarkName":"Furniture Assembly","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":47.5,"unit":"percent","family":"epoch-ai:epoch-furniture-assembly:overall","index":68.80690564166619,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":6.446761548504554,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:ioi:overall","benchmarkName":"IOI","category":"coding","domain":null,"eligible":true,"ranked":true,"score":47.278,"unit":"percent","family":"vals-ai:ioi:overall","index":63.86585443824798,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/ioi","measuredAt":"2026-10-29","stderr":10.721,"costPerTest":0.826826,"latency":753.816},{"benchmarkId":"frontierCode","benchmarkName":"FrontierCode 1.1 Main","category":"coding","score":46.4,"unit":"percent","family":"frontierCode","index":70.76993244113183,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"chartography","benchmarkName":"Chartography without tools","category":"multimodal","score":46.4,"unit":"percent","family":"chartography","index":null,"aboveFit":false,"run":null,"source":"Surge AI and Anthropic","sourceUrl":"https://surgehq.ai/blog/chartography","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tier-4-v2-private:overall","benchmarkName":"FrontierMath-Tier-4-v2-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":46.34146341463415,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tier-4-v2-private:overall","index":66.69355019739008,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":7.884502378168105,"costPerTest":null,"latency":null},{"benchmarkId":"hleNoTools","benchmarkName":"Humanity's Last Exam without tools","category":"knowledge","score":45.9,"unit":"percent","family":"hle","index":70.43778131018104,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:code-migration:overall","benchmarkName":"Code Migration","category":"coding","domain":null,"eligible":true,"ranked":true,"score":44.568,"unit":"percent","family":"vals-ai:code-migration:overall","index":68.86544594071923,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/code-migration","measuredAt":"2026-10-29","stderr":4.231,"costPerTest":6.23191,"latency":4971.306},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":44.4,"unit":"percent","family":"hle","index":68.93284219263634,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-ebr-bench:overall","benchmarkName":"EBR-bench","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":43.8095238095238,"unit":"percent","family":"epoch-ai:epoch-ebr-bench:overall","index":72.5321688085468,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.7817416127494958,"costPerTest":null,"latency":null},{"benchmarkId":"frontierSweV2","benchmarkName":"FrontierSWE v2","category":"coding","score":43.8,"unit":"percent","family":"frontierSweV2","index":74.33543712247453,"aboveFit":false,"run":null,"source":"Proximal","sourceUrl":"https://www.frontierswe.com/blog/v2","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:legal_research:overall","benchmarkName":"Legal Research Bench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":43.75,"unit":"percent","family":"vals-ai:legal_research:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_research","measuredAt":"2026-11-09","stderr":3.448,"costPerTest":1.647359,"latency":4979.806},{"benchmarkId":"mka","benchmarkName":"Multimodal Knowledge Agent Benchmark","category":"agentic","score":41,"unit":"percent","family":"mka","index":70.17785244220443,"aboveFit":false,"run":null,"source":"Google DeepMind, in partnership with Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mka","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hermesIndex","benchmarkName":"Hermes Index: agentic capability and cost per task inside Hermes Agent","category":"agentic","score":37,"unit":"percent","family":"hermesIndex","index":66.07415923755632,"aboveFit":false,"run":null,"source":"Nous Research","sourceUrl":"https://portal.nousresearch.com/bench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:terminal-bench-4:overall","benchmarkName":"Terminal-Bench 4.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":36.364,"unit":"percent","family":"terminal-bench-4","index":71.783045458771,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-4","measuredAt":"2026-11-09","stderr":2.314,"costPerTest":3.117464,"latency":3116.385},{"benchmarkId":"aaAutomationBench","benchmarkName":"Artificial Analysis AutomationBench","category":"agentic","score":35.4,"unit":"percent","family":"aaAutomationBench","index":47.92090968711676,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/automationbench-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:mysterymechanism:overall","benchmarkName":"MysteryMechanism","category":null,"domain":"Science","eligible":false,"ranked":true,"score":35.135,"unit":"percent","family":"vals-ai:mysterymechanism:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mysterymechanism","measuredAt":"2026-11-11","stderr":3.211,"costPerTest":0.541024,"latency":805.031},{"benchmarkId":"aaMlcr","benchmarkName":"Medical Long Context Reasoning (MLCR-AA)","category":"reasoning","score":35,"unit":"percent","family":"aaMlcr","index":79.12162422079982,"aboveFit":false,"run":null,"source":"Wisedocs and Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mlcr-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaFrontierCode2","benchmarkName":"FrontierCode 2.0, Artificial Analysis evaluation","category":"coding","score":34.9,"unit":"percent","family":"aaFrontierCode2","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"omniscienceAccuracy","benchmarkName":"Artificial Analysis Omniscience Accuracy","category":"knowledge","score":34.7,"unit":"percent","family":"omniscienceAccuracy","index":54.420833113490495,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaTerminalBench4","benchmarkName":"Artificial Analysis Terminal-Bench v4.0","category":"agentic","score":32.8,"unit":"percent","family":"aaTerminalBench4","index":66.58551711473808,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/terminalbench-v4-0","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:tax_agent_bench:overall","benchmarkName":"Tax Agent Bench","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":31.922,"unit":"percent","family":"vals-ai:tax_agent_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench","measuredAt":"2026-11-09","stderr":3.352,"costPerTest":1.164207,"latency":2973.647},{"benchmarkId":"epoch-ai:epoch-mystery-game-puzzles:overall","benchmarkName":"Mystery Game Puzzles","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":30,"unit":"percent","family":"epoch-ai:epoch-mystery-game-puzzles:overall","index":61.91344510322534,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":4.605661864718382,"costPerTest":null,"latency":null},{"benchmarkId":"frontierCode2","benchmarkName":"FrontierCode 2.0","category":"coding","score":27.7,"unit":"percent","family":"frontierCode2","index":72.56328333169627,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-simpleqa-verified:overall","benchmarkName":"SimpleQA Verified","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":23.799999999999997,"unit":"percent","family":"epoch-ai:epoch-simpleqa-verified:overall","index":43.74655333121472,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.3473586661967094,"costPerTest":null,"latency":null},{"benchmarkId":"bug-hunt-bench","benchmarkName":"Bug Hunt Bench","category":"coding","score":21.5,"unit":"percent","family":"bug-hunt-bench","index":64.56799076990987,"aboveFit":false,"run":null,"source":"Pawel Huryn","sourceUrl":"https://bughunt.productcompass.pm/method","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaGdpPdf","benchmarkName":"Artificial Analysis GDP.pdf","category":"agentic","score":20.8,"unit":"percent","family":"aaGdpPdf","index":67.24453871742048,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdp-pdf","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:vcb-1-100:overall","benchmarkName":"Vibe Code Bench 1-100","category":"coding","domain":null,"eligible":true,"ranked":true,"score":20.406,"unit":"percent","family":"vals-ai:vcb-1-100:overall","index":68.66705870089586,"aboveFit":false,"run":"OpenHands","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vcb-1-100","measuredAt":"2026-10-29","stderr":3.792,"costPerTest":8.304533,"latency":14744.334},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":18.9,"unit":"percent","family":"critpt","index":73.43731069838188,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaAutomationBenchStrict","benchmarkName":"Artificial Analysis AutomationBench, strict pass@1","category":"agentic","score":13.9,"unit":"percent","family":"aaAutomationBenchStrict","index":null,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/automationbench-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:terminal-bench-science:overall","benchmarkName":"Terminal-Bench-Science","category":null,"domain":"Science","eligible":false,"ranked":true,"score":10,"unit":"percent","family":"vals-ai:terminal-bench-science:overall","index":null,"aboveFit":false,"run":"Mini-SWE-agent","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-science","measuredAt":"2026-11-09","stderr":3.612,"costPerTest":4.523698,"latency":5637.472},{"benchmarkId":"vals-ai:srebench:overall","benchmarkName":"SRE Bench","category":null,"domain":"Other","eligible":false,"ranked":true,"score":5.725,"unit":"percent","family":"vals-ai:srebench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/srebench","measuredAt":"2026-11-05","stderr":1.438,"costPerTest":3.140034,"latency":3475.503},{"benchmarkId":"vals-ai:hlab:overall","benchmarkName":"HLAB-Vals-1.1","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":4.167,"unit":"percent","family":"vals-ai:hlab:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/hlab","measuredAt":"2026-11-09","stderr":1.518,"costPerTest":4.264376,"latency":2620.795},{"benchmarkId":"vals-ai:programbench:overall","benchmarkName":"ProgramBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":2.5,"unit":"percent","family":"vals-ai:programbench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/programbench","measuredAt":"2026-11-09","stderr":1.107,"costPerTest":6.828942,"latency":4389.65},{"benchmarkId":"lmarena:agent-steerability:overall","benchmarkName":"Agent Arena steerability","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":2.3623282936854286,"unit":"index","family":"agent-arena","index":68.4730055133895,"aboveFit":false,"run":"high effort","source":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset","measuredAt":"2026-11-11","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lmarena:agent-task-outcome:overall","benchmarkName":"Agent Arena task outcome","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":1.6031228120819807,"unit":"index","family":"agent-arena","index":67.84881958586011,"aboveFit":false,"run":"high effort","source":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset","measuredAt":"2026-11-11","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mazeBench","benchmarkName":"MazeBench: visual spatial reasoning in a 3D open world","category":"agentic","score":1,"unit":"percent","family":"mazeBench","index":63.79865130242642,"aboveFit":false,"run":null,"source":"Jonathan Pappas and David Pappas","sourceUrl":"https://mazebench.com/blog?post=maze-bench-results","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"arc-prize:v3-semi-private:overall","benchmarkName":"ARC-AGI-3 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":0.4513711877557703,"unit":"percent","family":"arc-prize:v3-semi-private:overall","index":65.4973581804731,"aboveFit":false,"run":"max effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":0.09999999999999999,"outputPerMillion":0.5,"batch":{"inputPerMillion":0.049999999999999996,"outputPerMillion":0.25},"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":159,"timeToFirstToken":1.5865,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":{"elo":1398.6963697768351,"eloUpper":1404.959770578902,"eloLower":1392.4329689747683,"votes":13239,"rank":159,"rankUpper":null,"rankLower":null,"arenaName":"Claude Haiku 5.5-high","organization":"anthropic","updatedAt":"2026-11-09"},"quality":{"coveragePercent":95,"benchmarkCount":50,"categoryCount":7,"confidence":"high","state":"available"},"hosts":[{"host":"amazon-bedrock","name":"Amazon Bedrock","kind":"cloud","url":"https://docs.aws.amazon.com/bedrock/latest/userguide/models-supported.html","inputPerMillion":0.1,"outputPerMillion":0.5,"cacheReadPerMillion":0.01,"contextTokens":1000000,"maxOutputTokens":128000,"deprecated":false},{"host":"anthropic","name":"Anthropic","kind":"maker","url":"https://docs.anthropic.com/en/docs/about-claude/models","inputPerMillion":0.1,"outputPerMillion":0.5,"cacheReadPerMillion":0.01,"contextTokens":1000000,"maxOutputTokens":128000,"deprecated":false},{"host":"digitalocean","name":"DigitalOcean","kind":"cloud","url":"https://docs.digitalocean.com/products/gradient-ai-platform/details/models/","inputPerMillion":0.1,"outputPerMillion":0.5,"cacheReadPerMillion":0.01,"contextTokens":1000000,"maxOutputTokens":128000,"deprecated":false},{"host":"google-vertex","name":"Google Vertex AI","kind":"cloud","url":"https://cloud.google.com/vertex-ai/generative-ai/docs/models","inputPerMillion":0.1,"outputPerMillion":0.5,"cacheReadPerMillion":0.01,"contextTokens":1000000,"maxOutputTokens":128000,"deprecated":false},{"host":"azure","name":"Microsoft Azure","kind":"cloud","url":"https://learn.microsoft.com/en-us/azure/ai-services/openai/concepts/models","inputPerMillion":0.1,"outputPerMillion":0.5,"cacheReadPerMillion":0.01,"contextTokens":1000000,"maxOutputTokens":128000,"deprecated":false},{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":0.1,"outputPerMillion":0.5,"cacheReadPerMillion":0.01,"contextTokens":1000000,"maxOutputTokens":128000,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:livebench","name":"LiveBench","url":"https://livebench.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:arc-prize-foundation","name":"ARC Prize Foundation","url":"https://arcprize.org","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:lmarena","name":"LMArena","url":"https://huggingface.co","license":"CC BY 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"glm-5-3","providerId":"z-ai","slug":"glm-5-3","name":"GLM-5.3","provider":"Z.AI","description":"GLM-5.3 is a reasoning model from Z.AI in the GLM-5 family. 63 benchmarks count toward its index, in 6 categories.","logoUrl":null,"releaseDate":"2026-08-14","contextWindowTokens":1048576,"modalities":["text"],"openWeights":true,"details":{"apiId":"z-ai/glm-5.3","maxOutputTokens":943718,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.0689,"weightsUrl":"https://huggingface.co/zai-org/GLM-5.3"},"reasoning":true,"overallScore":65.83764831309777,"overallScoreRange":{"low":62.30041980357804,"high":69.37487682261751},"badges":[],"capabilityScores":{"agentic":68.21373603284124,"coding":65.73700085856956,"reasoning":64.52347409785015,"multimodal":null,"knowledge":59.257974418642064,"multilingual":null,"instruction":66.81904256120521,"math":63.07196802870644},"capabilityCounts":{"coding":17,"knowledge":6,"math":5,"agentic":28,"reasoning":6,"instruction":1},"benchmarks":[{"benchmarkId":"vals-ai:swebench:overall","benchmarkName":"SWE-bench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":95.4,"unit":"percent","family":"swe-bench-verified","index":71.59092045003587,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/swebench","measuredAt":"2026-09-01","stderr":0.938,"costPerTest":0.338311,"latency":840.831},{"benchmarkId":"b3AgentSecurity","benchmarkName":"b3: Backbone Breaker Benchmark","category":"agentic","score":93,"unit":"percent","family":"b3AgentSecurity","index":null,"aboveFit":false,"run":null,"source":"Lakera with the UK AI Security Institute","sourceUrl":"https://www.lakera.ai/blog/the-backbone-breaker-benchmark","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":91.7,"unit":"percent","family":"gpqa-diamond","index":61.31383502799382,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":91.11111111111111,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":61.9269928211348,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":3.454754040013868,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":90.9090909090909,"unit":"percent","family":"gpqa-diamond","index":60.58843482146635,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.5851578202142853,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:medscribe:overall","benchmarkName":"MedScribe","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":88.81,"unit":"percent","family":"vals-ai:medscribe:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medscribe","measuredAt":"2026-10-29","stderr":1.999,"costPerTest":0.057236,"latency":121.931},{"benchmarkId":"terminalBench21","benchmarkName":"Terminal-Bench 2.1","category":"agentic","score":88.2,"unit":"percent","family":"terminal-bench-2-1","index":72.72536435387164,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:gpqa:overall","benchmarkName":"GPQA Diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":88.132,"unit":"percent","family":"gpqa-diamond","index":58.04136292388003,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","measuredAt":"2026-09-01","stderr":1.851,"costPerTest":0.055878,"latency":210.7},{"benchmarkId":"livebench:livebench-math:overall","benchmarkName":"LiveBench Mathematics","category":"math","domain":null,"eligible":true,"ranked":true,"score":87.898,"unit":"percent","family":"livebench:livebench-math:overall","index":62.27996271220916,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Mathematics","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.26574617414248025,"latency":null},{"benchmarkId":"cyberHarmfulRefusal","benchmarkName":"Refusal of harmful cyber requests (JailbreakBench, StrongREJECT, AgentHarm)","category":"agentic","score":87,"unit":"percent","family":"cyberHarmfulRefusal","index":null,"aboveFit":false,"run":null,"source":"Mistral AI (composite of JailbreakBench, StrongREJECT and AgentHarm)","sourceUrl":"https://mistral.ai/news/mistral-large-4-oss/","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:mmlu_pro:overall","benchmarkName":"MMLU Pro","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":86.77,"unit":"percent","family":"mmlu-pro","index":53.80054232492807,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","measuredAt":"2026-09-01","stderr":0.335,"costPerTest":0.016639,"latency":61.783},{"benchmarkId":"livebench:livebench-reasoning:overall","benchmarkName":"LiveBench Reasoning","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":85.803,"unit":"percent","family":"livebench","index":72.77184450678239,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Reasoning","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.2647658415841584,"latency":null},{"benchmarkId":"vals-ai:legal_bench:overall","benchmarkName":"LegalBench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":84.837,"unit":"percent","family":"vals-ai:legal_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","measuredAt":"2026-10-13","stderr":0.4,"costPerTest":0.006513,"latency":22.484},{"benchmarkId":"cyberGym","benchmarkName":"CyberGym","category":"agentic","score":84.5,"unit":"percent","family":"cyberGym","index":69.48829616452488,"aboveFit":false,"run":null,"source":"Zhun Wang, Tianneng Shi, Jingxuan He, Matthew Cai, Jialin Zhang, Dawn Song","sourceUrl":"https://www.cybergym.io/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaTerminalBench21","benchmarkName":"Artificial Analysis Terminal-Bench v2.1","category":"agentic","score":83.9,"unit":"percent","family":"aaTerminalBench21","index":66.49036565746557,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/terminalbench-v2-1","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-coding-2:overall","benchmarkName":"LiveBench Coding 2.0","category":"coding","domain":null,"eligible":true,"ranked":true,"score":82.8,"unit":"percent","family":"livebench:livebench-coding-2:overall","index":68.14603517184891,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Coding","measuredAt":"2026-10-14","stderr":null,"costPerTest":2.138393103448276,"latency":null},{"benchmarkId":"vals-ai:lcb:overall","benchmarkName":"LiveCodeBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":80.535,"unit":"percent","family":"livecodebench","index":56.16697638123437,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","measuredAt":"2026-09-01","stderr":1.065,"costPerTest":0.069397,"latency":246.75},{"benchmarkId":"livebench:livebench-language:overall","benchmarkName":"LiveBench Language","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":79.85633333333332,"unit":"percent","family":"livebench:livebench-language:overall","index":64.86105146726108,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Language","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.240666,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":79.7,"unit":"percent","family":"lcr","index":61.97866734834783,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-coding:overall","benchmarkName":"LiveBench Coding","category":"coding","domain":null,"eligible":true,"ranked":true,"score":78.95,"unit":"percent","family":"livebench:livebench-coding:overall","index":65.86017821660711,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Coding","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.17046068376068377,"latency":null},{"benchmarkId":"vulcanBench","benchmarkName":"VulcanBench v3","category":"coding","score":78.3,"unit":"percent","family":"vulcanBench","index":56.77167616335277,"aboveFit":false,"run":null,"source":"VulcanBench contributors","sourceUrl":"https://github.com/morganlinton/VulcanBench/tree/main","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:vibe-code:overall","benchmarkName":"Vibe Code Bench v1.1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":78.125,"unit":"percent","family":"vals-ai:vibe-code:overall","index":70.4382500267685,"aboveFit":false,"run":"OpenHands","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","measuredAt":"2026-11-09","stderr":3.94,"costPerTest":12.450185,"latency":3847.197},{"benchmarkId":"frontierSwe","benchmarkName":"FrontierSWE","category":"coding","score":78.1,"unit":"percent","family":"frontierSwe","index":null,"aboveFit":false,"run":null,"source":"Evan Chu, Rajan Agarwal, Abishek Thangamuthu, Brendan Graham, Justus Mattern","sourceUrl":"https://www.frontierswe.com/blog","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-design:overall","benchmarkName":"LiveBench Design","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":77.3,"unit":"percent","family":"livebench-2-agentic","index":75.12136521972153,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Design","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.955616,"latency":null},{"benchmarkId":"livebench:livebench-research:overall","benchmarkName":"LiveBench Research","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":73.2,"unit":"percent","family":"livebench-2-agentic","index":72.12290773054765,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Research","measuredAt":"2026-10-14","stderr":null,"costPerTest":1.634592,"latency":null},{"benchmarkId":"toolathlonVerified","benchmarkName":"Toolathlon-Verified","category":"agentic","score":73,"unit":"percent","family":"toolathlonVerified","index":67.89616164765641,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:tax_eval_v2:overall","benchmarkName":"TaxEval v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":72.363,"unit":"percent","family":"vals-ai:tax_eval_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","measuredAt":"2026-09-01","stderr":0.879,"costPerTest":0.032,"latency":98.323},{"benchmarkId":"vals-ai:cyber:overall","benchmarkName":"CyberBench v1.1","category":null,"domain":"Other","eligible":false,"ranked":true,"score":72.084,"unit":"percent","family":"vals-ai:cyber:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/cyber","measuredAt":"2026-11-05","stderr":5.408,"costPerTest":2.688034,"latency":1818.476},{"benchmarkId":"hermesBench","benchmarkName":"Hermes Bench: 150 agent tasks graded on the files and state the agent leaves behind","category":"agentic","score":71.82,"unit":"percent","family":"hermesBench","index":null,"aboveFit":false,"run":null,"source":"Nous Research","sourceUrl":"https://portal.nousresearch.com/bench","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:terminal-bench-2-1:overall","benchmarkName":"Terminal-Bench 2.1","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":71.536,"unit":"percent","family":"terminal-bench-2-1","index":62.97933456029394,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2-1","measuredAt":"2026-10-01","stderr":1.982,"costPerTest":0.30672,"latency":786.604},{"benchmarkId":"livebench:livebench-data-analysis:overall","benchmarkName":"LiveBench Data Analysis","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":70.24433333333333,"unit":"percent","family":"livebench","index":52.61661877207016,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Data%20Analysis","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.37310133333333334,"latency":null},{"benchmarkId":"livebench:livebench-instruction-following:overall","benchmarkName":"LiveBench Instruction Following","category":"instruction","domain":null,"eligible":true,"ranked":true,"score":69.304,"unit":"percent","family":"livebench:livebench-instruction-following:overall","index":66.81904256120521,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=IF","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.2712655,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tiers-1-3-v2-private:overall","benchmarkName":"FrontierMath-Tiers-1-3-v2-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":68.77192982456141,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tiers-1-3-v2-private:overall","index":67.09291925779611,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.7499133473298816,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:public-benefits-bench:overall","benchmarkName":"Public Benefits Bench v1.1","category":null,"domain":"Public benefits","eligible":false,"ranked":true,"score":68.539,"unit":"percent","family":"vals-ai:public-benefits-bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/public-benefits-bench","measuredAt":"2026-10-12","stderr":1.208,"costPerTest":0.946832,"latency":2150.623},{"benchmarkId":"vals-ai:ioi:overall","benchmarkName":"IOI","category":"coding","domain":null,"eligible":true,"ranked":true,"score":68.444,"unit":"percent","family":"vals-ai:ioi:overall","index":71.81204497355832,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/ioi","measuredAt":"2026-10-29","stderr":7.476,"costPerTest":7.67087,"latency":6374.465},{"benchmarkId":"deepSwe","benchmarkName":"DeepSWE","category":"agentic","score":66.9,"unit":"percent","family":"deepSwe","index":69.71115305206173,"aboveFit":false,"run":null,"source":"Datacurve AI","sourceUrl":"https://deepswe.datacurve.ai/blog","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"finch","benchmarkName":"Finch (FinWorkBench): Finance & Accounting across Spreadsheet-Centric Enterprise Workflows","category":"agentic","score":65.1,"unit":"percent","family":"finch","index":null,"aboveFit":false,"run":null,"source":"Finch authors (ACL 2026 Findings)","sourceUrl":"https://arxiv.org/abs/2512.13168","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hleWithTools","benchmarkName":"Humanity's Last Exam with tools","category":"agentic","score":62.5,"unit":"percent","family":"hleWithTools","index":70.69257163245777,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaAutomationBench","benchmarkName":"Artificial Analysis AutomationBench","category":"agentic","score":62.2,"unit":"percent","family":"aaAutomationBench","index":72.38008900621874,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/automationbench-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-agentic-coding:overall","benchmarkName":"LiveBench Agentic Coding","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":60.909,"unit":"percent","family":"livebench:livebench-agentic-coding:overall","index":71.97710467999575,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Agentic%20Coding","measuredAt":"2026-06-25","stderr":null,"costPerTest":1.5978291666666666,"latency":null},{"benchmarkId":"openHarmonyBench","benchmarkName":"OpenHarmony Bench v1.0","category":"coding","score":60.8,"unit":"percent","family":"openHarmonyBench","index":67.37351894160618,"aboveFit":false,"run":null,"source":"OpenHarmony Bench authors","sourceUrl":"https://arxiv.org/abs/2608.16022","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaSciCode","benchmarkName":"Artificial Analysis SciCode","category":"coding","score":59,"unit":"percent","family":"scicode-family","index":72.75757608098051,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/scicode","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"nl2Repo","benchmarkName":"NL2Repo","category":"coding","score":58,"unit":"percent","family":"nl2Repo","index":67.67508242880785,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gdpvalAaNormalized","benchmarkName":"GDPval-AA normalized","category":"agentic","score":57.5,"unit":"percent","family":"gdpvalAaNormalized","index":75.14493871447313,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:emb:overall","benchmarkName":"EMB","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":56.344,"unit":"percent","family":"vals-ai:emb:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/emb","measuredAt":"2026-10-20","stderr":3.319,"costPerTest":3.787827,"latency":2172.231},{"benchmarkId":"vals-ai:fabv2:overall","benchmarkName":"Finance Agent (v2)","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":55.84,"unit":"percent","family":"vals-ai:fabv2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/fabv2","measuredAt":"2026-10-29","stderr":2.066,"costPerTest":1.072768,"latency":949.539},{"benchmarkId":"aaCweBench","benchmarkName":"Artificial Analysis CWE-Bench-AA","category":"agentic","score":55.8,"unit":"percent","family":"aaCweBench","index":68.317633283132,"aboveFit":false,"run":null,"source":"Artificial Analysis; dataset by Collinear AI","sourceUrl":"https://artificialanalysis.ai/evaluations/cwe-bench-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaTau3Banking","benchmarkName":"Artificial Analysis Tau3-Banking","category":"agentic","score":50.3,"unit":"percent","family":"aaTau3Banking","index":76.75071833547202,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/tau3-banking","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:legal_research:overall","benchmarkName":"Legal Research Bench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":49.038,"unit":"percent","family":"vals-ai:legal_research:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_research","measuredAt":"2026-11-09","stderr":3.475,"costPerTest":2.241388,"latency":3110.171},{"benchmarkId":"vals-ai:proof_bench:overall","benchmarkName":"ProofBench v1.1","category":"math","domain":null,"eligible":true,"ranked":true,"score":49,"unit":"percent","family":"vals-ai:proof_bench:overall","index":64.07361093926399,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/proof_bench","measuredAt":"2026-11-09","stderr":5.024,"costPerTest":2.081538,"latency":2542.672},{"benchmarkId":"automationBench","benchmarkName":"AutomationBench","category":"agentic","score":48.2,"unit":"percent","family":"automationBench","index":73.77127323492056,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:skillsbench:overall","benchmarkName":"SkillsBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":47.509,"unit":"percent","family":"vals-ai:skillsbench:overall","index":60.50472366081305,"aboveFit":false,"run":"OpenHands","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/skillsbench","measuredAt":"2026-10-29","stderr":4.522,"costPerTest":0.71406,"latency":787.365},{"benchmarkId":"aaItbench","benchmarkName":"Artificial Analysis ITBench-AA","category":"agentic","score":46.1,"unit":"percent","family":"aaItbench","index":66.479939407938,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/itbench-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:code-migration:overall","benchmarkName":"Code Migration","category":"coding","domain":null,"eligible":true,"ranked":true,"score":44.222,"unit":"percent","family":"vals-ai:code-migration:overall","index":68.68163686829854,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/code-migration","measuredAt":"2026-10-29","stderr":4.289,"costPerTest":24.909755,"latency":13633.637},{"benchmarkId":"vals-ai:tax_agent_bench:overall","benchmarkName":"Tax Agent Bench","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":43.325,"unit":"percent","family":"vals-ai:tax_agent_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench","measuredAt":"2026-11-09","stderr":3.565,"costPerTest":1.80097,"latency":3052.932},{"benchmarkId":"vals-ai:medcode:overall","benchmarkName":"MedCode","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":42.864,"unit":"percent","family":"vals-ai:medcode:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medcode","measuredAt":"2026-10-29","stderr":2.111,"costPerTest":0.081905,"latency":169.01},{"benchmarkId":"cursorBench40","benchmarkName":"cursorBench40","category":"coding","score":42.6,"unit":"percent","family":"cursorBench40","index":69.06399965115962,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"sweMarathon","benchmarkName":"SWE-Marathon","category":"coding","score":42.5,"unit":"percent","family":"sweMarathon","index":null,"aboveFit":false,"run":null,"source":"Abundant AI and BenchFlow","sourceUrl":"https://www.swe-marathon.org/","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":42.3,"unit":"percent","family":"hle","index":66.82592742807378,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"cweBenchV2","benchmarkName":"CWE-bench v2","category":"agentic","score":42,"unit":"percent","family":"cweBenchV2","index":59.50323004758016,"aboveFit":false,"run":null,"source":"Collinear AI","sourceUrl":"https://cwe-bench.com/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaTerminalBench4","benchmarkName":"Artificial Analysis Terminal-Bench v4.0","category":"agentic","score":41.9,"unit":"percent","family":"aaTerminalBench4","index":70.98157661957522,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/terminalbench-v4-0","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"terminal-bench:terminal-bench-4-0-0:overall","benchmarkName":"Terminal-Bench 4.0.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":41.82,"unit":"percent","family":"terminal-bench-4","index":74.37376503179227,"aboveFit":false,"run":"max effort · Claude Code","source":"Terminal-Bench","sourceUrl":"https://www.tbench.ai/leaderboard","measuredAt":"2026-09-29","stderr":1.6479591836734695,"costPerTest":8.265545454545455,"latency":5831},{"benchmarkId":"terminal-bench:terminal-bench-4-1-0:overall","benchmarkName":"Terminal-Bench 4.1.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":41.21,"unit":"percent","family":"terminal-bench:terminal-bench-4-1-0:overall","index":72.18724402176082,"aboveFit":false,"run":"max effort · Claude Code","source":"Terminal-Bench","sourceUrl":"https://www.tbench.ai/leaderboard","measuredAt":"2026-09-29","stderr":1.673469387755102,"costPerTest":8.265545454545455,"latency":5831},{"benchmarkId":"epoch-ai:epoch-simpleqa-verified:overall","benchmarkName":"SimpleQA Verified","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":41,"unit":"percent","family":"epoch-ai:epoch-simpleqa-verified:overall","index":58.075671718375496,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.556091713692166,"costPerTest":null,"latency":null},{"benchmarkId":"postTrainBench","benchmarkName":"PostTrain Bench","category":"coding","score":39.8,"unit":"percent","family":"postTrainBench","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hermesIndex","benchmarkName":"Hermes Index: agentic capability and cost per task inside Hermes Agent","category":"agentic","score":39.23,"unit":"percent","family":"hermesIndex","index":67.69254436587,"aboveFit":false,"run":null,"source":"Nous Research","sourceUrl":"https://portal.nousresearch.com/bench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:terminal-bench-4:overall","benchmarkName":"Terminal-Bench 4.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":38.889,"unit":"percent","family":"terminal-bench-4","index":72.98201300310362,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-4","measuredAt":"2026-11-09","stderr":1.821,"costPerTest":9.37436,"latency":4952.947},{"benchmarkId":"aaAnalystAgent","benchmarkName":"Artificial Analysis AnalystAgent","category":"agentic","score":38.8,"unit":"percent","family":"aaAnalystAgent","index":59.353993343245605,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/aa-analyst-agent","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-chess-puzzles:overall","benchmarkName":"Chess Puzzles","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":37,"unit":"percent","family":"epoch-ai:epoch-chess-puzzles:overall","index":62.068237945692495,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":4.8523658709390975,"costPerTest":null,"latency":null},{"benchmarkId":"aaEnterpriseOpsGym","benchmarkName":"Artificial Analysis EnterpriseOps-Gym","category":"agentic","score":36.4,"unit":"percent","family":"aaEnterpriseOpsGym","index":null,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/enterprise-ops-gym-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaCyberIndex","benchmarkName":"Artificial Analysis Cyber Index","category":"agentic","score":36.4,"unit":"percent","family":"aaCyberIndex","index":64.60819637875103,"aboveFit":false,"run":null,"source":"Artificial Analysis with the Cyber Index Alliance (Collinear AI, IBM, NVIDIA, Pasta Lab, Vercel)","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-cyber-index","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-long-horizon:overall","benchmarkName":"LiveBench Long Agentic","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":35.9,"unit":"percent","family":"livebench-2-agentic","index":44.8442578900146,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Long%20Agentic","measuredAt":"2026-10-14","stderr":null,"costPerTest":18.38193888888889,"latency":null},{"benchmarkId":"epoch-ai:epoch-mystery-game-puzzles:overall","benchmarkName":"Mystery Game Puzzles","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":33,"unit":"percent","family":"epoch-ai:epoch-mystery-game-puzzles:overall","index":64.23668979334086,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":4.725815626252608,"costPerTest":null,"latency":null},{"benchmarkId":"omniscienceAccuracy","benchmarkName":"Artificial Analysis Omniscience Accuracy","category":"knowledge","score":32.3,"unit":"percent","family":"omniscienceAccuracy","index":51.396218751747604,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"frontierSweV2","benchmarkName":"FrontierSWE v2","category":"coding","score":30.2,"unit":"percent","family":"frontierSweV2","index":67.77673323959291,"aboveFit":false,"run":null,"source":"Proximal","sourceUrl":"https://www.frontierswe.com/blog/v2","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tier-4-v2-private:overall","benchmarkName":"FrontierMath-Tier-4-v2-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":29.268292682926827,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tier-4-v2-private:overall","index":59.986354413128154,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":7.194088392074452,"costPerTest":null,"latency":null},{"benchmarkId":"aaCyberGymE2e","benchmarkName":"Artificial Analysis CyberGym-E2E-AA","category":"agentic","score":29,"unit":"percent","family":"aaCyberGymE2e","index":66.48679054504251,"aboveFit":false,"run":null,"source":"Artificial Analysis; dataset by Berkeley RDI","sourceUrl":"https://artificialanalysis.ai/evaluations/cybergym-e2e-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"agentsLastExam","benchmarkName":"Agents' Last Exam","category":"agentic","score":28.5,"unit":"percent","family":"agentsLastExam","index":64.71819620871237,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"terminalBench3","benchmarkName":"terminalBench3","category":"agentic","score":28.3,"unit":"percent","family":"terminalBench3","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaDeepsecBench","benchmarkName":"Artificial Analysis DeepsecBench-AA","category":"agentic","score":24.3,"unit":"percent","family":"aaDeepsecBench","index":68.97671070207802,"aboveFit":false,"run":null,"source":"Artificial Analysis; dataset by Vercel","sourceUrl":"https://artificialanalysis.ai/evaluations/deepsecbench-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:mysterymechanism:overall","benchmarkName":"MysteryMechanism","category":null,"domain":"Science","eligible":false,"ranked":true,"score":22.973,"unit":"percent","family":"vals-ai:mysterymechanism:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mysterymechanism","measuredAt":"2026-11-11","stderr":2.83,"costPerTest":1.531337,"latency":1841.98},{"benchmarkId":"vals-ai:vcb-1-100:overall","benchmarkName":"Vibe Code Bench 1-100","category":"coding","domain":null,"eligible":true,"ranked":true,"score":19.994,"unit":"percent","family":"vals-ai:vcb-1-100:overall","index":68.28455898575257,"aboveFit":false,"run":"OpenHands","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vcb-1-100","measuredAt":"2026-10-29","stderr":3.811,"costPerTest":8.236953,"latency":5169.473},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":19.1,"unit":"percent","family":"critpt","index":73.82448736172167,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"programBench","benchmarkName":"ProgramBench: Can Language Models Rebuild Programs From Scratch?","category":"coding","score":19,"unit":"percent","family":"programBench","index":51.668638963225966,"aboveFit":false,"run":null,"source":"John Yang, Kilian Lieret, Jeffrey Ma, Parth Thakkar, Dmitrii Pedchenko, Sten Sootla, Emily McMilin, Pengcheng Yin, Rui Hou, Gabriel Synnaeve, Diyi Yang, Ofir Press","sourceUrl":"https://programbench.com/static/paper.pdf","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"bug-hunt-bench","benchmarkName":"Bug Hunt Bench","category":"coding","score":19,"unit":"percent","family":"bug-hunt-bench","index":62.9564643920395,"aboveFit":false,"run":null,"source":"Pawel Huryn","sourceUrl":"https://bughunt.productcompass.pm/method","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaMlcr","benchmarkName":"Medical Long Context Reasoning (MLCR-AA)","category":"reasoning","score":17.2,"unit":"percent","family":"aaMlcr","index":62.338530498571814,"aboveFit":false,"run":null,"source":"Wisedocs and Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mlcr-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"exploitGym","benchmarkName":"ExploitGym","category":"agentic","score":15,"unit":"percent","family":"exploitGym","index":68.08801287984903,"aboveFit":false,"run":null,"source":"Zhun Wang, Nico Schiller, Hongwei Li, Srijiith Sesha Narayana, Milad Nasr, Nicholas Carlini, Xiangyu Qi, Eric Wallace, Elie Bursztein, Luca Invernizzi, Kurt Thomas, Yan Shoshitaishvili, Wenbo Guo, Jingxuan He, Thorsten Holz, Dawn Song","sourceUrl":"https://arxiv.org/abs/2605.11086","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:hlab:overall","benchmarkName":"HLAB-Vals-1.1","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":14.583,"unit":"percent","family":"vals-ai:hlab:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/hlab","measuredAt":"2026-11-09","stderr":2.862,"costPerTest":3.987774,"latency":1911.01},{"benchmarkId":"aaGdpPdf","benchmarkName":"Artificial Analysis GDP.pdf","category":"agentic","score":11.2,"unit":"percent","family":"aaGdpPdf","index":56.1692648923047,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdp-pdf","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:srebench:overall","benchmarkName":"SRE Bench","category":null,"domain":"Other","eligible":false,"ranked":true,"score":9.16,"unit":"percent","family":"vals-ai:srebench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/srebench","measuredAt":"2026-11-05","stderr":1.786,"costPerTest":27.465302,"latency":10089.867},{"benchmarkId":"vals-ai:terminal-bench-science:overall","benchmarkName":"Terminal-Bench-Science","category":null,"domain":"Science","eligible":false,"ranked":true,"score":5.714,"unit":"percent","family":"vals-ai:terminal-bench-science:overall","index":null,"aboveFit":false,"run":"Mini-SWE-agent","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-science","measuredAt":"2026-11-09","stderr":2.794,"costPerTest":15.627955,"latency":9898.858},{"benchmarkId":"lmarena:agent-task-outcome:overall","benchmarkName":"Agent Arena task outcome","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":3.1242872981792402,"unit":"index","family":"agent-arena","index":69.0994552684082,"aboveFit":false,"run":"max effort","source":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset","measuredAt":"2026-11-11","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:programbench:overall","benchmarkName":"ProgramBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":1.5,"unit":"percent","family":"vals-ai:programbench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/programbench","measuredAt":"2026-11-09","stderr":0.862,"costPerTest":21.957364,"latency":15354.08},{"benchmarkId":"mazeBench","benchmarkName":"MazeBench: visual spatial reasoning in a 3D open world","category":"agentic","score":0,"unit":"percent","family":"mazeBench","index":61.97866734834783,"aboveFit":false,"run":null,"source":"Jonathan Pappas and David Pappas","sourceUrl":"https://mazebench.com/blog?post=maze-bench-results","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lmarena:agent-bash-recovery:overall","benchmarkName":"Agent Arena command recovery","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":-0.053530867991780084,"unit":"index","family":"agent-arena","index":66.48679054504251,"aboveFit":false,"run":"max effort","source":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset","measuredAt":"2026-10-08","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lmarena:agent-steerability:overall","benchmarkName":"Agent Arena steerability","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":-1.1792889639506199,"unit":"index","family":"agent-arena","index":65.56124090357297,"aboveFit":false,"run":"max effort","source":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset","measuredAt":"2026-11-11","stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":0.0699,"outputPerMillion":1.4,"batch":{"inputPerMillion":0.322,"outputPerMillion":0.989},"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":86,"timeToFirstToken":0.32,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":{"elo":1475.9274416653438,"eloUpper":1480.6478706383718,"eloLower":1471.2070126923159,"votes":31017,"rank":25,"rankUpper":null,"rankLower":null,"arenaName":"GLM-5.3-max","organization":"zai","updatedAt":"2026-11-09"},"quality":{"coveragePercent":85,"benchmarkCount":63,"categoryCount":6,"confidence":"medium","state":"available"},"hosts":[{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":0.0699,"outputPerMillion":1.4,"cacheReadPerMillion":0.0689,"contextTokens":1048576,"maxOutputTokens":943718,"deprecated":false},{"host":"coreweave","name":"CoreWeave","kind":"host","url":"https://docs.wandb.ai/inference","inputPerMillion":0.75,"outputPerMillion":3,"cacheReadPerMillion":0.19,"contextTokens":1048576,"maxOutputTokens":1048576,"deprecated":false},{"host":"vultr","name":"Vultr","kind":"cloud","url":"https://api.vultrinference.com/","inputPerMillion":0.75,"outputPerMillion":3,"cacheReadPerMillion":null,"contextTokens":1048576,"maxOutputTokens":131072,"deprecated":false},{"host":"fireworks-ai","name":"Fireworks AI","kind":"host","url":"https://fireworks.ai/docs/","inputPerMillion":0.98,"outputPerMillion":3.08,"cacheReadPerMillion":0.182,"contextTokens":1048573,"maxOutputTokens":262144,"deprecated":false},{"host":"friendli","name":"Friendli","kind":"host","url":"https://friendli.ai/docs/guides/serverless_endpoints/introduction","inputPerMillion":1.26,"outputPerMillion":3.96,"cacheReadPerMillion":0.234,"contextTokens":1048576,"maxOutputTokens":131072,"deprecated":false},{"host":"deepinfra","name":"Deep Infra","kind":"host","url":"https://deepinfra.com/models","inputPerMillion":0.9,"outputPerMillion":4,"cacheReadPerMillion":0.2,"contextTokens":1048576,"maxOutputTokens":131072,"deprecated":false},{"host":"zhipu-ai","name":"Zhipu AI","kind":"maker","url":"https://docs.bigmodel.cn/cn/guide/start/pricing","inputPerMillion":1.19516,"outputPerMillion":4.18306,"cacheReadPerMillion":0.29879,"contextTokens":1000000,"maxOutputTokens":131072,"deprecated":false},{"host":"arcee-ai","name":"Arcee AI","kind":"host","url":"https://docs.arcee.ai","inputPerMillion":1.4,"outputPerMillion":4.4,"cacheReadPerMillion":0.26,"contextTokens":1000000,"maxOutputTokens":131072,"deprecated":false},{"host":"baseten","name":"Baseten","kind":"host","url":"https://docs.baseten.co/inference/model-apis/overview","inputPerMillion":1.4,"outputPerMillion":4.4,"cacheReadPerMillion":0.14,"contextTokens":1048576,"maxOutputTokens":262144,"deprecated":false},{"host":"cloudflare","name":"Cloudflare Workers AI","kind":"cloud","url":"https://developers.cloudflare.com/workers-ai/models/","inputPerMillion":1.4,"outputPerMillion":4.4,"cacheReadPerMillion":0.26,"contextTokens":1048576,"maxOutputTokens":1048576,"deprecated":false},{"host":"digitalocean","name":"DigitalOcean","kind":"cloud","url":"https://docs.digitalocean.com/products/gradient-ai-platform/details/models/","inputPerMillion":1.4,"outputPerMillion":4.4,"cacheReadPerMillion":0.26,"contextTokens":1048576,"maxOutputTokens":128000,"deprecated":false},{"host":"mistral","name":"Mistral","kind":"host","url":"https://docs.mistral.ai/getting-started/models/","inputPerMillion":1.4,"outputPerMillion":4.4,"cacheReadPerMillion":0.14,"contextTokens":1048576,"maxOutputTokens":131072,"deprecated":false},{"host":"modal","name":"Modal","kind":"host","url":"https://modal.com/docs/guide/endpoints","inputPerMillion":1.4,"outputPerMillion":4.4,"cacheReadPerMillion":0.26,"contextTokens":1000000,"maxOutputTokens":131072,"deprecated":false},{"host":"nebius","name":"Nebius","kind":"host","url":"https://docs.tokenfactory.nebius.com/","inputPerMillion":1.4,"outputPerMillion":4.4,"cacheReadPerMillion":0.26,"contextTokens":1024000,"maxOutputTokens":1024000,"deprecated":false},{"host":"novita-ai","name":"Novita AI","kind":"host","url":"https://novita.ai/docs/guides/introduction","inputPerMillion":1.4,"outputPerMillion":4.4,"cacheReadPerMillion":0.26,"contextTokens":1048576,"maxOutputTokens":131072,"deprecated":false},{"host":"siliconflow","name":"SiliconFlow","kind":"host","url":"https://cloud.siliconflow.com/models","inputPerMillion":1.4,"outputPerMillion":4.4,"cacheReadPerMillion":0.26,"contextTokens":1049000,"maxOutputTokens":262000,"deprecated":false},{"host":"together-ai","name":"Together AI","kind":"host","url":"https://docs.together.ai/docs/serverless-models","inputPerMillion":1.4,"outputPerMillion":4.4,"cacheReadPerMillion":0.26,"contextTokens":1048575,"maxOutputTokens":262144,"deprecated":false},{"host":"z-ai","name":"Z.AI","kind":"maker","url":"https://docs.z.ai/guides/overview/pricing","inputPerMillion":1.4,"outputPerMillion":4.4,"cacheReadPerMillion":0.26,"contextTokens":1000000,"maxOutputTokens":131072,"deprecated":false},{"host":"amazon-bedrock","name":"Amazon Bedrock","kind":"cloud","url":"https://docs.aws.amazon.com/bedrock/latest/userguide/models-supported.html","inputPerMillion":1.68,"outputPerMillion":5.28,"cacheReadPerMillion":0.312,"contextTokens":1000000,"maxOutputTokens":128000,"deprecated":false},{"host":"tinfoil","name":"Tinfoil","kind":"host","url":"https://docs.tinfoil.sh","inputPerMillion":1.8,"outputPerMillion":5.75,"cacheReadPerMillion":0.45,"contextTokens":1048576,"maxOutputTokens":131072,"deprecated":false},{"host":"scaleway","name":"Scaleway","kind":"cloud","url":"https://www.scaleway.com/en/docs/generative-apis/","inputPerMillion":2.01,"outputPerMillion":6.15,"cacheReadPerMillion":null,"contextTokens":256000,"maxOutputTokens":16384,"deprecated":false},{"host":"privatemode","name":"Privatemode","kind":"host","url":"https://docs.privatemode.ai/api/overview","inputPerMillion":1.791,"outputPerMillion":6.6326,"cacheReadPerMillion":0.1733,"contextTokens":1000000,"maxOutputTokens":131072,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:livebench","name":"LiveBench","url":"https://livebench.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:terminal-bench","name":"Terminal-Bench","url":"https://www.tbench.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:lmarena","name":"LMArena","url":"https://huggingface.co","license":"CC BY 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"qwen3-8-max","providerId":"alibaba","slug":"qwen3-8-max","name":"Qwen3.8 Max","provider":"Alibaba","description":"Qwen3.8 Max is a reasoning model from Alibaba. 64 benchmarks count toward its index, in 7 categories.","logoUrl":null,"releaseDate":"2026-08-03","contextWindowTokens":1000000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":true,"overallScore":65.56026544686976,"overallScoreRange":{"low":63.03901544686976,"high":68.08151544686976},"badges":[],"capabilityScores":{"agentic":63.71201246495429,"coding":64.26286065219253,"reasoning":66.69744080555876,"multimodal":67.1610708203873,"knowledge":62.63155723455324,"multilingual":null,"instruction":71.4456386571553,"math":67.58413396713306},"capabilityCounts":{"math":5,"multimodal":10,"knowledge":5,"reasoning":5,"coding":18,"agentic":19,"instruction":2},"benchmarks":[{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":99.44444444444444,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":66.77527653557718,"aboveFit":false,"run":"xhigh effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0.3883473863441843,"costPerTest":null,"latency":null},{"benchmarkId":"mathVisionPython","benchmarkName":"MathVision with Python","category":"multimodal","score":97.7,"unit":"percent","family":"mathVisionPython","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mathVision","benchmarkName":"MathVision","category":"multimodal","score":95.2,"unit":"percent","family":"mathVision","index":65.84341041275312,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:gpqa:overall","benchmarkName":"GPQA Diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":93.686,"unit":"percent","family":"gpqa-diamond","index":63.13533996038451,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","measuredAt":"2026-09-01","stderr":1.221,"costPerTest":0.077028,"latency":266.606},{"benchmarkId":"charxiv","benchmarkName":"CharXiv Reasoning","category":"multimodal","score":93.5,"unit":"percent","family":"charxiv","index":70.63380382188436,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"paperBench","benchmarkName":"PaperBench","category":"coding","score":93,"unit":"percent","family":"paperBench","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mrcrv2","benchmarkName":"MRCRv2","category":"reasoning","score":92.9,"unit":"percent","family":"mrcrv2","index":67.98312586409992,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":92.67676767676768,"unit":"percent","family":"gpqa-diamond","index":62.209699650870014,"aboveFit":false,"run":"xhigh effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.686498021450146,"costPerTest":null,"latency":null},{"benchmarkId":"gpqa","benchmarkName":"Graduate-Level Google-Proof Q&A","category":"knowledge","score":92.6,"unit":"percent","family":"gpqa-diamond","index":62.139290435421614,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gpqaDiamond","benchmarkName":"GPQA Diamond","category":"knowledge","score":92.6,"unit":"percent","family":"gpqa-diamond","index":62.139290435421614,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"omniDocBench15","benchmarkName":"OmniDocBench 1.5","category":"multimodal","score":92.1,"unit":"percent","family":"omniDocBench15","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-math:overall","benchmarkName":"LiveBench Mathematics","category":"math","domain":null,"eligible":true,"ranked":true,"score":91.31225,"unit":"percent","family":"livebench:livebench-math:overall","index":67.03846881001968,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Mathematics","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.10270395778364115,"latency":null},{"benchmarkId":"babyVisionPython","benchmarkName":"BabyVision with Python","category":"multimodal","score":91.3,"unit":"percent","family":"babyVisionPython","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mlvuAvg","benchmarkName":"MLVU mean average","category":"multimodal","score":90.8,"unit":"percent","family":"mlvuAvg","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"videoMmeWithSub","benchmarkName":"Video-MME with subtitle","category":"multimodal","score":90.4,"unit":"percent","family":"videoMmeWithSub","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"videoMmmu","benchmarkName":"VideoMMMU","category":"multimodal","score":88.7,"unit":"percent","family":"videoMmmu","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:mmlu_pro:overall","benchmarkName":"MMLU Pro","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":88.602,"unit":"percent","family":"mmlu-pro","index":56.10864304016491,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","measuredAt":"2026-09-01","stderr":0.313,"costPerTest":0.018585,"latency":63.576},{"benchmarkId":"charxivNoTools","benchmarkName":"CharXiv Reasoning without tools","category":"multimodal","score":88.4,"unit":"percent","family":"charxivNoTools","index":70.38012320081941,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-reasoning:overall","benchmarkName":"LiveBench Reasoning","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":88.2115,"unit":"percent","family":"livebench","index":75.89189728650138,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Reasoning","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.13149059405940594,"latency":null},{"benchmarkId":"vals-ai:mmmu:overall","benchmarkName":"MMMU Pro","category":"multimodal","domain":null,"eligible":true,"ranked":true,"score":88.035,"unit":"percent","family":"mmmu-pro","index":69.72264378760137,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmmu","measuredAt":"2026-09-01","stderr":0.78,"costPerTest":0.015881,"latency":43.853},{"benchmarkId":"realWorldQa","benchmarkName":"RealWorldQA","category":"multimodal","score":88,"unit":"percent","family":"realWorldQa","index":64.34848136704858,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:lcb:overall","benchmarkName":"LiveCodeBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":87.852,"unit":"percent","family":"livecodebench","index":62.238192953847424,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","measuredAt":"2026-09-01","stderr":0.951,"costPerTest":0.091412,"latency":337.788},{"benchmarkId":"terminalBench21","benchmarkName":"Terminal-Bench 2.1","category":"agentic","score":86.6,"unit":"percent","family":"terminal-bench-2-1","index":71.78959577071487,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"osWorldVerified","benchmarkName":"OSWorld-Verified","category":"agentic","score":86.1,"unit":"percent","family":"osWorldVerified","index":69.9858435474188,"aboveFit":false,"run":null,"source":"Tianbao Xie, Danyang Zhang, Jixuan Chen, Xiaochuan Li, Siheng Zhao, Ruisheng Cao, Toh Jing Hua, Zhoujun Cheng, Dongchan Shin, Fangyu Lei, Yitao Liu, Yiheng Xu, Shuyan Zhou, Silvio Savarese, Caiming Xiong, Victor Zhong, Tao Yu","sourceUrl":"https://os-world.github.io/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:swebench:overall","benchmarkName":"SWE-bench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":85.6,"unit":"percent","family":"swe-bench-verified","index":64.36285618775567,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/swebench","measuredAt":"2026-09-01","stderr":1.572,"costPerTest":1.128442,"latency":2502.097},{"benchmarkId":"androidWorld","benchmarkName":"AndroidWorld","category":"agentic","score":85.3,"unit":"percent","family":"androidWorld","index":64.46439788726856,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:medscribe:overall","benchmarkName":"MedScribe","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":84.947,"unit":"percent","family":"vals-ai:medscribe:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medscribe","measuredAt":"2026-10-29","stderr":1.999,"costPerTest":0.089616,"latency":251.26},{"benchmarkId":"screenSpotPro","benchmarkName":"ScreenSpot Pro","category":"multimodal","score":84.5,"unit":"percent","family":"screenSpotPro","index":64.42458983241086,"aboveFit":false,"run":null,"source":"Kaixin Li, Ziyang Meng, Hongzhan Lin, Ziyang Luo, Yuchen Tian, Jing Ma, Zhiyong Huang, Tat-Seng Chua","sourceUrl":"https://arxiv.org/abs/2504.07981","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:legal_bench:overall","benchmarkName":"LegalBench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":83.609,"unit":"percent","family":"vals-ai:legal_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","measuredAt":"2026-10-13","stderr":0.418,"costPerTest":0.010412,"latency":36.373},{"benchmarkId":"ifBench","benchmarkName":"Instruction Following Benchmark","category":"instruction","score":82.8,"unit":"percent","family":"ifbench-family","index":66.40110829994775,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmvu","benchmarkName":"Multimodal Multi-disciplinary Video Understanding","category":"multimodal","score":82.4,"unit":"percent","family":"mmvu","index":null,"aboveFit":false,"run":null,"source":"MMVU benchmark maintainers","sourceUrl":"https://www.kimi.com/blog/kimi-k2-5.html","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmmuPro","benchmarkName":"Massive Multi-discipline Multimodal Understanding Pro","category":"multimodal","score":82.3,"unit":"percent","family":"mmmu-pro","index":62.25904444714297,"aboveFit":false,"run":null,"source":"MMMU-Pro authors","sourceUrl":"https://arxiv.org/abs/2409.02813","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"babyVision","benchmarkName":"BabyVision","category":"multimodal","score":82,"unit":"percent","family":"babyVision","index":70.04403746746101,"aboveFit":false,"run":null,"source":"Meta AI","sourceUrl":"https://ai.meta.com/static-resource/muse-spark-1-1-evaluation-report","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"wideResearch","benchmarkName":"WideResearch","category":"agentic","score":81.9,"unit":"percent","family":"wideResearch","index":64.10531244057069,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lvBench","benchmarkName":"LVBench","category":"multimodal","score":81.8,"unit":"percent","family":"lvBench","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:sage:overall","benchmarkName":"SAGE","category":null,"domain":"Education","eligible":false,"ranked":true,"score":81.762,"unit":"percent","family":"vals-ai:sage:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/sage","measuredAt":"2026-11-05","stderr":2.079,"costPerTest":0.086734,"latency":112.397},{"benchmarkId":"vulcanBench","benchmarkName":"VulcanBench v3","category":"coding","score":81.2,"unit":"percent","family":"vulcanBench","index":60.653631379597456,"aboveFit":false,"run":null,"source":"VulcanBench contributors","sourceUrl":"https://github.com/morganlinton/VulcanBench/tree/main","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"medXpertQaMm","benchmarkName":"MedXpertQA Multimodal","category":"multimodal","score":80.4,"unit":"percent","family":"medXpertQaMm","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-language:overall","benchmarkName":"LiveBench Language","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":79.68733333333334,"unit":"percent","family":"livebench:livebench-language:overall","index":64.66994463851562,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Language","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.135232,"latency":null},{"benchmarkId":"ccOcr","benchmarkName":"CC-OCR","category":"multimodal","score":79.6,"unit":"percent","family":"ccOcr","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-design:overall","benchmarkName":"LiveBench Design","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":78.6,"unit":"percent","family":"livebench-2-agentic","index":76.07209564311812,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Design","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.59624,"latency":null},{"benchmarkId":"livebench:livebench-data-analysis:overall","benchmarkName":"LiveBench Data Analysis","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":78.413,"unit":"percent","family":"livebench","index":63.198587275259335,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Data%20Analysis","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.21894533333333333,"latency":null},{"benchmarkId":"mobileWorld","benchmarkName":"MobileWorld","category":"agentic","score":77.8,"unit":"percent","family":"mobileWorld","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"erqa","benchmarkName":"ERQA","category":"multimodal","score":77.8,"unit":"percent","family":"erqa","index":70.20670421031559,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:tax_eval_v2:overall","benchmarkName":"TaxEval v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":75.552,"unit":"percent","family":"vals-ai:tax_eval_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","measuredAt":"2026-09-01","stderr":0.838,"costPerTest":0.07645,"latency":265.751},{"benchmarkId":"simpleVqa","benchmarkName":"SimpleVQA","category":"multimodal","score":75,"unit":"percent","family":"simpleVqa","index":65.06386819946039,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"coworkBench","benchmarkName":"CoWorkBench","category":"agentic","score":74.8,"unit":"percent","family":"coworkBench","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tiers-1-3-v2-private:overall","benchmarkName":"FrontierMath-Tiers-1-3-v2-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":74.73684210526315,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tiers-1-3-v2-private:overall","index":70.23256104449021,"aboveFit":false,"run":"xhigh effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.57841025556124,"costPerTest":null,"latency":null},{"benchmarkId":"ocrBenchV2","benchmarkName":"OCRBench V2","category":"multimodal","score":74.2,"unit":"percent","family":"ocrBenchV2","index":null,"aboveFit":false,"run":null,"source":"OCRBench authors","sourceUrl":"https://arxiv.org/abs/2501.00321","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-instruction-following:overall","benchmarkName":"LiveBench Instruction Following","category":"instruction","domain":null,"eligible":true,"ranked":true,"score":74.08349999999999,"unit":"percent","family":"livebench:livebench-instruction-following:overall","index":76.49016901436283,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=IF","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.0820675,"latency":null},{"benchmarkId":"frontierSwe","benchmarkName":"FrontierSWE","category":"coding","score":73.5,"unit":"percent","family":"frontierSwe","index":null,"aboveFit":false,"run":null,"source":"Evan Chu, Rajan Agarwal, Abishek Thangamuthu, Brendan Graham, Justus Mattern","sourceUrl":"https://www.frontierswe.com/blog","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:ioi-v1:overall","benchmarkName":"IOI v1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":73,"unit":"percent","family":"vals-ai:ioi-v1:overall","index":76.4620559551821,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/ioi-v1","measuredAt":"2026-08-09","stderr":1,"costPerTest":2.392116,"latency":3946.139},{"benchmarkId":"livebench:livebench-coding:overall","benchmarkName":"LiveBench Coding","category":"coding","domain":null,"eligible":true,"ranked":true,"score":72.872,"unit":"percent","family":"livebench:livebench-coding:overall","index":57.3121543994268,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Coding","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.11293162393162393,"latency":null},{"benchmarkId":"toolathlonVerified","benchmarkName":"Toolathlon-Verified","category":"agentic","score":72.5,"unit":"percent","family":"toolathlonVerified","index":67.50930720341734,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hermesBench","benchmarkName":"Hermes Bench: 150 agent tasks graded on the files and state the agent leaves behind","category":"agentic","score":70.5,"unit":"percent","family":"hermesBench","index":null,"aboveFit":false,"run":null,"source":"Nous Research","sourceUrl":"https://portal.nousresearch.com/bench","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-research:overall","benchmarkName":"LiveBench Research","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":69.9,"unit":"percent","family":"livebench-2-agentic","index":69.70951511731015,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Research","measuredAt":"2026-10-14","stderr":null,"costPerTest":1.9546727272727273,"latency":null},{"benchmarkId":"vision2Web","benchmarkName":"Vision2Web","category":"multimodal","score":69,"unit":"percent","family":"vision2Web","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:ioi:overall","benchmarkName":"IOI","category":"coding","domain":null,"eligible":true,"ranked":true,"score":68.889,"unit":"percent","family":"vals-ai:ioi:overall","index":71.97910794191384,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/ioi","measuredAt":"2026-10-29","stderr":5.204,"costPerTest":9.468852,"latency":6398.213},{"benchmarkId":"swePro","benchmarkName":"SWE-bench Pro","category":"coding","score":67.7,"unit":"percent","family":"swePro","index":66.49337940033998,"aboveFit":false,"run":null,"source":"Xiang Deng, Jeff Da, Edwin Pan, Yannis Yiming He, Charles Ide, Kanak Garg, Niklas Lauffer, Andrew Park, Nitin Pasari, Chetan Rane, Karmini Sampath, Maya Krishnan, Srivatsa Kundurthy, Sean Hendryx, Zifan Wang, Chen Bo Calvin Zhang, Noah Jacobson, Bing Liu, Brad Kenstler","sourceUrl":"https://arxiv.org/abs/2509.16941","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:terminal-bench-2-1:overall","benchmarkName":"Terminal-Bench 2.1","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":67.416,"unit":"percent","family":"terminal-bench-2-1","index":60.569730458665276,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2-1","measuredAt":"2026-10-01","stderr":0.649,"costPerTest":0.448084,"latency":890.917},{"benchmarkId":"vals-ai:public-benefits-bench:overall","benchmarkName":"Public Benefits Bench v1.1","category":null,"domain":"Public benefits","eligible":false,"ranked":true,"score":67.118,"unit":"percent","family":"vals-ai:public-benefits-bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/public-benefits-bench","measuredAt":"2026-10-12","stderr":1.222,"costPerTest":0.991386,"latency":2443.976},{"benchmarkId":"webArenaVerified","benchmarkName":"WebArena-Verified Browser Agent Benchmark","category":"agentic","score":66.8,"unit":"percent","family":"webArenaVerified","index":null,"aboveFit":false,"run":null,"source":"Amine El Hattami, Megh Thakkar, Nicolas Chapados, Christopher Pal","sourceUrl":"https://openreview.net/forum?id=94tlGxmqkN","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"longBenchV2","benchmarkName":"LongBench v2","category":"reasoning","score":66.3,"unit":"percent","family":"longBenchV2","index":58.95069690390343,"aboveFit":false,"run":null,"source":"LongBench v2 authors","sourceUrl":"https://arxiv.org/abs/2412.15204","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:corp_fin_v2:overall","benchmarkName":"CorpFin v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":65.851,"unit":"percent","family":"vals-ai:corp_fin_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","measuredAt":"2026-08-12","stderr":0.935,"costPerTest":0.179323,"latency":46.649},{"benchmarkId":"vals-ai:vibe-code:overall","benchmarkName":"Vibe Code Bench v1.1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":64.696,"unit":"percent","family":"vals-ai:vibe-code:overall","index":65.5928558413547,"aboveFit":false,"run":"OpenHands","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","measuredAt":"2026-11-09","stderr":5.362,"costPerTest":8.238852,"latency":10135.139},{"benchmarkId":"livebench:livebench-agentic-coding:overall","benchmarkName":"LiveBench Agentic Coding","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":64.64666666666666,"unit":"percent","family":"livebench:livebench-agentic-coding:overall","index":75.83169665458435,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Agentic%20Coding","measuredAt":"2026-06-25","stderr":null,"costPerTest":1.7473416666666668,"latency":null},{"benchmarkId":"vals-ai:mortgage_tax:overall","benchmarkName":"MortgageTax","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":63.99,"unit":"percent","family":"vals-ai:mortgage_tax:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mortgage_tax","measuredAt":"2026-09-01","stderr":0.942,"costPerTest":0.027349,"latency":47.499},{"benchmarkId":"scarfBench","benchmarkName":"ScarfBench: Cross-Framework Application Migration in Enterprise Java","category":"coding","score":63.7,"unit":"percent","family":"scarfBench","index":69.6471797057518,"aboveFit":false,"run":null,"source":"Advait Pavuluri, Bridget McGinn, Ashita Saxena, George Safta, Srikanth Tamilselvam, Raju Pavuluri, Michele Merler, Baishakhi Ray, Rahul Krishna (IBM Research)","sourceUrl":"https://arxiv.org/abs/2605.06754","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"perceptionBench","benchmarkName":"PerceptionBench (Internal)","category":"multimodal","score":63.5,"unit":"percent","family":"perceptionBench","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"openHarmonyBench","benchmarkName":"OpenHarmony Bench v1.0","category":"coding","score":60.8,"unit":"percent","family":"openHarmonyBench","index":67.37351894160618,"aboveFit":false,"run":null,"source":"OpenHarmony Bench authors","sourceUrl":"https://arxiv.org/abs/2608.16022","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-coding-2:overall","benchmarkName":"LiveBench Coding 2.0","category":"coding","domain":null,"eligible":true,"ranked":true,"score":60.3,"unit":"percent","family":"livebench:livebench-coding-2:overall","index":51.84872096153311,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Coding","measuredAt":"2026-10-14","stderr":null,"costPerTest":1.9869172413793104,"latency":null},{"benchmarkId":"vals-ai:emb:overall","benchmarkName":"EMB","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":60.066,"unit":"percent","family":"vals-ai:emb:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/emb","measuredAt":"2026-10-20","stderr":3.352,"costPerTest":2.655924,"latency":3467.209},{"benchmarkId":"vals-ai:proof_bench:overall","benchmarkName":"ProofBench v1.1","category":"math","domain":null,"eligible":true,"ranked":true,"score":58,"unit":"percent","family":"vals-ai:proof_bench:overall","index":67.18081324818813,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/proof_bench","measuredAt":"2026-11-09","stderr":4.96,"costPerTest":1.503984,"latency":1867.622},{"benchmarkId":"deepSwe","benchmarkName":"DeepSWE","category":"agentic","score":56.6,"unit":"percent","family":"deepSwe","index":62.22334058692399,"aboveFit":false,"run":null,"source":"Datacurve AI","sourceUrl":"https://deepswe.datacurve.ai/blog","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hleWithTools","benchmarkName":"Humanity's Last Exam with tools","category":"agentic","score":56.2,"unit":"percent","family":"hleWithTools","index":65.32441842087766,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"nl2Repo","benchmarkName":"NL2Repo","category":"coding","score":55.9,"unit":"percent","family":"nl2Repo","index":66.19451183561902,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"tau2-bench:banking-knowledge:overall","benchmarkName":"τ²-bench Banking","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":55.154639175257735,"unit":"percent","family":"tau2-bench","index":39.38857569977796,"aboveFit":false,"run":"xhigh effort · Sierra","source":"Sierra Research","sourceUrl":"https://github.com/sierra-research/tau2-bench","measuredAt":"2026-10-20","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"jobBench","benchmarkName":"JobBench","category":"agentic","score":53.4,"unit":"percent","family":"jobBench","index":66.02908788069593,"aboveFit":false,"run":null,"source":"Yuetai Li, Yichen Feng, Zhangchen Xu, Zixian Ma, Kaiyuan Zheng, Fengqing Jiang, Xinghua Sun, Rulin Shao, Zichen Chen, Yue Huang, Xinyang Han, Brian Lee, Kayla Xu, Shenglai Zeng, Hang Hua, Xiangliang Zhang, Basel Alomair, Ranjay Krishna, Luke Zettlemoyer, Pang Wei Koh, Bhaskar Ramasubramanian, Luyao Niu, Xiang Yue, Radha Poovendran","sourceUrl":"https://arxiv.org/abs/2605.26329","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"agentsLastExam","benchmarkName":"Agents' Last Exam","category":"agentic","score":52.4,"unit":"percent","family":"agentsLastExam","index":84.5590264398374,"aboveFit":true,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-mirrorcode:overall","benchmarkName":"MirrorCode","category":"coding","domain":null,"eligible":true,"ranked":true,"score":52.22222222222223,"unit":"percent","family":"epoch-ai:epoch-mirrorcode:overall","index":72.5001395704332,"aboveFit":false,"run":"xhigh effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":10.894718232315695,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:fabv2:overall","benchmarkName":"Finance Agent (v2)","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":50.593,"unit":"percent","family":"vals-ai:fabv2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/fabv2","measuredAt":"2026-10-29","stderr":0.578,"costPerTest":1.236977,"latency":1174.928},{"benchmarkId":"zeroBenchPython","benchmarkName":"ZeroBench_main with Python","category":"multimodal","score":49,"unit":"percent","family":"zeroBenchPython","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:legal_research:overall","benchmarkName":"Legal Research Bench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":47.596,"unit":"percent","family":"vals-ai:legal_research:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_research","measuredAt":"2026-11-09","stderr":3.471,"costPerTest":2.486351,"latency":3677.636},{"benchmarkId":"epoch-ai:epoch-frontiermath-tier-4-v2-private:overall","benchmarkName":"FrontierMath-Tier-4-v2-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":46.34146341463415,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tier-4-v2-private:overall","index":66.69355019739008,"aboveFit":false,"run":"xhigh effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":7.884502378168105,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-chess-puzzles:overall","benchmarkName":"Chess Puzzles","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":46,"unit":"percent","family":"epoch-ai:epoch-chess-puzzles:overall","index":68.89937470204336,"aboveFit":false,"run":"xhigh effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":5.009082659620331,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-simpleqa-verified:overall","benchmarkName":"SimpleQA Verified","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":45.800000000000004,"unit":"percent","family":"epoch-ai:epoch-simpleqa-verified:overall","index":62.07449545432734,"aboveFit":false,"run":"xhigh effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.5763390640483554,"costPerTest":null,"latency":null},{"benchmarkId":"cweBenchV2","benchmarkName":"CWE-bench v2","category":"agentic","score":45.4,"unit":"percent","family":"cweBenchV2","index":62.272187771423745,"aboveFit":false,"run":null,"source":"Collinear AI","sourceUrl":"https://cwe-bench.com/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hle","benchmarkName":"Humanity's Last Exam","category":"knowledge","score":43.6,"unit":"percent","family":"hle","index":68.13020799661251,"aboveFit":false,"run":null,"source":"Center for AI Safety, Scale AI, and thousands of expert contributors","sourceUrl":"https://lastexam.ai/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hleNoTools","benchmarkName":"Humanity's Last Exam without tools","category":"knowledge","score":43.6,"unit":"percent","family":"hle","index":68.13020799661251,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-long-horizon:overall","benchmarkName":"LiveBench Long Agentic","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":42.3,"unit":"percent","family":"livebench-2-agentic","index":49.52477689750553,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Long%20Agentic","measuredAt":"2026-10-14","stderr":null,"costPerTest":14.930050000000001,"latency":null},{"benchmarkId":"vals-ai:skillsbench:overall","benchmarkName":"SkillsBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":42.011,"unit":"percent","family":"vals-ai:skillsbench:overall","index":56.4127152831644,"aboveFit":false,"run":"OpenHands","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/skillsbench","measuredAt":"2026-10-29","stderr":4.435,"costPerTest":0.524988,"latency":948.433},{"benchmarkId":"mlsBenchLite","benchmarkName":"MLS-Bench Lite","category":"coding","score":41,"unit":"percent","family":"mlsBenchLite","index":null,"aboveFit":false,"run":null,"source":"MLS-Bench","sourceUrl":"https://mls-bench.com/","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:medcode:overall","benchmarkName":"MedCode","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":40.668,"unit":"percent","family":"vals-ai:medcode:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medcode","measuredAt":"2026-10-29","stderr":2.029,"costPerTest":0.120827,"latency":378.022},{"benchmarkId":"epoch-ai:epoch-mystery-game-puzzles:overall","benchmarkName":"Mystery Game Puzzles","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":38,"unit":"percent","family":"epoch-ai:epoch-mystery-game-puzzles:overall","index":68.10876427686672,"aboveFit":false,"run":"xhigh effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":4.878317312145635,"costPerTest":null,"latency":null},{"benchmarkId":"hermesIndex","benchmarkName":"Hermes Index: agentic capability and cost per task inside Hermes Agent","category":"agentic","score":36.17,"unit":"percent","family":"hermesIndex","index":65.47180064719741,"aboveFit":false,"run":null,"source":"Nous Research","sourceUrl":"https://portal.nousresearch.com/bench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:terminal-bench-4:overall","benchmarkName":"Terminal-Bench 4.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":34.343,"unit":"percent","family":"terminal-bench-4","index":70.8233965846735,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-4","measuredAt":"2026-11-09","stderr":3.945,"costPerTest":10.633329,"latency":8900.152},{"benchmarkId":"vals-ai:tax_agent_bench:overall","benchmarkName":"Tax Agent Bench","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":32.095,"unit":"percent","family":"vals-ai:tax_agent_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench","measuredAt":"2026-11-09","stderr":3.367,"costPerTest":1.552558,"latency":2436.106},{"benchmarkId":"livebench:livebench-browser:overall","benchmarkName":"LiveBench Browser Use","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":32,"unit":"percent","family":"livebench-2-agentic","index":41.992066619824826,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Browser","measuredAt":"2026-10-14","stderr":null,"costPerTest":1.5425559999999998,"latency":null},{"benchmarkId":"vals-ai:cyber:overall","benchmarkName":"CyberBench v1.1","category":null,"domain":"Other","eligible":false,"ranked":true,"score":28.572,"unit":"percent","family":"vals-ai:cyber:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/cyber","measuredAt":"2026-11-05","stderr":3.306,"costPerTest":0.784693,"latency":580.168},{"benchmarkId":"automationBench","benchmarkName":"AutomationBench","category":"agentic","score":27.3,"unit":"percent","family":"automationBench","index":34.01771816945907,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"bug-hunt-bench","benchmarkName":"Bug Hunt Bench","category":"coding","score":25.7,"unit":"percent","family":"bug-hunt-bench","index":67.27535508473208,"aboveFit":false,"run":null,"source":"Pawel Huryn","sourceUrl":"https://bughunt.productcompass.pm/method","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"zeroBench","benchmarkName":"ZeroBench","category":"multimodal","score":24,"unit":"percent","family":"zeroBench","index":64.67484557434756,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:code-migration:overall","benchmarkName":"Code Migration","category":"coding","domain":null,"eligible":true,"ranked":true,"score":23.958,"unit":"percent","family":"vals-ai:code-migration:overall","index":57.91658760953322,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/code-migration","measuredAt":"2026-10-29","stderr":4.309,"costPerTest":10.53816,"latency":16065.728},{"benchmarkId":"vals-ai:mysterymechanism:overall","benchmarkName":"MysteryMechanism","category":null,"domain":"Science","eligible":false,"ranked":true,"score":23.874,"unit":"percent","family":"vals-ai:mysterymechanism:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mysterymechanism","measuredAt":"2026-11-11","stderr":2.868,"costPerTest":1.309463,"latency":2268.77},{"benchmarkId":"osWorld2","benchmarkName":"OSWorld 2.0","category":"agentic","score":19.4,"unit":"percent","family":"osWorld2","index":61.91517562603979,"aboveFit":false,"run":null,"source":"Mengqi Yuan, Zilong Zhou, Xinzhuang Xiong, Weiming Wu, Jiayang Sun, Jiamin Song, Kaiqian Cui, Bowen Wang, Haoyuan Wu, Yitong Li, Dunjie Lu, Haikong Lu, Qi Zhen, Xinyuan Wang, Jiaqi Deng, Yuhao Yang, Cheng Chen, Boyuan Zheng, Alex Su, Xiao Yu, Hao Zou, Saaket Agashe, Xing Han Lu, Manpreet Kaur, Zhengyang Qi, Vincent Sunn Chen, Frederic Sala, Dayiheng Liu, Junyang Lin, Zhou Yu, Yu Su, Siva Reddy, Xin Eric Wang, Peng Qi, Tianbao Xie, Tao Yu","sourceUrl":"https://arxiv.org/abs/2606.29537","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"frontierSweV2","benchmarkName":"FrontierSWE v2","category":"coding","score":15.8,"unit":"percent","family":"frontierSweV2","index":60.83222324595356,"aboveFit":false,"run":null,"source":"Proximal","sourceUrl":"https://www.frontierswe.com/blog/v2","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:terminal-bench-science:overall","benchmarkName":"Terminal-Bench-Science","category":null,"domain":"Science","eligible":false,"ranked":true,"score":12.857,"unit":"percent","family":"vals-ai:terminal-bench-science:overall","index":null,"aboveFit":false,"run":"Mini-SWE-agent","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-science","measuredAt":"2026-11-09","stderr":4.03,"costPerTest":16.682189,"latency":12243.338},{"benchmarkId":"vals-ai:vcb-1-100:overall","benchmarkName":"Vibe Code Bench 1-100","category":"coding","domain":null,"eligible":true,"ranked":true,"score":12.833,"unit":"percent","family":"vals-ai:vcb-1-100:overall","index":61.636305441720836,"aboveFit":false,"run":"OpenHands","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vcb-1-100","measuredAt":"2026-10-29","stderr":3.183,"costPerTest":9.675734,"latency":5735.815},{"benchmarkId":"lmarena:agent-task-outcome:overall","benchmarkName":"Agent Arena task outcome","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":4.491406828957157,"unit":"index","family":"agent-arena","index":70.22344184785162,"aboveFit":false,"run":null,"source":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset","measuredAt":"2026-11-11","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lmarena:agent-bash-recovery:overall","benchmarkName":"Agent Arena command recovery","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":2.7983391648861877,"unit":"index","family":"agent-arena","index":68.8314748035197,"aboveFit":false,"run":null,"source":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset","measuredAt":"2026-10-08","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mazeBench","benchmarkName":"MazeBench: visual spatial reasoning in a 3D open world","category":"agentic","score":0,"unit":"percent","family":"mazeBench","index":61.97866734834783,"aboveFit":false,"run":null,"source":"Jonathan Pappas and David Pappas","sourceUrl":"https://mazebench.com/blog?post=maze-bench-results","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:programbench:overall","benchmarkName":"ProgramBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":0,"unit":"percent","family":"vals-ai:programbench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/programbench","measuredAt":"2026-11-09","stderr":0,"costPerTest":12.357107,"latency":21263.689},{"benchmarkId":"lmarena:agent-steerability:overall","benchmarkName":"Agent Arena steerability","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":-0.7752682922551712,"unit":"index","family":"agent-arena","index":65.89340923456375,"aboveFit":false,"run":null,"source":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset","measuredAt":"2026-11-11","stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":41,"timeToFirstToken":52.16,"source":"BenchLM","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":{"elo":1485.2093041935336,"eloUpper":1489.7959141573651,"eloLower":1480.6226942297021,"votes":36324,"rank":16,"rankUpper":null,"rankLower":null,"arenaName":"Qwen3.8-Max","organization":"alibaba","updatedAt":"2026-11-09"},"quality":{"coveragePercent":95,"benchmarkCount":64,"categoryCount":7,"confidence":"high","state":"available"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:livebench","name":"LiveBench","url":"https://livebench.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:sierra-research","name":"Sierra Research","url":"https://github.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:lmarena","name":"LMArena","url":"https://huggingface.co","license":"CC BY 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gpt-6-luna","providerId":"openai","slug":"gpt-6-luna","name":"GPT-6 Luna","provider":"OpenAI","description":"GPT-6 Luna is a reasoning model from OpenAI in the GPT-6 family. 57 benchmarks count toward its index, in 8 categories.","logoUrl":null,"releaseDate":"2026-09-22","contextWindowTokens":1050000,"modalities":["file","image","text"],"openWeights":false,"details":{"apiId":"openai/gpt-6-luna","maxOutputTokens":128000,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.01,"weightsUrl":null},"reasoning":true,"overallScore":65.51503351362845,"overallScoreRange":{"low":63.373264692254274,"high":67.65680233500262},"badges":[],"capabilityScores":{"agentic":66.88559437458619,"coding":66.75369921317554,"reasoning":64.29523477370019,"multimodal":64.97069254793462,"knowledge":63.46535676122103,"multilingual":53.34461743581888,"instruction":39.76386649183439,"math":68.53346341215222},"capabilityCounts":{"math":5,"multimodal":3,"knowledge":7,"reasoning":10,"coding":9,"agentic":21,"multilingual":1,"instruction":1},"benchmarks":[{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":98.88888888888889,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":66.45205762128103,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0.6674027922425511,"costPerTest":null,"latency":null},{"benchmarkId":"arc-prize:v1-public-eval:overall","benchmarkName":"ARC-AGI-1 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":92.5,"unit":"percent","family":"arc-prize:v1-public-eval:overall","index":null,"aboveFit":false,"run":"max effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.01291232625,"latency":null},{"benchmarkId":"extractBenchLlamaIndex","benchmarkName":"ExtractBench: schema-guided enterprise document extraction (LlamaIndex leaderboard)","category":"multimodal","score":91.01,"unit":"percent","family":"extractBenchLlamaIndex","index":72.84061207302207,"aboveFit":false,"run":null,"source":"Boyang Zhang, Adrian Lyjak, Eli Stewart, Zhaoqi Li and Simon Suo (LlamaIndex)","sourceUrl":"https://arxiv.org/abs/2607.29677","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":90.46717171717171,"unit":"percent","family":"gpqa-diamond","index":60.18311861411543,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.8537378631915502,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-math:overall","benchmarkName":"LiveBench Mathematics","category":"math","domain":null,"eligible":true,"ranked":true,"score":89.1235,"unit":"percent","family":"livebench:livebench-math:overall","index":63.98796570638558,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Mathematics","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.013850395778364116,"latency":null},{"benchmarkId":"arc-prize:v1-semi-private:overall","benchmarkName":"ARC-AGI-1 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":86.66666666666667,"unit":"percent","family":"arc-prize:v1-semi-private:overall","index":66.83115445130515,"aboveFit":false,"run":"max effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.018388256999999998,"latency":null},{"benchmarkId":"livebench:livebench-coding-2:overall","benchmarkName":"LiveBench Coding 2.0","category":"coding","domain":null,"eligible":true,"ranked":true,"score":84.5,"unit":"percent","family":"livebench:livebench-coding-2:overall","index":69.37738780107277,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Coding","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.0950551724137931,"latency":null},{"benchmarkId":"vals-ai:medscribe:overall","benchmarkName":"MedScribe","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":83.71,"unit":"percent","family":"vals-ai:medscribe:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medscribe","measuredAt":"2026-10-29","stderr":1.949,"costPerTest":0.009562,"latency":169.606},{"benchmarkId":"vals-ai:sage:overall","benchmarkName":"SAGE","category":null,"domain":"Education","eligible":false,"ranked":true,"score":83.404,"unit":"percent","family":"vals-ai:sage:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/sage","measuredAt":"2026-11-05","stderr":2.296,"costPerTest":0.005772,"latency":47.63},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":83.3,"unit":"percent","family":"lcr","index":64.4485572404562,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-design:overall","benchmarkName":"LiveBench Design","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":82.5,"unit":"percent","family":"livebench-2-agentic","index":78.9242869133079,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Design","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.030567999999999998,"latency":null},{"benchmarkId":"livebench:livebench-reasoning:overall","benchmarkName":"LiveBench Reasoning","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":81.76925,"unit":"percent","family":"livebench","index":67.54638762482622,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Reasoning","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.021404950495049507,"latency":null},{"benchmarkId":"vals-ai:vibe-code:overall","benchmarkName":"Vibe Code Bench v1.1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":81.649,"unit":"percent","family":"vals-ai:vibe-code:overall","index":71.70976459296094,"aboveFit":false,"run":"OpenHands","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","measuredAt":"2026-11-09","stderr":3.377,"costPerTest":1.345703,"latency":2219.515},{"benchmarkId":"aaMmmuPro","benchmarkName":"Artificial Analysis MMMU-Pro","category":"multimodal","score":79.7,"unit":"percent","family":"mmmu-pro","index":58.875372557833145,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmmu-pro","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-coding:overall","benchmarkName":"LiveBench Coding","category":"coding","domain":null,"eligible":true,"ranked":true,"score":78.95,"unit":"percent","family":"livebench:livebench-coding:overall","index":65.86017821660711,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Coding","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.005235897435897436,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tiers-1-3-v2-private:overall","benchmarkName":"FrontierMath-Tiers-1-3-v2-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":78.94736842105263,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tiers-1-3-v2-private:overall","index":72.44877877627431,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.4191503146153774,"costPerTest":null,"latency":null},{"benchmarkId":"aaCyberGymE2e","benchmarkName":"Artificial Analysis CyberGym-E2E-AA","category":"agentic","score":77.9,"unit":"percent","family":"aaCyberGymE2e","index":95,"aboveFit":true,"run":null,"source":"Artificial Analysis; dataset by Berkeley RDI","sourceUrl":"https://artificialanalysis.ai/evaluations/cybergym-e2e-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:cyber:overall","benchmarkName":"CyberBench v1.1","category":null,"domain":"Other","eligible":false,"ranked":true,"score":76.25,"unit":"percent","family":"vals-ai:cyber:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/cyber","measuredAt":"2026-11-05","stderr":5.288,"costPerTest":0.133802,"latency":949.797},{"benchmarkId":"livebench:livebench-language:overall","benchmarkName":"LiveBench Language","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":73.82766666666667,"unit":"percent","family":"livebench:livebench-language:overall","index":58.04377709903455,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Language","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.014226666666666665,"latency":null},{"benchmarkId":"livebench:livebench-data-analysis:overall","benchmarkName":"LiveBench Data Analysis","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":73.36966666666667,"unit":"percent","family":"livebench","index":56.66528182136022,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Data%20Analysis","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.016606000000000003,"latency":null},{"benchmarkId":"vals-ai:terminal-bench-2-1:overall","benchmarkName":"Terminal-Bench 2.1","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":73.034,"unit":"percent","family":"terminal-bench-2-1","index":63.85544789627447,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2-1","measuredAt":"2026-10-01","stderr":1.716,"costPerTest":0.029222,"latency":425.094},{"benchmarkId":"livebench:livebench-browser:overall","benchmarkName":"LiveBench Browser Use","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":72,"unit":"percent","family":"livebench-2-agentic","index":71.24531041664311,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Browser","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.23820799999999998,"latency":null},{"benchmarkId":"hermesBench","benchmarkName":"Hermes Bench: 150 agent tasks graded on the files and state the agent leaves behind","category":"agentic","score":70.47,"unit":"percent","family":"hermesBench","index":null,"aboveFit":false,"run":null,"source":"Nous Research","sourceUrl":"https://portal.nousresearch.com/bench","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:emb:overall","benchmarkName":"EMB","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":68.515,"unit":"percent","family":"vals-ai:emb:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/emb","measuredAt":"2026-10-20","stderr":2.809,"costPerTest":0.141708,"latency":1041.052},{"benchmarkId":"deepSwe","benchmarkName":"DeepSWE","category":"agentic","score":66.6,"unit":"percent","family":"deepSwe","index":69.49306142686353,"aboveFit":false,"run":null,"source":"Datacurve AI","sourceUrl":"https://deepswe.datacurve.ai/blog","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"parseBenchLlamaIndex","benchmarkName":"ParseBench: document parsing for AI agents (LlamaIndex leaderboard)","category":"multimodal","score":64.13,"unit":"percent","family":"parseBenchLlamaIndex","index":63.19609301294863,"aboveFit":false,"run":null,"source":"Boyang Zhang, Sebastián G. Acosta, Preston Carlson, Sacha Bron, Pierre-Loïc Doulcet, Daniel B. Ospina and Simon Suo (LlamaIndex)","sourceUrl":"https://arxiv.org/abs/2604.08538","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:proof_bench:overall","benchmarkName":"ProofBench v1.1","category":"math","domain":null,"eligible":true,"ranked":true,"score":64,"unit":"percent","family":"vals-ai:proof_bench:overall","index":69.25228145413756,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/proof_bench","measuredAt":"2026-11-09","stderr":4.824,"costPerTest":0.041637,"latency":451.481},{"benchmarkId":"perceptionBenchTools","benchmarkName":"PerceptionBench with tools","category":"multimodal","score":62.8,"unit":"percent","family":"perceptionBenchTools","index":null,"aboveFit":false,"run":null,"source":"Meta (reported run)","sourceUrl":"https://research.meta.ai/blog/muse-ember","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-french:overall","benchmarkName":"EuroEval French","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":62.73125000000001,"unit":"percent","family":"euroeval","index":57.33210765910544,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/french","measuredAt":null,"stderr":0.3799342048302574,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-italian:overall","benchmarkName":"EuroEval Italian","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":62.14750000000001,"unit":"percent","family":"euroeval","index":56.540093226153786,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/italian","measuredAt":null,"stderr":0.5782557284627624,"costPerTest":null,"latency":null},{"benchmarkId":"arc-prize:v2-public-eval:overall","benchmarkName":"ARC-AGI-2 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":61.80555555555556,"unit":"percent","family":"arc-prize:v2-public-eval:overall","index":null,"aboveFit":false,"run":"max effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.06202490583333333,"latency":null},{"benchmarkId":"vals-ai:biomysterybench:overall","benchmarkName":"BioMysteryBench","category":null,"domain":"Science","eligible":false,"ranked":true,"score":61.481,"unit":"percent","family":"vals-ai:biomysterybench:overall","index":null,"aboveFit":false,"run":"Terminus 2","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/biomysterybench","measuredAt":"2026-11-09","stderr":2.593,"costPerTest":0.053236,"latency":523.805},{"benchmarkId":"healthBenchProfessionalRaw","benchmarkName":"HealthBench Professional raw score","category":"knowledge","score":61.2,"unit":"percent","family":"healthBenchProfessionalRaw","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":61.17875,"unit":"percent","family":"euroeval","index":55.22572237917834,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":0.653772705150651,"costPerTest":null,"latency":null},{"benchmarkId":"healthBenchProfessional","benchmarkName":"HealthBench Professional","category":"knowledge","score":60.8,"unit":"percent","family":"healthBenchProfessional","index":73.18265203378209,"aboveFit":false,"run":null,"source":"Rebecca Soskin Hicks, Mikhail Trofimov, Dominick Lim, Rahul K. Arora, Foivos Tsimpourlas, Preston Bowman, Michael Sharman, Chi Tong, Kavin Karthik, Arnav Dugar, Akshay Jagadeesh, Khaled Saab, Johannes Heidecke, Ashley Alexander, Nate Gross, Karan Singhal","sourceUrl":"https://arxiv.org/abs/2604.27470","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-portuguese:overall","benchmarkName":"EuroEval Portuguese","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":60.55125000000001,"unit":"percent","family":"euroeval","index":54.37434926281491,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/portuguese","measuredAt":null,"stderr":0.45845699089445674,"costPerTest":null,"latency":null},{"benchmarkId":"arc-prize:v2-semi-private:overall","benchmarkName":"ARC-AGI-2 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":59.30555555555556,"unit":"percent","family":"arc-prize:v2-semi-private:overall","index":66.41775391846735,"aboveFit":false,"run":"max effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.06181100083333334,"latency":null},{"benchmarkId":"euroeval:euroeval-dutch:overall","benchmarkName":"EuroEval Dutch","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":59.03333333333333,"unit":"percent","family":"euroeval","index":52.31488560882285,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/dutch","measuredAt":null,"stderr":0.4382893232377416,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:public-benefits-bench:overall","benchmarkName":"Public Benefits Bench v1.1","category":null,"domain":"Public benefits","eligible":false,"ranked":true,"score":57.645,"unit":"percent","family":"vals-ai:public-benefits-bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/public-benefits-bench","measuredAt":"2026-10-12","stderr":1.285,"costPerTest":0.224168,"latency":3178.908},{"benchmarkId":"agentsLastExamLinux","benchmarkName":"Agents' Last Exam, Linux subset","category":"agentic","score":57.1,"unit":"percent","family":"agentsLastExamLinux","index":null,"aboveFit":false,"run":null,"source":"Meta (reported run)","sourceUrl":"https://research.meta.ai/blog/muse-ember","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaCweBench","benchmarkName":"Artificial Analysis CWE-Bench-AA","category":"agentic","score":56.7,"unit":"percent","family":"aaCweBench","index":69.08108704954687,"aboveFit":false,"run":null,"source":"Artificial Analysis; dataset by Collinear AI","sourceUrl":"https://artificialanalysis.ai/evaluations/cwe-bench-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tier-4-v2-private:overall","benchmarkName":"FrontierMath-Tier-4-v2-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":56.09756097560976,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tier-4-v2-private:overall","index":70.5262335026826,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":7.846686801046543,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":55.97285714285715,"unit":"percent","family":"euroeval","index":48.162523613855804,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.4638349460129726,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-instruction-following:overall","benchmarkName":"LiveBench Instruction Following","category":"instruction","domain":null,"eligible":true,"ranked":true,"score":55.93325,"unit":"percent","family":"livebench:livebench-instruction-following:overall","index":39.76386649183439,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=IF","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.034782,"latency":null},{"benchmarkId":"vals-ai:ioi:overall","benchmarkName":"IOI","category":"coding","domain":null,"eligible":true,"ranked":true,"score":55.556,"unit":"percent","family":"vals-ai:ioi:overall","index":66.9736010720616,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/ioi","measuredAt":"2026-10-29","stderr":8.866,"costPerTest":0.199412,"latency":2147.141},{"benchmarkId":"aaSciCode","benchmarkName":"Artificial Analysis SciCode","category":"coding","score":54.6,"unit":"percent","family":"scicode-family","index":67.02133503675569,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/scicode","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"healthBenchLengthAdjusted","benchmarkName":"HealthBench length-adjusted score","category":"knowledge","score":54.5,"unit":"percent","family":"healthBenchLengthAdjusted","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaAutomationBench","benchmarkName":"Artificial Analysis AutomationBench","category":"agentic","score":53.2,"unit":"percent","family":"aaAutomationBench","index":64.16618550353525,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/automationbench-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaCyberIndex","benchmarkName":"Artificial Analysis Cyber Index","category":"agentic","score":52.7,"unit":"percent","family":"aaCyberIndex","index":81.33094634396139,"aboveFit":false,"run":null,"source":"Artificial Analysis with the Cyber Index Alliance (Collinear AI, IBM, NVIDIA, Pasta Lab, Vercel)","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-cyber-index","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-spanish:overall","benchmarkName":"EuroEval Spanish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":52.146249999999995,"unit":"percent","family":"euroeval","index":42.970698198217704,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/spanish","measuredAt":null,"stderr":0.49991874339736453,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-agentic-coding:overall","benchmarkName":"LiveBench Agentic Coding","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":51.211999999999996,"unit":"percent","family":"livebench:livebench-agentic-coding:overall","index":61.97675373620218,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Agentic%20Coding","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.03205833333333333,"latency":null},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":50.76375,"unit":"percent","family":"euroeval","index":41.09496380240503,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.5375508696858372,"costPerTest":null,"latency":null},{"benchmarkId":"healthBench","benchmarkName":"HealthBench raw score","category":"knowledge","score":50,"unit":"percent","family":"healthBench","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:fabv2:overall","benchmarkName":"Finance Agent (v2)","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":49.873,"unit":"percent","family":"vals-ai:fabv2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/fabv2","measuredAt":"2026-10-29","stderr":0.232,"costPerTest":0.124338,"latency":897.881},{"benchmarkId":"osWorld21Offline","benchmarkName":"OSWorld 2.1 offline subset (82 tasks)","category":"agentic","score":48.9,"unit":"percent","family":"osWorld21Offline","index":60.580197511115074,"aboveFit":false,"run":null,"source":"Mengqi Yuan, Zilong Zhou, Xinzhuang Xiong, Weiming Wu, Jiayang Sun, Jiamin Song, Kaiqian Cui, Bowen Wang, Haoyuan Wu, Yitong Li, Dunjie Lu, Haikong Lu, Qi Zhen, Xinyuan Wang, Jiaqi Deng, Yuhao Yang, Cheng Chen, Boyuan Zheng, Alex Su, Xiao Yu, Hao Zou, Saaket Agashe, Xing Han Lu, Manpreet Kaur, Zhengyang Qi, Vincent Sunn Chen, Frederic Sala, Dayiheng Liu, Junyang Lin, Zhou Yu, Yu Su, Siva Reddy, Xin Eric Wang, Peng Qi, Tianbao Xie, Tao Yu","sourceUrl":"https://github.com/xlang-ai/OSWorld-V2/releases","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-long-horizon:overall","benchmarkName":"LiveBench Long Agentic","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":48.1,"unit":"percent","family":"livebench-2-agentic","index":53.76649724804418,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Long%20Agentic","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.9676111111111112,"latency":null},{"benchmarkId":"gdpvalAaNormalized","benchmarkName":"GDPval-AA normalized","category":"agentic","score":46.6,"unit":"percent","family":"gdpvalAaNormalized","index":67.35169201203216,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:medcode:overall","benchmarkName":"MedCode","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":44.685,"unit":"percent","family":"vals-ai:medcode:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medcode","measuredAt":"2026-10-29","stderr":2.303,"costPerTest":0.007323,"latency":108.772},{"benchmarkId":"epoch-ai:epoch-furniture-assembly:overall","benchmarkName":"Furniture Assembly","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":44.166666666666664,"unit":"percent","family":"epoch-ai:epoch-furniture-assembly:overall","index":67.25298505902941,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":6.410142328406845,"costPerTest":null,"latency":null},{"benchmarkId":"aaAnalystAgent","benchmarkName":"Artificial Analysis AnalystAgent","category":"agentic","score":43.8,"unit":"percent","family":"aaAnalystAgent","index":65.00447886013762,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/aa-analyst-agent","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-research:overall","benchmarkName":"LiveBench Research","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":43.5,"unit":"percent","family":"livebench-2-agentic","index":50.40237421141008,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Research","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.3553454545454545,"latency":null},{"benchmarkId":"vals-ai:code-migration:overall","benchmarkName":"Code Migration","category":"coding","domain":null,"eligible":true,"ranked":true,"score":42.554,"unit":"percent","family":"vals-ai:code-migration:overall","index":67.7955283920046,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/code-migration","measuredAt":"2026-10-29","stderr":4.415,"costPerTest":0.600876,"latency":3023.191},{"benchmarkId":"epoch-ai:epoch-simpleqa-verified:overall","benchmarkName":"SimpleQA Verified","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":41.4,"unit":"percent","family":"epoch-ai:epoch-simpleqa-verified:overall","index":58.408907029704814,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.5583544104177554,"costPerTest":null,"latency":null},{"benchmarkId":"omniscienceAccuracy","benchmarkName":"Artificial Analysis Omniscience Accuracy","category":"knowledge","score":39.6,"unit":"percent","family":"omniscienceAccuracy","index":60.59608743538222,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":38.5,"unit":"percent","family":"hle","index":63.01341499696056,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"bioBasicsBench","benchmarkName":"BioBasicsBench: everyday wet-lab science","category":"reasoning","score":37.56,"unit":"percent","family":"bioBasicsBench","index":null,"aboveFit":false,"run":null,"source":"OpenAI, Ligo","sourceUrl":"https://openai.com/index/measuring-how-ai-agents-handle-wet-lab-science/","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mka","benchmarkName":"Multimodal Knowledge Agent Benchmark","category":"agentic","score":36.8,"unit":"percent","family":"mka","index":66.55986726168156,"aboveFit":false,"run":null,"source":"Google DeepMind, in partnership with Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mka","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-chess-puzzles:overall","benchmarkName":"Chess Puzzles","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":36,"unit":"percent","family":"epoch-ai:epoch-chess-puzzles:overall","index":61.309222750542396,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":4.824181513244218,"costPerTest":null,"latency":null},{"benchmarkId":"hermesIndex","benchmarkName":"Hermes Index: agentic capability and cost per task inside Hermes Agent","category":"agentic","score":33.89,"unit":"percent","family":"hermesIndex","index":63.817128856813916,"aboveFit":false,"run":null,"source":"Nous Research","sourceUrl":"https://portal.nousresearch.com/bench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"healthBenchHard","benchmarkName":"HealthBench Hard","category":"knowledge","score":31.4,"unit":"percent","family":"healthBenchHard","index":70.82954011956754,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:legal_research:overall","benchmarkName":"Legal Research Bench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":30.288,"unit":"percent","family":"vals-ai:legal_research:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_research","measuredAt":"2026-11-09","stderr":3.194,"costPerTest":0.440296,"latency":2547.747},{"benchmarkId":"epoch-ai:epoch-mirrorcode:overall","benchmarkName":"MirrorCode","category":"coding","domain":null,"eligible":true,"ranked":true,"score":26.666666666666668,"unit":"percent","family":"epoch-ai:epoch-mirrorcode:overall","index":61.276876495258676,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":9.026709338484402,"costPerTest":null,"latency":null},{"benchmarkId":"aaMlcr","benchmarkName":"Medical Long Context Reasoning (MLCR-AA)","category":"reasoning","score":26.1,"unit":"percent","family":"aaMlcr","index":70.73007735968582,"aboveFit":false,"run":null,"source":"Wisedocs and Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mlcr-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaFrontierCode2","benchmarkName":"FrontierCode 2.0, Artificial Analysis evaluation","category":"coding","score":25.2,"unit":"percent","family":"aaFrontierCode2","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaDeepsecBench","benchmarkName":"Artificial Analysis DeepsecBench-AA","category":"agentic","score":23.5,"unit":"percent","family":"aaDeepsecBench","index":68.3613094470664,"aboveFit":false,"run":null,"source":"Artificial Analysis; dataset by Vercel","sourceUrl":"https://artificialanalysis.ai/evaluations/deepsecbench-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaGdpPdf","benchmarkName":"Artificial Analysis GDP.pdf","category":"agentic","score":22.8,"unit":"percent","family":"aaGdpPdf","index":69.55188743098628,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdp-pdf","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"frontierCode2","benchmarkName":"FrontierCode 2.0","category":"coding","score":21.9,"unit":"percent","family":"frontierCode2","index":68.26338430562275,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaAutomationBenchStrict","benchmarkName":"Artificial Analysis AutomationBench, strict pass@1","category":"agentic","score":20.2,"unit":"percent","family":"aaAutomationBenchStrict","index":null,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/automationbench-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:tax_agent_bench:overall","benchmarkName":"Tax Agent Bench","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":19.947,"unit":"percent","family":"vals-ai:tax_agent_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench","measuredAt":"2026-11-09","stderr":2.866,"costPerTest":0.190956,"latency":1995.736},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":19.4,"unit":"percent","family":"critpt","index":74.40525235673132,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:mysterymechanism:overall","benchmarkName":"MysteryMechanism","category":null,"domain":"Science","eligible":false,"ranked":true,"score":19.369,"unit":"percent","family":"vals-ai:mysterymechanism:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mysterymechanism","measuredAt":"2026-11-11","stderr":2.658,"costPerTest":0.025854,"latency":361.208},{"benchmarkId":"bug-hunt-bench","benchmarkName":"Bug Hunt Bench","category":"coding","score":18.3,"unit":"percent","family":"bug-hunt-bench","index":62.5052370062358,"aboveFit":false,"run":null,"source":"Pawel Huryn","sourceUrl":"https://bughunt.productcompass.pm/method","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"terminal-bench:terminal-bench-4-0-0:overall","benchmarkName":"Terminal-Bench 4.0.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":16.36,"unit":"percent","family":"terminal-bench-4","index":62.28437347586821,"aboveFit":false,"run":"max effort · Codex","source":"Terminal-Bench","sourceUrl":"https://www.tbench.ai/leaderboard","measuredAt":"2026-09-29","stderr":1.3877551020408165,"costPerTest":0.2186060606060606,"latency":2284.1},{"benchmarkId":"terminal-bench:terminal-bench-4-1-0:overall","benchmarkName":"Terminal-Bench 4.1.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":15.15,"unit":"percent","family":"terminal-bench:terminal-bench-4-1-0:overall","index":60.04562119126917,"aboveFit":false,"run":"max effort · Codex","source":"Terminal-Bench","sourceUrl":"https://www.tbench.ai/leaderboard","measuredAt":"2026-09-29","stderr":1.3571428571428572,"costPerTest":0.2186060606060606,"latency":2284.1},{"benchmarkId":"vals-ai:terminal-bench-4:overall","benchmarkName":"Terminal-Bench 4.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":13.636,"unit":"percent","family":"terminal-bench-4","index":60.990913043883246,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-4","measuredAt":"2026-11-09","stderr":1.515,"costPerTest":0.354726,"latency":1965.568},{"benchmarkId":"aaTerminalBench4","benchmarkName":"Artificial Analysis Terminal-Bench v4.0","category":"agentic","score":12.6,"unit":"percent","family":"aaTerminalBench4","index":56.82723118092374,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/terminalbench-v4-0","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"exploitGym","benchmarkName":"ExploitGym","category":"agentic","score":11.6,"unit":"percent","family":"exploitGym","index":66.02483478419867,"aboveFit":false,"run":null,"source":"Zhun Wang, Nico Schiller, Hongwei Li, Srijiith Sesha Narayana, Milad Nasr, Nicholas Carlini, Xiangyu Qi, Eric Wallace, Elie Bursztein, Luca Invernizzi, Kurt Thomas, Yan Shoshitaishvili, Wenbo Guo, Jingxuan He, Thorsten Holz, Dawn Song","sourceUrl":"https://arxiv.org/abs/2605.11086","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-mystery-game-puzzles:overall","benchmarkName":"Mystery Game Puzzles","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":7.000000000000001,"unit":"percent","family":"epoch-ai:epoch-mystery-game-puzzles:overall","index":44.1019024790064,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.564323999762428,"costPerTest":null,"latency":null},{"benchmarkId":"lmarena:agent-bash-recovery:overall","benchmarkName":"Agent Arena command recovery","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":5.20701982384284,"unit":"index","family":"agent-arena","index":70.81178791724699,"aboveFit":false,"run":"max effort","source":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset","measuredAt":"2026-10-08","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:terminal-bench-science:overall","benchmarkName":"Terminal-Bench-Science","category":null,"domain":"Science","eligible":false,"ranked":true,"score":4.286,"unit":"percent","family":"vals-ai:terminal-bench-science:overall","index":null,"aboveFit":false,"run":"Mini-SWE-agent","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-science","measuredAt":"2026-11-09","stderr":2.438,"costPerTest":0.240966,"latency":4559.47},{"benchmarkId":"vals-ai:hlab:overall","benchmarkName":"HLAB-Vals-1.1","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":4.167,"unit":"percent","family":"vals-ai:hlab:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/hlab","measuredAt":"2026-11-09","stderr":1.629,"costPerTest":0.298018,"latency":1305.488},{"benchmarkId":"lmarena:agent-steerability:overall","benchmarkName":"Agent Arena steerability","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":3.5476635651704944,"unit":"index","family":"agent-arena","index":69.4475369331345,"aboveFit":false,"run":"max effort","source":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset","measuredAt":"2026-11-11","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:srebench:overall","benchmarkName":"SRE Bench","category":null,"domain":"Other","eligible":false,"ranked":true,"score":2.672,"unit":"percent","family":"vals-ai:srebench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/srebench","measuredAt":"2026-11-05","stderr":0.998,"costPerTest":1.550056,"latency":4871.097},{"benchmarkId":"vals-ai:programbench:overall","benchmarkName":"ProgramBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":0.5,"unit":"percent","family":"vals-ai:programbench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/programbench","measuredAt":"2026-11-09","stderr":0.5,"costPerTest":0.180944,"latency":1734.863},{"benchmarkId":"arc-prize:v3-semi-private:overall","benchmarkName":"ARC-AGI-3 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":0.10360997760319558,"unit":"percent","family":"arc-prize:v3-semi-private:overall","index":65.34960739868455,"aboveFit":false,"run":"max effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mazeBench","benchmarkName":"MazeBench: visual spatial reasoning in a 3D open world","category":"agentic","score":0,"unit":"percent","family":"mazeBench","index":61.97866734834783,"aboveFit":false,"run":null,"source":"Jonathan Pappas and David Pappas","sourceUrl":"https://mazebench.com/blog?post=maze-bench-results","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"extractBenchPro","benchmarkName":"ExtractBench Pro: long-tail document extraction challenges (LlamaIndex)","category":"multimodal","score":0,"unit":"percent","family":"extractBenchPro","index":null,"aboveFit":false,"run":null,"source":"LlamaIndex","sourceUrl":"https://www.extractbench.ai/pro","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-spireclimb-slay-the-spire-2-ascension-10:overall","benchmarkName":"SpireClimb (Slay the Spire 2, Ascension 10)","category":null,"domain":"Other","eligible":false,"ranked":true,"score":0,"unit":"percent","family":"epoch-ai:epoch-spireclimb-slay-the-spire-2-ascension-10:overall","index":null,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-spireclimb-slay-the-spire-1-ascension-20-heart:overall","benchmarkName":"SpireClimb (Slay the Spire 1, Ascension 20 Heart)","category":null,"domain":"Other","eligible":false,"ranked":true,"score":0,"unit":"percent","family":"epoch-ai:epoch-spireclimb-slay-the-spire-1-ascension-20-heart:overall","index":null,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-spireclimb:overall","benchmarkName":"SpireClimb","category":null,"domain":"Other","eligible":false,"ranked":true,"score":0,"unit":"percent","family":"epoch-ai:epoch-spireclimb:overall","index":null,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0,"costPerTest":null,"latency":null},{"benchmarkId":"lmarena:agent-task-outcome:overall","benchmarkName":"Agent Arena task outcome","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":-0.3624921754413313,"unit":"index","family":"agent-arena","index":66.2327759140446,"aboveFit":false,"run":"max effort","source":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset","measuredAt":"2026-11-11","stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":0.09999999999999999,"outputPerMillion":0.5,"batch":{"inputPerMillion":0.049999999999999996,"outputPerMillion":0.25},"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":66,"timeToFirstToken":2.766,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":{"elo":1388.3808756273925,"eloUpper":1392.45296286774,"eloLower":1384.308788387045,"votes":107021,"rank":173,"rankUpper":null,"rankLower":null,"arenaName":"GPT-6 Luna-max","organization":"openai","updatedAt":"2026-11-09"},"quality":{"coveragePercent":100,"benchmarkCount":57,"categoryCount":8,"confidence":"high","state":"available"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:livebench","name":"LiveBench","url":"https://livebench.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:arc-prize-foundation","name":"ARC Prize Foundation","url":"https://arcprize.org","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:terminal-bench","name":"Terminal-Bench","url":"https://www.tbench.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:lmarena","name":"LMArena","url":"https://huggingface.co","license":"CC BY 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"deepseek-v4-pro-max","providerId":"deepseek","slug":"deepseek-v4-pro-0813","name":"DeepSeek V4 Pro 0813","provider":"DeepSeek","description":"DeepSeek V4 Pro 0813 is a reasoning model from DeepSeek in the DeepSeek V4 Pro family. 52 benchmarks count toward its index, in 6 categories.","logoUrl":null,"releaseDate":"2026-08-13","contextWindowTokens":1048576,"modalities":["text"],"openWeights":true,"details":{"apiId":"deepseek/deepseek-v4-pro-0813","maxOutputTokens":943718,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.09699999999999999,"weightsUrl":"https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro-0813"},"reasoning":true,"overallScore":65.47924526350428,"overallScoreRange":{"low":61.73686972026379,"high":69.22162080674477},"badges":[],"capabilityScores":{"agentic":63.66064001081888,"coding":65.09035355699729,"reasoning":68.4553544120958,"multimodal":null,"knowledge":64.99615804347317,"multilingual":null,"instruction":62.04355217736967,"math":64.05485519664022},"capabilityCounts":{"math":7,"coding":13,"agentic":17,"knowledge":6,"reasoning":7,"instruction":2},"benchmarks":[{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":98.61111111111111,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":66.29044816413295,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0.990980650017605,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:swebench:overall","benchmarkName":"SWE-bench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":96.4,"unit":"percent","family":"swe-bench-verified","index":72.32847802781956,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/swebench","measuredAt":"2026-09-01","stderr":0.834,"costPerTest":0.103104,"latency":240.413},{"benchmarkId":"tau2Bench","benchmarkName":"τ²-Bench Tool-Agent-User Evaluation","category":"agentic","score":96.2,"unit":"percent","family":"tau2-bench","index":66.62517351219736,"aboveFit":false,"run":null,"source":"Victor Barres, Honghua Dong, Soham Ray, Xujie Si, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2506.07982","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hmmtFeb2026","benchmarkName":"Harvard-MIT Mathematics Tournament February 2026","category":"math","score":95.2,"unit":"percent","family":"hmmtFeb2026","index":60.02346344621243,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-math:overall","benchmarkName":"LiveBench Mathematics","category":"math","domain":null,"eligible":true,"ranked":true,"score":95.08624999999999,"unit":"percent","family":"livebench:livebench-math:overall","index":72.29836542366449,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Mathematics","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.03136411609498681,"latency":null},{"benchmarkId":"liveCodeBenchPass1Cot","benchmarkName":"LiveCodeBench Pass@1 with Chain-of-Thought","category":"coding","score":93.5,"unit":"percent","family":"liveCodeBenchPass1Cot","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"arc-prize:v1-public-eval:overall","benchmarkName":"ARC-AGI-1 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":93.25,"unit":"percent","family":"arc-prize:v1-public-eval:overall","index":null,"aboveFit":false,"run":"max effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.23599243470000006,"latency":null},{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":92.8,"unit":"percent","family":"gpqa-diamond","index":62.322724970405574,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:gpqa:overall","benchmarkName":"GPQA Diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":92.424,"unit":"percent","family":"gpqa-diamond","index":61.97786804463575,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","measuredAt":"2026-09-01","stderr":2.02,"costPerTest":0.065819,"latency":279.412},{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":91.66666666666666,"unit":"percent","family":"gpqa-diamond","index":61.28326260549648,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.5242573629934337,"costPerTest":null,"latency":null},{"benchmarkId":"apexShortlist","benchmarkName":"Apex Shortlist","category":"math","score":90.2,"unit":"percent","family":"apexShortlist","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gpqa","benchmarkName":"Graduate-Level Google-Proof Q&A","category":"knowledge","score":90.1,"unit":"percent","family":"gpqa-diamond","index":59.84635874812216,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gpqaDiamond","benchmarkName":"GPQA Diamond","category":"knowledge","score":90.1,"unit":"percent","family":"gpqa-diamond","index":59.84635874812216,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"arc-prize:v1-semi-private:overall","benchmarkName":"ARC-AGI-1 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":90,"unit":"percent","family":"arc-prize:v1-semi-private:overall","index":68.36360812788647,"aboveFit":false,"run":"max effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.29617841879999995,"latency":null},{"benchmarkId":"imoAnswerBench","benchmarkName":"IMOAnswerBench","category":"math","score":89.8,"unit":"percent","family":"imoAnswerBench","index":61.44796858939903,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"terminalBench21","benchmarkName":"Terminal-Bench 2.1","category":"agentic","score":87.9,"unit":"percent","family":"terminal-bench-2-1","index":72.54990774452975,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:lcb:overall","benchmarkName":"LiveCodeBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":87.525,"unit":"percent","family":"livecodebench","index":61.966867571963526,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","measuredAt":"2026-09-01","stderr":0.958,"costPerTest":0.063876,"latency":273.233},{"benchmarkId":"mmluPro","benchmarkName":"Massive Multitask Language Understanding Professional","category":"knowledge","score":87.5,"unit":"percent","family":"mmlu-pro","index":54.720254946174194,"aboveFit":false,"run":null,"source":"Yubo Wang, Xueguang Ma, Ge Zhang, Yuansheng Ni, Abhranil Chandra, Shiguang Guo, Weiming Ren, Aaran Arulraj, Xuan He, Ziyan Jiang, Tianle Li, Max Ku, Kai Wang, Alex Zhuang, Rongqi Fan, Xiang Yue, Wenhu Chen","sourceUrl":"https://arxiv.org/abs/2406.01574","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:mmlu_pro:overall","benchmarkName":"MMLU Pro","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":86.972,"unit":"percent","family":"mmlu-pro","index":54.05503814614959,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","measuredAt":"2026-09-01","stderr":0.341,"costPerTest":0.01283,"latency":59.624},{"benchmarkId":"livebench:livebench-reasoning:overall","benchmarkName":"LiveBench Reasoning","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":85.84125,"unit":"percent","family":"livebench","index":72.821394857135,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Reasoning","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.03674207920792079,"latency":null},{"benchmarkId":"chineseSimpleQa","benchmarkName":"Chinese-SimpleQA","category":"knowledge","score":84.4,"unit":"percent","family":"chineseSimpleQa","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mrcr1m","benchmarkName":"MRCR 1M","category":"reasoning","score":83.5,"unit":"percent","family":"mrcr1m","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"browseComp","benchmarkName":"BrowseComp","category":"agentic","score":83.4,"unit":"percent","family":"browseComp","index":65.79664587094089,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"cyberGym","benchmarkName":"CyberGym","category":"agentic","score":83.3,"unit":"percent","family":"cyberGym","index":68.74643479502345,"aboveFit":false,"run":null,"source":"Zhun Wang, Tianneng Shi, Jingxuan He, Matthew Cai, Jialin Zhang, Dawn Song","sourceUrl":"https://www.cybergym.io/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:legal_bench:overall","benchmarkName":"LegalBench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":82.361,"unit":"percent","family":"vals-ai:legal_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","measuredAt":"2026-10-13","stderr":0.437,"costPerTest":0.004081,"latency":20.049},{"benchmarkId":"vals-ai:vibe-code:overall","benchmarkName":"Vibe Code Bench v1.1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":82.297,"unit":"percent","family":"vals-ai:vibe-code:overall","index":71.9435731738045,"aboveFit":false,"run":"OpenHands","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","measuredAt":"2026-11-09","stderr":3.148,"costPerTest":0.356023,"latency":3982.6},{"benchmarkId":"livebench:livebench-language:overall","benchmarkName":"LiveBench Language","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":82.074,"unit":"percent","family":"livebench:livebench-language:overall","index":67.36881030679505,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Language","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.021058,"latency":null},{"benchmarkId":"sweVerified","benchmarkName":"Software Engineering Benchmark Verified","category":"coding","score":80.6,"unit":"percent","family":"swe-bench-verified","index":60.67506829883721,"aboveFit":false,"run":null,"source":"Carlos E. Jimenez, John Yang, Alexander Wettig, Shunyu Yao, Kexin Pei, Ofir Press, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2310.06770","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":80.3,"unit":"percent","family":"lcr","index":62.39031566369921,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:medscribe:overall","benchmarkName":"MedScribe","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":80.174,"unit":"percent","family":"vals-ai:medscribe:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medscribe","measuredAt":"2026-10-29","stderr":2.004,"costPerTest":0.041127,"latency":155.859},{"benchmarkId":"livebench:livebench-data-analysis:overall","benchmarkName":"LiveBench Data Analysis","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":79.24333333333334,"unit":"percent","family":"livebench","index":64.27422930339304,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Data%20Analysis","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.05051933333333334,"latency":null},{"benchmarkId":"livebench:livebench-coding:overall","benchmarkName":"LiveBench Coding","category":"coding","domain":null,"eligible":true,"ranked":true,"score":77.1585,"unit":"percent","family":"livebench:livebench-coding:overall","index":63.340634835811045,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Coding","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.027605128205128206,"latency":null},{"benchmarkId":"aaIfBench","benchmarkName":"Artificial Analysis IFBench","category":"instruction","score":76.5,"unit":"percent","family":"ifbench-family","index":60.51369142652189,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"toolathlonVerified","benchmarkName":"Toolathlon-Verified","category":"agentic","score":74.1,"unit":"percent","family":"toolathlonVerified","index":68.74724142498236,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mcpAtlas","benchmarkName":"MCP Atlas","category":"agentic","score":73.6,"unit":"percent","family":"mcpAtlas","index":60.624199182492134,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:tax_eval_v2:overall","benchmarkName":"TaxEval v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":73.058,"unit":"percent","family":"vals-ai:tax_eval_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","measuredAt":"2026-09-01","stderr":0.872,"costPerTest":0.045816,"latency":226.296},{"benchmarkId":"vals-ai:cyber:overall","benchmarkName":"CyberBench v1.1","category":null,"domain":"Other","eligible":false,"ranked":true,"score":72.857,"unit":"percent","family":"vals-ai:cyber:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/cyber","measuredAt":"2026-11-05","stderr":5.5,"costPerTest":0.593028,"latency":1353.188},{"benchmarkId":"dsBenchFullStack","benchmarkName":"DeepSeek DSBench FullStack","category":"coding","score":71.1,"unit":"percent","family":"dsBenchFullStack","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"terminalBench2","benchmarkName":"Terminal-Bench 2.0","category":"agentic","score":67.9,"unit":"percent","family":"terminalBench2","index":62.197997758572654,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-instruction-following:overall","benchmarkName":"LiveBench Instruction Following","category":"instruction","domain":null,"eligible":true,"ranked":true,"score":67.69999999999999,"unit":"percent","family":"livebench:livebench-instruction-following:overall","index":63.57341292821745,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=IF","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.020699000000000002,"latency":null},{"benchmarkId":"finch","benchmarkName":"Finch (FinWorkBench): Finance & Accounting across Spreadsheet-Centric Enterprise Workflows","category":"agentic","score":67.4,"unit":"percent","family":"finch","index":null,"aboveFit":false,"run":null,"source":"Finch authors (ACL 2026 Findings)","sourceUrl":"https://arxiv.org/abs/2512.13168","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"dsBenchHard","benchmarkName":"DeepSeek DSBench Hard","category":"coding","score":67.2,"unit":"percent","family":"dsBenchHard","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:corp_fin_v2:overall","benchmarkName":"CorpFin v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":65.423,"unit":"percent","family":"vals-ai:corp_fin_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","measuredAt":"2026-08-12","stderr":0.937,"costPerTest":0.002183,"latency":32.854},{"benchmarkId":"epoch-ai:epoch-frontiermath-tiers-1-3-v2-private:overall","benchmarkName":"FrontierMath-Tiers-1-3-v2-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":64.56140350877193,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tiers-1-3-v2-private:overall","index":64.87670152601203,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.8383475205435627,"costPerTest":null,"latency":null},{"benchmarkId":"deepSwe","benchmarkName":"DeepSWE","category":"agentic","score":62.7,"unit":"percent","family":"deepSwe","index":66.65787029928713,"aboveFit":false,"run":null,"source":"Datacurve AI","sourceUrl":"https://deepswe.datacurve.ai/blog","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"corpusQa1m","benchmarkName":"CorpusQA 1M","category":"reasoning","score":62,"unit":"percent","family":"corpusQa1m","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"nl2Repo","benchmarkName":"NL2Repo","category":"coding","score":61.5,"unit":"percent","family":"nl2Repo","index":70.14270008412257,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"arc-prize:v2-semi-private:overall","benchmarkName":"ARC-AGI-2 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":61.25000000000001,"unit":"percent","family":"arc-prize:v2-semi-private:overall","index":67.31051583703211,"aboveFit":false,"run":"max effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.5976109150000003,"latency":null},{"benchmarkId":"hleWithTools","benchmarkName":"Humanity's Last Exam with tools","category":"agentic","score":60,"unit":"percent","family":"hleWithTools","index":68.56235210405296,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"arc-prize:v2-public-eval:overall","benchmarkName":"ARC-AGI-2 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":59.44444444444444,"unit":"percent","family":"arc-prize:v2-public-eval:overall","index":null,"aboveFit":false,"run":"max effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.6375456450000001,"latency":null},{"benchmarkId":"openHarmonyBench","benchmarkName":"OpenHarmony Bench v1.0","category":"coding","score":59,"unit":"percent","family":"openHarmonyBench","index":65.77003215245863,"aboveFit":false,"run":null,"source":"OpenHarmony Bench authors","sourceUrl":"https://arxiv.org/abs/2608.16022","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"swePro","benchmarkName":"SWE-bench Pro","category":"coding","score":55.4,"unit":"percent","family":"swePro","index":56.01857663172778,"aboveFit":false,"run":null,"source":"Xiang Deng, Jeff Da, Edwin Pan, Yannis Yiming He, Charles Ide, Kanak Garg, Niklas Lauffer, Andrew Park, Nitin Pasari, Chetan Rane, Karmini Sampath, Maya Krishnan, Srivatsa Kundurthy, Sean Hendryx, Zifan Wang, Chen Bo Calvin Zhang, Noah Jacobson, Bing Liu, Brad Kenstler","sourceUrl":"https://arxiv.org/abs/2509.16941","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-agentic-coding:overall","benchmarkName":"LiveBench Agentic Coding","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":54.949666666666666,"unit":"percent","family":"livebench:livebench-agentic-coding:overall","index":65.83134571079079,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Agentic%20Coding","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.08431944444444445,"latency":null},{"benchmarkId":"vals-ai:terminal-bench-2-1:overall","benchmarkName":"Terminal-Bench 2.1","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":54.682,"unit":"percent","family":"terminal-bench-2-1","index":53.122182247466384,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2-1","measuredAt":"2026-10-01","stderr":1.498,"costPerTest":0.199883,"latency":862.374},{"benchmarkId":"gdpvalAaNormalized","benchmarkName":"GDPval-AA normalized","category":"agentic","score":54.5,"unit":"percent","family":"gdpvalAaNormalized","index":73.00000842939764,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:skillsbench:overall","benchmarkName":"SkillsBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":53.829,"unit":"percent","family":"vals-ai:skillsbench:overall","index":65.20852376025638,"aboveFit":false,"run":"OpenHands","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/skillsbench","measuredAt":"2026-10-29","stderr":4.49,"costPerTest":0.374596,"latency":674.652},{"benchmarkId":"epoch-ai:epoch-simpleqa-verified:overall","benchmarkName":"SimpleQA Verified","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":52.90581162324649,"unit":"percent","family":"epoch-ai:epoch-simpleqa-verified:overall","index":67.99426382562746,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.5808394665706598,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:emb:overall","benchmarkName":"EMB","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":52.803,"unit":"percent","family":"vals-ai:emb:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/emb","measuredAt":"2026-10-20","stderr":3.059,"costPerTest":1.08141,"latency":2247.236},{"benchmarkId":"toolathlon","benchmarkName":"Toolathlon","category":"agentic","score":51.8,"unit":"percent","family":"toolathlon","index":64.11811930461617,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:ioi:overall","benchmarkName":"IOI","category":"coding","domain":null,"eligible":true,"ranked":true,"score":51.611,"unit":"percent","family":"vals-ai:ioi:overall","index":65.49255970090978,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/ioi","measuredAt":"2026-10-29","stderr":2.4,"costPerTest":2.151589,"latency":4045.972},{"benchmarkId":"aaSciCode","benchmarkName":"Artificial Analysis SciCode","category":"coding","score":51,"unit":"percent","family":"scicode-family","index":62.32804690966265,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/scicode","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:fabv2:overall","benchmarkName":"Finance Agent (v2)","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":50.393,"unit":"percent","family":"vals-ai:fabv2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/fabv2","measuredAt":"2026-10-29","stderr":0.249,"costPerTest":0.880465,"latency":974.518},{"benchmarkId":"vals-ai:proof_bench:overall","benchmarkName":"ProofBench v1.1","category":"math","domain":null,"eligible":true,"ranked":true,"score":50,"unit":"percent","family":"vals-ai:proof_bench:overall","index":64.41885564025556,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/proof_bench","measuredAt":"2026-11-09","stderr":5.025,"costPerTest":0.262008,"latency":732.365},{"benchmarkId":"aaEnterpriseOpsGym","benchmarkName":"Artificial Analysis EnterpriseOps-Gym","category":"agentic","score":49.6,"unit":"percent","family":"aaEnterpriseOpsGym","index":null,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/enterprise-ops-gym-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"omniscienceAccuracy","benchmarkName":"Artificial Analysis Omniscience Accuracy","category":"knowledge","score":49.1,"unit":"percent","family":"omniscienceAccuracy","index":72.56851928394782,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-chess-puzzles:overall","benchmarkName":"Chess Puzzles","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":46,"unit":"percent","family":"epoch-ai:epoch-chess-puzzles:overall","index":68.89937470204336,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":5.009082659620331,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-mystery-game-puzzles:overall","benchmarkName":"Mystery Game Puzzles","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":43,"unit":"percent","family":"epoch-ai:epoch-mystery-game-puzzles:overall","index":71.98083876039257,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":4.97569851956243,"costPerTest":null,"latency":null},{"benchmarkId":"hle","benchmarkName":"Humanity's Last Exam","category":"knowledge","score":42.7,"unit":"percent","family":"hle","index":67.2272445260857,"aboveFit":false,"run":null,"source":"Center for AI Safety, Scale AI, and thousands of expert contributors","sourceUrl":"https://lastexam.ai/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:medcode:overall","benchmarkName":"MedCode","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":42.47,"unit":"percent","family":"vals-ai:medcode:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medcode","measuredAt":"2026-10-29","stderr":2.16,"costPerTest":0.06106,"latency":180.938},{"benchmarkId":"vals-ai:code-migration:overall","benchmarkName":"Code Migration","category":"coding","domain":null,"eligible":true,"ranked":true,"score":41.54,"unit":"percent","family":"vals-ai:code-migration:overall","index":67.25685093699137,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/code-migration","measuredAt":"2026-10-29","stderr":4.301,"costPerTest":18.581309,"latency":12726.884},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":41,"unit":"percent","family":"hle","index":65.52164685953505,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:legal_research:overall","benchmarkName":"Legal Research Bench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":40.865,"unit":"percent","family":"vals-ai:legal_research:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_research","measuredAt":"2026-11-09","stderr":3.417,"costPerTest":1.088017,"latency":2527.177},{"benchmarkId":"aaTau3Banking","benchmarkName":"Artificial Analysis Tau3-Banking","category":"agentic","score":39.6,"unit":"percent","family":"aaTau3Banking","index":63.51087786376911,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/tau3-banking","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"apex","benchmarkName":"Apex","category":"math","score":38.3,"unit":"percent","family":"apex","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-mirrorcode:overall","benchmarkName":"MirrorCode","category":"coding","domain":null,"eligible":true,"ranked":true,"score":33.33333333333333,"unit":"percent","family":"epoch-ai:epoch-mirrorcode:overall","index":64.20468425399986,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":10.026420125303973,"costPerTest":null,"latency":null},{"benchmarkId":"automationBench","benchmarkName":"AutomationBench","category":"agentic","score":31.8,"unit":"percent","family":"automationBench","index":42.57709605436705,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tier-4-v2-private:overall","benchmarkName":"FrontierMath-Tier-4-v2-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":26.82926829268293,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tier-4-v2-private:overall","index":59.02818358680503,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":7.005564203095157,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:tax_agent_bench:overall","benchmarkName":"Tax Agent Bench","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":26.756,"unit":"percent","family":"vals-ai:tax_agent_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench","measuredAt":"2026-11-09","stderr":3.093,"costPerTest":0.729136,"latency":1596.755},{"benchmarkId":"agentsLastExam","benchmarkName":"Agents' Last Exam","category":"agentic","score":25.7,"unit":"percent","family":"agentsLastExam","index":62.39374747870609,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"apexAgentsAa","benchmarkName":"APEX-Agents-AA","category":"agentic","score":24.3,"unit":"percent","family":"apexAgentsAa","index":58.77501884597738,"aboveFit":false,"run":null,"source":"Artificial Analysis / Mercor","sourceUrl":"https://artificialanalysis.ai/evaluations/apex-agents-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":18,"unit":"percent","family":"critpt","index":71.69501571335289,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:vcb-1-100:overall","benchmarkName":"Vibe Code Bench 1-100","category":"coding","domain":null,"eligible":true,"ranked":true,"score":17.533,"unit":"percent","family":"vals-ai:vcb-1-100:overall","index":65.99977306592824,"aboveFit":false,"run":"OpenHands","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vcb-1-100","measuredAt":"2026-10-29","stderr":3.414,"costPerTest":5.892792,"latency":2262.422},{"benchmarkId":"vals-ai:terminal-bench-4:overall","benchmarkName":"Terminal-Bench 4.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":14.141,"unit":"percent","family":"terminal-bench-4","index":61.23070655274977,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-4","measuredAt":"2026-11-09","stderr":2.02,"costPerTest":3.311817,"latency":5493.603},{"benchmarkId":"vals-ai:hlab:overall","benchmarkName":"HLAB-Vals-1.1","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":10.833,"unit":"percent","family":"vals-ai:hlab:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/hlab","measuredAt":"2026-11-09","stderr":2.723,"costPerTest":0.802111,"latency":655.413},{"benchmarkId":"vals-ai:terminal-bench-science:overall","benchmarkName":"Terminal-Bench-Science","category":null,"domain":"Science","eligible":false,"ranked":true,"score":4.286,"unit":"percent","family":"vals-ai:terminal-bench-science:overall","index":null,"aboveFit":false,"run":"Mini-SWE-agent","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-science","measuredAt":"2026-11-09","stderr":2.438,"costPerTest":4.630215,"latency":8181.756},{"benchmarkId":"vals-ai:srebench:overall","benchmarkName":"SRE Bench","category":null,"domain":"Other","eligible":false,"ranked":true,"score":1.908,"unit":"percent","family":"vals-ai:srebench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/srebench","measuredAt":"2026-11-05","stderr":0.847,"costPerTest":5.579756,"latency":8425.728},{"benchmarkId":"vals-ai:programbench:overall","benchmarkName":"ProgramBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":0,"unit":"percent","family":"vals-ai:programbench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/programbench","measuredAt":"2026-11-09","stderr":0,"costPerTest":3.412918,"latency":4970.276}],"price":{"inputPerMillion":0.111,"outputPerMillion":4.4399999999999995,"batch":{"inputPerMillion":0.66,"outputPerMillion":1.9800000000000002},"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":42,"timeToFirstToken":0.374,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":85,"benchmarkCount":52,"categoryCount":6,"confidence":"medium","state":"available"},"hosts":[{"host":"deepseek","name":"DeepSeek","kind":"maker","url":"https://api-docs.deepseek.com/quick_start/pricing","inputPerMillion":0.66,"outputPerMillion":1.98,"cacheReadPerMillion":0.022,"contextTokens":1048576,"maxOutputTokens":393216,"deprecated":false},{"host":"deepinfra","name":"Deep Infra","kind":"host","url":"https://deepinfra.com/models","inputPerMillion":1.3,"outputPerMillion":2.6,"cacheReadPerMillion":0.1,"contextTokens":1048576,"maxOutputTokens":384000,"deprecated":false},{"host":"coreweave","name":"CoreWeave","kind":"host","url":"https://docs.wandb.ai/inference","inputPerMillion":1.31,"outputPerMillion":3.96,"cacheReadPerMillion":0.044,"contextTokens":1048576,"maxOutputTokens":1048576,"deprecated":false},{"host":"arcee-ai","name":"Arcee AI","kind":"host","url":"https://docs.arcee.ai","inputPerMillion":1.32,"outputPerMillion":3.96,"cacheReadPerMillion":0.044,"contextTokens":1048576,"maxOutputTokens":384000,"deprecated":false},{"host":"baseten","name":"Baseten","kind":"host","url":"https://docs.baseten.co/inference/model-apis/overview","inputPerMillion":1.32,"outputPerMillion":3.96,"cacheReadPerMillion":0.132,"contextTokens":1048576,"maxOutputTokens":262144,"deprecated":false},{"host":"cloudflare","name":"Cloudflare Workers AI","kind":"cloud","url":"https://developers.cloudflare.com/workers-ai/models/","inputPerMillion":1.32,"outputPerMillion":3.96,"cacheReadPerMillion":0.044,"contextTokens":1048576,"maxOutputTokens":1048576,"deprecated":true},{"host":"digitalocean","name":"DigitalOcean","kind":"cloud","url":"https://docs.digitalocean.com/products/gradient-ai-platform/details/models/","inputPerMillion":1.32,"outputPerMillion":3.96,"cacheReadPerMillion":0.044,"contextTokens":1048576,"maxOutputTokens":1048576,"deprecated":false},{"host":"nebius","name":"Nebius","kind":"host","url":"https://docs.tokenfactory.nebius.com/","inputPerMillion":1.32,"outputPerMillion":3.96,"cacheReadPerMillion":0.04,"contextTokens":979000,"maxOutputTokens":979000,"deprecated":false},{"host":"novita-ai","name":"Novita AI","kind":"host","url":"https://novita.ai/docs/guides/introduction","inputPerMillion":1.32,"outputPerMillion":3.96,"cacheReadPerMillion":0.044,"contextTokens":1048576,"maxOutputTokens":393216,"deprecated":false},{"host":"siliconflow","name":"SiliconFlow","kind":"host","url":"https://cloud.siliconflow.com/models","inputPerMillion":1.32,"outputPerMillion":3.96,"cacheReadPerMillion":0.044,"contextTokens":1000000,"maxOutputTokens":384000,"deprecated":false},{"host":"together-ai","name":"Together AI","kind":"host","url":"https://docs.together.ai/docs/serverless-models","inputPerMillion":1.32,"outputPerMillion":3.96,"cacheReadPerMillion":0.13,"contextTokens":1048576,"maxOutputTokens":384000,"deprecated":false},{"host":"volcengine","name":"Volcengine Ark","kind":"cloud","url":"https://www.volcengine.com/docs/82379/1330310","inputPerMillion":1.3359,"outputPerMillion":4.00771,"cacheReadPerMillion":0.04453,"contextTokens":1000000,"maxOutputTokens":384000,"deprecated":false},{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":0.111,"outputPerMillion":4.44,"cacheReadPerMillion":0.097,"contextTokens":1048576,"maxOutputTokens":943718,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:livebench","name":"LiveBench","url":"https://livebench.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:arc-prize-foundation","name":"ARC Prize Foundation","url":"https://arcprize.org","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gpt-5-6-luna","providerId":"openai","slug":"gpt-5-6-luna","name":"GPT-5.6 Luna","provider":"OpenAI","description":"GPT-5.6 Luna is a reasoning model from OpenAI in the GPT-5.6 family. 61 benchmarks count toward its index, in 8 categories.","logoUrl":null,"releaseDate":"2026-07-09","contextWindowTokens":1050000,"modalities":["file","image","text"],"openWeights":false,"details":{"apiId":"openai/gpt-5.6-luna","maxOutputTokens":128000,"knowledgeCutoff":"2026-02-16","retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.02,"weightsUrl":null},"reasoning":true,"overallScore":65.45297687898093,"overallScoreRange":{"low":63.382620530466006,"high":67.52333322749587},"badges":[],"capabilityScores":{"agentic":64.69446251156987,"coding":67.40815025973292,"reasoning":66.27571500689847,"multimodal":65.8299561926726,"knowledge":62.117753530821126,"multilingual":53.13742736467199,"instruction":48.23761056636331,"math":68.37237010448362},"capabilityCounts":{"math":5,"coding":15,"knowledge":8,"multimodal":3,"reasoning":9,"agentic":19,"multilingual":1,"instruction":1},"benchmarks":[{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":98.33333333333333,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":66.12883870698487,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.1650421590325524,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:swebench:overall","benchmarkName":"SWE-bench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":93,"unit":"percent","family":"swe-bench-verified","index":69.820782263355,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/swebench","measuredAt":"2026-09-01","stderr":1.142,"costPerTest":0.042712,"latency":201.075},{"benchmarkId":"gpqa","benchmarkName":"Graduate-Level Google-Proof Q&A","category":"knowledge","score":92.3,"unit":"percent","family":"gpqa-diamond","index":61.86413863294568,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gpqaDiamond","benchmarkName":"GPQA Diamond","category":"knowledge","score":92.3,"unit":"percent","family":"gpqa-diamond","index":61.86413863294568,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:gpqa:overall","benchmarkName":"GPQA Diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":91.666,"unit":"percent","family":"gpqa-diamond","index":61.282651157046544,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","measuredAt":"2026-09-01","stderr":1.736,"costPerTest":0.010616,"latency":53.521},{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":91.60353535353535,"unit":"percent","family":"gpqa-diamond","index":61.225360290160644,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.728284355494035,"costPerTest":null,"latency":null},{"benchmarkId":"extractBenchLlamaIndex","benchmarkName":"ExtractBench: schema-guided enterprise document extraction (LlamaIndex leaderboard)","category":"multimodal","score":91.16,"unit":"percent","family":"extractBenchLlamaIndex","index":73.0406526875836,"aboveFit":false,"run":null,"source":"Boyang Zhang, Adrian Lyjak, Eli Stewart, Zhaoqi Li and Simon Suo (LlamaIndex)","sourceUrl":"https://arxiv.org/abs/2607.29677","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":91.1,"unit":"percent","family":"gpqa-diamond","index":60.76353142304194,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"arc-prize:v1-public-eval:overall","benchmarkName":"ARC-AGI-1 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":90.75,"unit":"percent","family":"arc-prize:v1-public-eval:overall","index":null,"aboveFit":false,"run":"max effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.18,"latency":null},{"benchmarkId":"arc-prize:v1-semi-private:overall","benchmarkName":"ARC-AGI-1 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":88,"unit":"percent","family":"arc-prize:v1-semi-private:overall","index":67.44413592193767,"aboveFit":false,"run":"max effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.32,"latency":null},{"benchmarkId":"livebench:livebench-math:overall","benchmarkName":"LiveBench Mathematics","category":"math","domain":null,"eligible":true,"ranked":true,"score":87.20075,"unit":"percent","family":"livebench:livebench-math:overall","index":61.308191934820755,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Mathematics","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.06186437994722955,"latency":null},{"benchmarkId":"vals-ai:mmlu_pro:overall","benchmarkName":"MMLU Pro","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":86.036,"unit":"percent","family":"mmlu-pro","index":52.87579018246965,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","measuredAt":"2026-09-01","stderr":0.35,"costPerTest":0.002842,"latency":16.948},{"benchmarkId":"livebench:livebench-reasoning:overall","benchmarkName":"LiveBench Reasoning","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":85.64425,"unit":"percent","family":"livebench","index":72.56619435989413,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Reasoning","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.13845148514851485,"latency":null},{"benchmarkId":"vulcanBench","benchmarkName":"VulcanBench v3","category":"coding","score":85.5,"unit":"percent","family":"vulcanBench","index":66.40963394161543,"aboveFit":false,"run":null,"source":"VulcanBench contributors","sourceUrl":"https://github.com/morganlinton/VulcanBench/tree/main","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:mmmu:overall","benchmarkName":"MMMU Pro","category":"multimodal","domain":null,"eligible":true,"ranked":true,"score":85.029,"unit":"percent","family":"mmmu-pro","index":65.81059851865317,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmmu","measuredAt":"2026-09-01","stderr":0.858,"costPerTest":0.006419,"latency":35.102},{"benchmarkId":"terminalBench21","benchmarkName":"Terminal-Bench 2.1","category":"agentic","score":84.7,"unit":"percent","family":"terminal-bench-2-1","index":70.67837057821622,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:medscribe:overall","benchmarkName":"MedScribe","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":84.391,"unit":"percent","family":"vals-ai:medscribe:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medscribe","measuredAt":"2026-10-29","stderr":2.585,"costPerTest":0.022813,"latency":116.886},{"benchmarkId":"vals-ai:legal_bench:overall","benchmarkName":"LegalBench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":84.034,"unit":"percent","family":"vals-ai:legal_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","measuredAt":"2026-10-13","stderr":0.421,"costPerTest":0.000942,"latency":5.886},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":83.7,"unit":"percent","family":"lcr","index":64.72298945069046,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"browseComp","benchmarkName":"BrowseComp","category":"agentic","score":83.3,"unit":"percent","family":"browseComp","index":65.71764389034095,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-coding:overall","benchmarkName":"LiveBench Coding","category":"coding","domain":null,"eligible":true,"ranked":true,"score":82.91499999999999,"unit":"percent","family":"livebench:livebench-coding:overall","index":71.43650504041753,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Coding","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.052017948717948716,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tiers-1-3-v2-private:overall","benchmarkName":"FrontierMath-Tiers-1-3-v2-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":82.10526315789474,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tiers-1-3-v2-private:overall","index":74.11094207511238,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.274515950332741,"costPerTest":null,"latency":null},{"benchmarkId":"aaTerminalBench21","benchmarkName":"Artificial Analysis Terminal-Bench v2.1","category":"agentic","score":80.9,"unit":"percent","family":"aaTerminalBench21","index":63.44712716964386,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/terminalbench-v2-1","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmmuProPython","benchmarkName":"MMMU-Pro with Python","category":"multimodal","score":79.5,"unit":"percent","family":"mmmuProPython","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:terminal-bench-2-1:overall","benchmarkName":"Terminal-Bench 2.1","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":79.026,"unit":"percent","family":"terminal-bench-2-1","index":67.35990124019654,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2-1","measuredAt":"2026-10-01","stderr":0.991,"costPerTest":0.053767,"latency":355.751},{"benchmarkId":"aaMmmuPro","benchmarkName":"Artificial Analysis MMMU-Pro","category":"multimodal","score":78.6,"unit":"percent","family":"mmmu-pro","index":57.44381906620205,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmmu-pro","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmmuPro","benchmarkName":"Massive Multi-discipline Multimodal Understanding Pro","category":"multimodal","score":78.4,"unit":"percent","family":"mmmu-pro","index":57.183536613178234,"aboveFit":false,"run":null,"source":"MMMU-Pro authors","sourceUrl":"https://arxiv.org/abs/2409.02813","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-data-analysis:overall","benchmarkName":"LiveBench Data Analysis","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":78.03333333333333,"unit":"percent","family":"livebench","index":62.706754168055696,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Data%20Analysis","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.13372866666666666,"latency":null},{"benchmarkId":"cyberGym","benchmarkName":"CyberGym","category":"agentic","score":77.9,"unit":"percent","family":"cyberGym","index":65.40805863226691,"aboveFit":false,"run":null,"source":"Zhun Wang, Tianneng Shi, Jingxuan He, Matthew Cai, Jialin Zhang, Dawn Song","sourceUrl":"https://www.cybergym.io/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:vibe-code:overall","benchmarkName":"Vibe Code Bench v1.1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":77.055,"unit":"percent","family":"vals-ai:vibe-code:overall","index":70.05217721580769,"aboveFit":false,"run":"OpenHands","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","measuredAt":"2026-11-09","stderr":3.069,"costPerTest":0.726426,"latency":1551.831},{"benchmarkId":"vals-ai:tax_eval_v2:overall","benchmarkName":"TaxEval v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":76.166,"unit":"percent","family":"vals-ai:tax_eval_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","measuredAt":"2026-09-01","stderr":0.839,"costPerTest":0.012217,"latency":76.52},{"benchmarkId":"livebench:livebench-design:overall","benchmarkName":"LiveBench Design","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":76,"unit":"percent","family":"livebench-2-agentic","index":74.17063479632493,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Design","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.08954000000000001,"latency":null},{"benchmarkId":"vals-ai:cyber:overall","benchmarkName":"CyberBench v1.1","category":null,"domain":"Other","eligible":false,"ranked":true,"score":73.631,"unit":"percent","family":"vals-ai:cyber:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/cyber","measuredAt":"2026-11-05","stderr":5.564,"costPerTest":0.404245,"latency":975.203},{"benchmarkId":"livebench:livebench-coding-2:overall","benchmarkName":"LiveBench Coding 2.0","category":"coding","domain":null,"eligible":true,"ranked":true,"score":73.6,"unit":"percent","family":"livebench:livebench-coding-2:overall","index":61.48224447251978,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Coding","measuredAt":"2026-10-14","stderr":null,"costPerTest":1.8232034482758621,"latency":null},{"benchmarkId":"vals-ai:ioi-v1:overall","benchmarkName":"IOI v1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":72.917,"unit":"percent","family":"vals-ai:ioi-v1:overall","index":76.42075871465977,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/ioi-v1","measuredAt":"2026-08-09","stderr":5.083,"costPerTest":0.262867,"latency":1670.558},{"benchmarkId":"livebench:livebench-language:overall","benchmarkName":"LiveBench Language","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":72.567,"unit":"percent","family":"livebench:livebench-language:overall","index":56.61820308263329,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Language","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.09699733333333334,"latency":null},{"benchmarkId":"euroeval:euroeval-french:overall","benchmarkName":"EuroEval French","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":68.94125,"unit":"percent","family":"euroeval","index":65.75764877881375,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/french","measuredAt":null,"stderr":0.43537519738726504,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-browser:overall","benchmarkName":"LiveBench Browser Use","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":68,"unit":"percent","family":"livebench-2-agentic","index":68.31998603696127,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Browser","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.09734400000000001,"latency":null},{"benchmarkId":"vals-ai:mortgage_tax:overall","benchmarkName":"MortgageTax","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":67.29,"unit":"percent","family":"vals-ai:mortgage_tax:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mortgage_tax","measuredAt":"2026-09-01","stderr":0.921,"costPerTest":0.004804,"latency":22.835},{"benchmarkId":"deepSwe","benchmarkName":"DeepSWE","category":"agentic","score":67.2,"unit":"percent","family":"deepSwe","index":69.92924467725992,"aboveFit":false,"run":null,"source":"Datacurve AI","sourceUrl":"https://deepswe.datacurve.ai/blog","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:emb:overall","benchmarkName":"EMB","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":67.116,"unit":"percent","family":"vals-ai:emb:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/emb","measuredAt":"2026-10-20","stderr":2.936,"costPerTest":0.366502,"latency":660.047},{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":66.91874999999999,"unit":"percent","family":"euroeval","index":63.01358164280567,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":0.47320053888388586,"costPerTest":null,"latency":null},{"benchmarkId":"parseBenchLlamaIndex","benchmarkName":"ParseBench: document parsing for AI agents (LlamaIndex leaderboard)","category":"multimodal","score":66.86,"unit":"percent","family":"parseBenchLlamaIndex","index":67.00539682423214,"aboveFit":false,"run":null,"source":"Boyang Zhang, Sebastián G. Acosta, Preston Carlson, Sacha Bron, Pierre-Loïc Doulcet, Daniel B. Ospina and Simon Suo (LlamaIndex)","sourceUrl":"https://arxiv.org/abs/2604.08538","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:corp_fin_v2:overall","benchmarkName":"CorpFin v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":64.219,"unit":"percent","family":"vals-ai:corp_fin_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","measuredAt":"2026-08-12","stderr":0.945,"costPerTest":0.019039,"latency":16.968},{"benchmarkId":"swePro","benchmarkName":"SWE-bench Pro","category":"coding","score":62.7,"unit":"percent","family":"swePro","index":62.235329494400055,"aboveFit":false,"run":null,"source":"Xiang Deng, Jeff Da, Edwin Pan, Yannis Yiming He, Charles Ide, Kanak Garg, Niklas Lauffer, Andrew Park, Nitin Pasari, Chetan Rane, Karmini Sampath, Maya Krishnan, Srivatsa Kundurthy, Sean Hendryx, Zifan Wang, Chen Bo Calvin Zhang, Noah Jacobson, Bing Liu, Brad Kenstler","sourceUrl":"https://arxiv.org/abs/2509.16941","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:ioi:overall","benchmarkName":"IOI","category":"coding","domain":null,"eligible":true,"ranked":true,"score":61.778,"unit":"percent","family":"vals-ai:ioi:overall","index":69.30947924983259,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/ioi","measuredAt":"2026-10-29","stderr":11.613,"costPerTest":0.57961,"latency":4175.412},{"benchmarkId":"vals-ai:biomysterybench:overall","benchmarkName":"BioMysteryBench","category":null,"domain":"Science","eligible":false,"ranked":true,"score":61.481,"unit":"percent","family":"vals-ai:biomysterybench:overall","index":null,"aboveFit":false,"run":"Terminus 2","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/biomysterybench","measuredAt":"2026-11-09","stderr":0.741,"costPerTest":0.088859,"latency":1111.274},{"benchmarkId":"vals-ai:public-benefits-bench:overall","benchmarkName":"Public Benefits Bench v1.1","category":null,"domain":"Public benefits","eligible":false,"ranked":true,"score":61.164,"unit":"percent","family":"vals-ai:public-benefits-bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/public-benefits-bench","measuredAt":"2026-10-12","stderr":1.268,"costPerTest":0.270182,"latency":2319.748},{"benchmarkId":"cursorBench32","benchmarkName":"cursorBench32","category":"coding","score":61.1,"unit":"percent","family":"cursorBench32","index":66.04018758333716,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tier-4-v2-private:overall","benchmarkName":"FrontierMath-Tier-4-v2-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":60.97560975609756,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tier-4-v2-private:overall","index":72.44257515532887,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":7.712872341874097,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-portuguese:overall","benchmarkName":"EuroEval Portuguese","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":60.746249999999996,"unit":"percent","family":"euroeval","index":54.638919394593174,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/portuguese","measuredAt":null,"stderr":0.4264845395791036,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:skillsbench:overall","benchmarkName":"SkillsBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":60.447,"unit":"percent","family":"vals-ai:skillsbench:overall","index":70.13411696565448,"aboveFit":false,"run":"OpenHands","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/skillsbench","measuredAt":"2026-10-29","stderr":4.75,"costPerTest":0.206587,"latency":320.466},{"benchmarkId":"arc-prize:v2-public-eval:overall","benchmarkName":"ARC-AGI-2 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":60.17,"unit":"percent","family":"arc-prize:v2-public-eval:overall","index":null,"aboveFit":false,"run":"max effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.69,"latency":null},{"benchmarkId":"livebench:livebench-instruction-following:overall","benchmarkName":"LiveBench Instruction Following","category":"instruction","domain":null,"eligible":true,"ranked":true,"score":60.120999999999995,"unit":"percent","family":"livebench:livebench-instruction-following:overall","index":48.23761056636331,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=IF","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.153001,"latency":null},{"benchmarkId":"euroeval:euroeval-italian:overall","benchmarkName":"EuroEval Italian","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":60.002500000000005,"unit":"percent","family":"euroeval","index":53.62982177659269,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/italian","measuredAt":null,"stderr":0.5191579721048305,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:proof_bench:overall","benchmarkName":"ProofBench v1.1","category":"math","domain":null,"eligible":true,"ranked":true,"score":60,"unit":"percent","family":"vals-ai:proof_bench:overall","index":67.87130265017127,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/proof_bench","measuredAt":"2026-11-09","stderr":4.924,"costPerTest":0.123288,"latency":513.687},{"benchmarkId":"arc-prize:v2-semi-private:overall","benchmarkName":"ARC-AGI-2 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":59.540000000000006,"unit":"percent","family":"arc-prize:v2-semi-private:overall","index":66.52539549836288,"aboveFit":false,"run":"max effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.67,"latency":null},{"benchmarkId":"euroeval:euroeval-dutch:overall","benchmarkName":"EuroEval Dutch","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":59.27666666666667,"unit":"percent","family":"euroeval","index":52.645032952751315,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/dutch","measuredAt":null,"stderr":0.5765928019488791,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-spanish:overall","benchmarkName":"EuroEval Spanish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":57.86124999999999,"unit":"percent","family":"euroeval","index":50.72463821418116,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/spanish","measuredAt":null,"stderr":0.4223575943439398,"costPerTest":null,"latency":null},{"benchmarkId":"healthBenchProfessional","benchmarkName":"HealthBench Professional","category":"knowledge","score":55.7,"unit":"percent","family":"healthBenchProfessional","index":68.02593853910936,"aboveFit":false,"run":null,"source":"Rebecca Soskin Hicks, Mikhail Trofimov, Dominick Lim, Rahul K. Arora, Foivos Tsimpourlas, Preston Bowman, Michael Sharman, Chi Tong, Kavin Karthik, Arnav Dugar, Akshay Jagadeesh, Khaled Saab, Johannes Heidecke, Ashley Alexander, Nate Gross, Karan Singhal","sourceUrl":"https://arxiv.org/abs/2604.27470","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"frontierCode11Extended","benchmarkName":"FrontierCode 1.1 Extended","category":"coding","score":55.1,"unit":"percent","family":"frontierCode11Extended","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:fabv2:overall","benchmarkName":"Finance Agent (v2)","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":55.044,"unit":"percent","family":"vals-ai:fabv2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/fabv2","measuredAt":"2026-10-29","stderr":0.309,"costPerTest":0.279224,"latency":772.153},{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":54.175714285714285,"unit":"percent","family":"euroeval","index":45.72421426750358,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.36540165208567243,"costPerTest":null,"latency":null},{"benchmarkId":"aaSciCode","benchmarkName":"Artificial Analysis SciCode","category":"coding","score":53.6,"unit":"percent","family":"scicode-family","index":65.71764389034095,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/scicode","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"toolathlon","benchmarkName":"Toolathlon","category":"agentic","score":53.4,"unit":"percent","family":"toolathlon","index":65.40564341366438,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":52.6275,"unit":"percent","family":"euroeval","index":43.62364371574744,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.6428622519638247,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-research:overall","benchmarkName":"LiveBench Research","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":51,"unit":"percent","family":"livebench-2-agentic","index":55.88735742331351,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Research","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.8207772727272726,"latency":null},{"benchmarkId":"livebench:livebench-agentic-coding:overall","benchmarkName":"LiveBench Agentic Coding","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":48.434333333333335,"unit":"percent","family":"livebench:livebench-agentic-coding:overall","index":59.11219341120024,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Agentic%20Coding","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.4937291666666667,"latency":null},{"benchmarkId":"gdpvalAaNormalized","benchmarkName":"GDPval-AA normalized","category":"agentic","score":48,"unit":"percent","family":"gdpvalAaNormalized","index":68.35265947840072,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"osWorld2","benchmarkName":"OSWorld 2.0","category":"agentic","score":45.6,"unit":"percent","family":"osWorld2","index":72.50866461338788,"aboveFit":false,"run":null,"source":"Mengqi Yuan, Zilong Zhou, Xinzhuang Xiong, Weiming Wu, Jiayang Sun, Jiamin Song, Kaiqian Cui, Bowen Wang, Haoyuan Wu, Yitong Li, Dunjie Lu, Haikong Lu, Qi Zhen, Xinyuan Wang, Jiaqi Deng, Yuhao Yang, Cheng Chen, Boyuan Zheng, Alex Su, Xiao Yu, Hao Zou, Saaket Agashe, Xing Han Lu, Manpreet Kaur, Zhengyang Qi, Vincent Sunn Chen, Frederic Sala, Dayiheng Liu, Junyang Lin, Zhou Yu, Yu Su, Siva Reddy, Xin Eric Wang, Peng Qi, Tianbao Xie, Tao Yu","sourceUrl":"https://arxiv.org/abs/2606.29537","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:code-migration:overall","benchmarkName":"Code Migration","category":"coding","domain":null,"eligible":true,"ranked":true,"score":44.55,"unit":"percent","family":"vals-ai:code-migration:overall","index":68.8558836190326,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/code-migration","measuredAt":"2026-10-29","stderr":4.244,"costPerTest":1.882918,"latency":3514.758},{"benchmarkId":"epoch-ai:epoch-chess-puzzles:overall","benchmarkName":"Chess Puzzles","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":43,"unit":"percent","family":"epoch-ai:epoch-chess-puzzles:overall","index":66.62232911659308,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":4.97569851956243,"costPerTest":null,"latency":null},{"benchmarkId":"omniscienceAccuracy","benchmarkName":"Artificial Analysis Omniscience Accuracy","category":"knowledge","score":42.7,"unit":"percent","family":"omniscienceAccuracy","index":64.50288098596678,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-furniture-assembly:overall","benchmarkName":"Furniture Assembly","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":42.5,"unit":"percent","family":"epoch-ai:epoch-furniture-assembly:overall","index":66.47602476771101,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":6.380695638890652,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:medcode:overall","benchmarkName":"MedCode","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":42.391,"unit":"percent","family":"vals-ai:medcode:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medcode","measuredAt":"2026-10-29","stderr":2.27,"costPerTest":0.01597,"latency":81.283},{"benchmarkId":"epoch-ai:epoch-simpleqa-verified:overall","benchmarkName":"SimpleQA Verified","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":41,"unit":"percent","family":"epoch-ai:epoch-simpleqa-verified:overall","index":58.075671718375496,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.5560917136921657,"costPerTest":null,"latency":null},{"benchmarkId":"aaEnterpriseOpsGym","benchmarkName":"Artificial Analysis EnterpriseOps-Gym","category":"agentic","score":40.8,"unit":"percent","family":"aaEnterpriseOpsGym","index":null,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/enterprise-ops-gym-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaItbench","benchmarkName":"Artificial Analysis ITBench-AA","category":"agentic","score":40.3,"unit":"percent","family":"aaItbench","index":58.53488664899706,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/itbench-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":39.5,"unit":"percent","family":"hle","index":64.01670774199036,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:legal_research:overall","benchmarkName":"Legal Research Bench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":36.538,"unit":"percent","family":"vals-ai:legal_research:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_research","measuredAt":"2026-11-09","stderr":3.347,"costPerTest":0.845008,"latency":2373.859},{"benchmarkId":"cursorBench40","benchmarkName":"cursorBench40","category":"coding","score":35.9,"unit":"percent","family":"cursorBench40","index":63.67208260914359,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"apexAgentsAa","benchmarkName":"APEX-Agents-AA","category":"agentic","score":35.8,"unit":"percent","family":"apexAgentsAa","index":66.67794098404416,"aboveFit":false,"run":null,"source":"Artificial Analysis / Mercor","sourceUrl":"https://artificialanalysis.ai/evaluations/apex-agents-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"healthBenchHard","benchmarkName":"HealthBench Hard","category":"knowledge","score":32,"unit":"percent","family":"healthBenchHard","index":71.54418483897754,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-long-horizon:overall","benchmarkName":"LiveBench Long Agentic","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":30.4,"unit":"percent","family":"livebench-2-agentic","index":40.82193686795209,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Long%20Agentic","measuredAt":"2026-10-14","stderr":null,"costPerTest":1.1011611111111113,"latency":null},{"benchmarkId":"vals-ai:tax_agent_bench:overall","benchmarkName":"Tax Agent Bench","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":24.815,"unit":"percent","family":"vals-ai:tax_agent_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench","measuredAt":"2026-11-09","stderr":3.113,"costPerTest":0.465359,"latency":2565.695},{"benchmarkId":"vals-ai:vcb-1-100:overall","benchmarkName":"Vibe Code Bench 1-100","category":"coding","domain":null,"eligible":true,"ranked":true,"score":22.594,"unit":"percent","family":"vals-ai:vcb-1-100:overall","index":70.69839213956944,"aboveFit":false,"run":"OpenHands","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vcb-1-100","measuredAt":"2026-10-29","stderr":4.003,"costPerTest":4.914059,"latency":7374.465},{"benchmarkId":"epoch-ai:epoch-mirrorcode:overall","benchmarkName":"MirrorCode","category":"coding","domain":null,"eligible":true,"ranked":true,"score":21.11111111111111,"unit":"percent","family":"epoch-ai:epoch-mirrorcode:overall","index":58.837036696307685,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":9.822049844066655,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-mystery-game-puzzles:overall","benchmarkName":"Mystery Game Puzzles","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":21,"unit":"percent","family":"epoch-ai:epoch-mystery-game-puzzles:overall","index":54.9437110328788,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":4.093601807403323,"costPerTest":null,"latency":null},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":20.6,"unit":"percent","family":"critpt","index":76.72831233676999,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"terminal-bench:terminal-bench-4-1-0:overall","benchmarkName":"Terminal-Bench 4.1.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":17.58,"unit":"percent","family":"terminal-bench:terminal-bench-4-1-0:overall","index":61.17778325873251,"aboveFit":false,"run":"max effort · Codex","source":"Terminal-Bench","sourceUrl":"https://www.tbench.ai/leaderboard","measuredAt":"2026-09-29","stderr":1.4846938775510206,"costPerTest":1.0505151515151516,"latency":4088.3},{"benchmarkId":"terminal-bench:terminal-bench-4-0-0:overall","benchmarkName":"Terminal-Bench 4.0.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":17.27,"unit":"percent","family":"terminal-bench-4","index":62.71647663045937,"aboveFit":false,"run":"max effort · Codex","source":"Terminal-Bench","sourceUrl":"https://www.tbench.ai/leaderboard","measuredAt":"2026-09-29","stderr":1.4540816326530612,"costPerTest":1.0505151515151516,"latency":4088.3},{"benchmarkId":"vals-ai:mysterymechanism:overall","benchmarkName":"MysteryMechanism","category":null,"domain":"Science","eligible":false,"ranked":true,"score":14.414,"unit":"percent","family":"vals-ai:mysterymechanism:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mysterymechanism","measuredAt":"2026-11-11","stderr":2.363,"costPerTest":0.109246,"latency":493.052},{"benchmarkId":"frontierBench","benchmarkName":"Terminal-Bench 3.0","category":"agentic","score":14.32,"unit":"percent","family":"frontierBench","index":63.22605974367238,"aboveFit":false,"run":null,"source":"Ryan Marten, Alex Shaw, Andy Konwinski, Harbor, and the Laude Institute","sourceUrl":"https://www.frontierbench.ai/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"exploitGym","benchmarkName":"ExploitGym","category":"agentic","score":12.4,"unit":"percent","family":"exploitGym","index":66.51028845376347,"aboveFit":false,"run":null,"source":"Zhun Wang, Nico Schiller, Hongwei Li, Srijiith Sesha Narayana, Milad Nasr, Nicholas Carlini, Xiangyu Qi, Eric Wallace, Elie Bursztein, Luca Invernizzi, Kurt Thomas, Yan Shoshitaishvili, Wenbo Guo, Jingxuan He, Thorsten Holz, Dawn Song","sourceUrl":"https://arxiv.org/abs/2605.11086","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:terminal-bench-4:overall","benchmarkName":"Terminal-Bench 4.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":11.616,"unit":"percent","family":"terminal-bench-4","index":60.03173900841716,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-4","measuredAt":"2026-11-09","stderr":1.01,"costPerTest":0.731646,"latency":2155.227},{"benchmarkId":"apprenticeBench","benchmarkName":"ApprenticeBench: end-to-end computer use, continual learning, and long-horizon agency on a real accounts-payable job","category":"agentic","score":7,"unit":"percent","family":"apprenticeBench","index":61.757221677741256,"aboveFit":false,"run":null,"source":"NeoCognition","sourceUrl":"https://neocognition.io/blog/apprentice-bench/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:srebench:overall","benchmarkName":"SRE Bench","category":null,"domain":"Other","eligible":false,"ranked":true,"score":1.527,"unit":"percent","family":"vals-ai:srebench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/srebench","measuredAt":"2026-11-05","stderr":0.759,"costPerTest":3.439085,"latency":6114.391},{"benchmarkId":"lmarena:agent-bash-recovery:overall","benchmarkName":"Agent Arena command recovery","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":0.5144852705668113,"unit":"index","family":"agent-arena","index":66.95378885958188,"aboveFit":false,"run":"xhigh effort","source":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset","measuredAt":"2026-10-08","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"arc-prize:v3-semi-private:overall","benchmarkName":"ARC-AGI-3 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":0.18,"unit":"percent","family":"arc-prize:v3-semi-private:overall","index":65.38206267316735,"aboveFit":false,"run":"max effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mazeBench","benchmarkName":"MazeBench: visual spatial reasoning in a 3D open world","category":"agentic","score":0,"unit":"percent","family":"mazeBench","index":61.97866734834783,"aboveFit":false,"run":null,"source":"Jonathan Pappas and David Pappas","sourceUrl":"https://mazebench.com/blog?post=maze-bench-results","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:programbench:overall","benchmarkName":"ProgramBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":0,"unit":"percent","family":"vals-ai:programbench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/programbench","measuredAt":"2026-11-09","stderr":0,"costPerTest":0.942725,"latency":2493.897},{"benchmarkId":"vals-ai:terminal-bench-science:overall","benchmarkName":"Terminal-Bench-Science","category":null,"domain":"Science","eligible":false,"ranked":true,"score":0,"unit":"percent","family":"vals-ai:terminal-bench-science:overall","index":null,"aboveFit":false,"run":"Mini-SWE-agent","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-science","measuredAt":"2026-11-09","stderr":0,"costPerTest":0.581456,"latency":7396.452}],"price":{"inputPerMillion":0.19999999999999998,"outputPerMillion":1.2,"batch":{"inputPerMillion":0.09999999999999999,"outputPerMillion":0.6},"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":51,"timeToFirstToken":1.747,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":{"elo":1431.009952898797,"eloUpper":1435.3148556872789,"eloLower":1426.705050110315,"votes":40641,"rank":94,"rankUpper":null,"rankLower":null,"arenaName":"GPT-5.6 Luna-xhigh","organization":"openai","updatedAt":"2026-11-09"},"quality":{"coveragePercent":100,"benchmarkCount":61,"categoryCount":8,"confidence":"high","state":"available"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:arc-prize-foundation","name":"ARC Prize Foundation","url":"https://arcprize.org","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:livebench","name":"LiveBench","url":"https://livebench.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:terminal-bench","name":"Terminal-Bench","url":"https://www.tbench.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:lmarena","name":"LMArena","url":"https://huggingface.co","license":"CC BY 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"muse-spark-open","providerId":"meta","slug":"muse-spark-open","name":"Muse Spark Open","provider":"Meta","description":"Muse Spark Open is a reasoning model from Meta. 31 benchmarks count toward its index, in 6 categories.","logoUrl":null,"releaseDate":"2026-10-15","contextWindowTokens":1048576,"modalities":["text","image","video","file","audio"],"openWeights":true,"details":{"apiId":"meta/muse-spark-open","maxOutputTokens":262144,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.15,"weightsUrl":"https://huggingface.co/meta-models/Muse-Spark-Open-816B-A42B"},"reasoning":true,"overallScore":65.05891016373216,"overallScoreRange":{"low":61.15469242995921,"high":68.96312789750512},"badges":[],"capabilityScores":{"agentic":64.81666218129665,"coding":63.251323794345225,"reasoning":62.31950903771488,"multimodal":69.78435011843023,"knowledge":65.4413763643352,"multilingual":null,"instruction":null,"math":64.8529093933434},"capabilityCounts":{"math":3,"agentic":9,"knowledge":4,"reasoning":6,"multimodal":4,"coding":5},"benchmarks":[{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":98.05555555555556,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":65.96722924983679,"aboveFit":false,"run":"xhigh effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.253502388483925,"costPerTest":null,"latency":null},{"benchmarkId":"arc-prize:v1-public-eval:overall","benchmarkName":"ARC-AGI-1 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":94,"unit":"percent","family":"arc-prize:v1-public-eval:overall","index":null,"aboveFit":false,"run":"xhigh effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.22026259625000005,"latency":null},{"benchmarkId":"mcpAtlas","benchmarkName":"MCP Atlas","category":"agentic","score":89.2,"unit":"percent","family":"mcpAtlas","index":71.12917048121194,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":89,"unit":"percent","family":"gpqa-diamond","index":58.8374688057104,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"arc-prize:v1-semi-private:overall","benchmarkName":"ARC-AGI-1 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":89,"unit":"percent","family":"arc-prize:v1-semi-private:overall","index":67.90387202491206,"aboveFit":false,"run":"xhigh effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.287208805,"latency":null},{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":88.57323232323232,"unit":"percent","family":"gpqa-diamond","index":58.44604915404008,"aboveFit":false,"run":"xhigh effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.8706209910489153,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:sage:overall","benchmarkName":"SAGE","category":null,"domain":"Education","eligible":false,"ranked":true,"score":85.772,"unit":"percent","family":"vals-ai:sage:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/sage","measuredAt":"2026-11-05","stderr":1.737,"costPerTest":0.068527,"latency":64.512},{"benchmarkId":"aaMmmuPro","benchmarkName":"Artificial Analysis MMMU-Pro","category":"multimodal","score":80.9,"unit":"percent","family":"mmmu-pro","index":60.43706727597615,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmmu-pro","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-research:overall","benchmarkName":"LiveBench Research","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":80.3,"unit":"percent","family":"livebench-2-agentic","index":77.3153585044829,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Research","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.7320454545454546,"latency":null},{"benchmarkId":"babyVision","benchmarkName":"BabyVision","category":"multimodal","score":80.2,"unit":"percent","family":"babyVision","index":68.99723335937867,"aboveFit":false,"run":null,"source":"Meta AI","sourceUrl":"https://ai.meta.com/static-resource/muse-spark-1-1-evaluation-report","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":78.3,"unit":"percent","family":"lcr","index":61.018154612527894,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:vibe-code:overall","benchmarkName":"Vibe Code Bench v1.1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":76.187,"unit":"percent","family":"vals-ai:vibe-code:overall","index":69.73898917850491,"aboveFit":false,"run":"OpenHands","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","measuredAt":"2026-11-09","stderr":3.317,"costPerTest":1.655609,"latency":1869.172},{"benchmarkId":"livebench:livebench-reasoning:overall","benchmarkName":"LiveBench Reasoning","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":73.7,"unit":"percent","family":"livebench","index":57.093206851420625,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Reasoning","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.04265504950495049,"latency":null},{"benchmarkId":"simpleVqa","benchmarkName":"SimpleVQA","category":"multimodal","score":73.6,"unit":"percent","family":"simpleVqa","index":63.751139370161965,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-coding-2:overall","benchmarkName":"LiveBench Coding 2.0","category":"coding","domain":null,"eligible":true,"ranked":true,"score":73.6,"unit":"percent","family":"livebench:livebench-coding-2:overall","index":61.48224447251978,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Coding","measuredAt":"2026-10-14","stderr":null,"costPerTest":1.2525379310344826,"latency":null},{"benchmarkId":"toolathlonVerified","benchmarkName":"Toolathlon-Verified","category":"agentic","score":72.8,"unit":"percent","family":"toolathlonVerified","index":67.74141986996078,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tiers-1-3-v2-private:overall","benchmarkName":"FrontierMath-Tiers-1-3-v2-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":69.82456140350877,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tiers-1-3-v2-private:overall","index":67.64697369074213,"aboveFit":false,"run":"xhigh effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.7237781633151585,"costPerTest":null,"latency":null},{"benchmarkId":"jobBench","benchmarkName":"JobBench","category":"agentic","score":61.8,"unit":"percent","family":"jobBench","index":70.67512175219444,"aboveFit":false,"run":null,"source":"Yuetai Li, Yichen Feng, Zhangchen Xu, Zixian Ma, Kaiyuan Zheng, Fengqing Jiang, Xinghua Sun, Rulin Shao, Zichen Chen, Yue Huang, Xinyang Han, Brian Lee, Kayla Xu, Shenglai Zeng, Hang Hua, Xiangliang Zhang, Basel Alomair, Ranjay Krishna, Luke Zettlemoyer, Pang Wei Koh, Bhaskar Ramasubramanian, Luyao Niu, Xiang Yue, Radha Poovendran","sourceUrl":"https://arxiv.org/abs/2605.26329","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:fabv2:overall","benchmarkName":"Finance Agent (v2)","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":61.336,"unit":"percent","family":"vals-ai:fabv2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/fabv2","measuredAt":"2026-10-29","stderr":2.088,"costPerTest":0.770844,"latency":1453.439},{"benchmarkId":"hleWithTools","benchmarkName":"Humanity's Last Exam with tools","category":"agentic","score":61,"unit":"percent","family":"hleWithTools","index":69.41443991541489,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-design:overall","benchmarkName":"LiveBench Design","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":60.3,"unit":"percent","family":"livebench-2-agentic","index":62.68873660607375,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Design","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.355892,"latency":null},{"benchmarkId":"livebench:livebench-data-analysis:overall","benchmarkName":"LiveBench Data Analysis","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":60.2,"unit":"percent","family":"livebench","index":39.604847903442106,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Data%20Analysis","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.044909660000000004,"latency":null},{"benchmarkId":"deepSwe","benchmarkName":"DeepSWE","category":"agentic","score":60,"unit":"percent","family":"deepSwe","index":64.69504567250344,"aboveFit":false,"run":null,"source":"Datacurve AI","sourceUrl":"https://deepswe.datacurve.ai/blog","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaSciCode","benchmarkName":"Artificial Analysis SciCode","category":"coding","score":58.4,"unit":"percent","family":"scicode-family","index":71.97536139313168,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/scicode","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"sweAtlasCodebaseQna","benchmarkName":"SWE-Atlas Codebase QnA","category":"coding","score":58,"unit":"percent","family":"sweAtlasCodebaseQna","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"arc-prize:v2-semi-private:overall","benchmarkName":"ARC-AGI-2 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":57.49999999999999,"unit":"percent","family":"arc-prize:v2-semi-private:overall","index":65.58876070837152,"aboveFit":false,"run":"xhigh effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.7284857499999998,"latency":null},{"benchmarkId":"arc-prize:v2-public-eval:overall","benchmarkName":"ARC-AGI-2 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":57.22222222222223,"unit":"percent","family":"arc-prize:v2-public-eval:overall","index":null,"aboveFit":false,"run":"xhigh effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.8068895291666668,"latency":null},{"benchmarkId":"epoch-ai:epoch-simpleqa-verified:overall","benchmarkName":"SimpleQA Verified","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":56.599999999999994,"unit":"percent","family":"epoch-ai:epoch-simpleqa-verified:overall","index":71.07184886021898,"aboveFit":false,"run":"xhigh effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.5680876566375042,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:emb:overall","benchmarkName":"EMB","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":53.667,"unit":"percent","family":"vals-ai:emb:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/emb","measuredAt":"2026-10-20","stderr":3.196,"costPerTest":5.348821,"latency":5078.614},{"benchmarkId":"zeroBench","benchmarkName":"ZeroBench","category":"multimodal","score":50,"unit":"percent","family":"zeroBench","index":85.95196046820409,"aboveFit":true,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gdpvalAaNormalized","benchmarkName":"GDPval-AA normalized","category":"agentic","score":45.1,"unit":"percent","family":"gdpvalAaNormalized","index":66.27922686949441,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":43.9,"unit":"percent","family":"hle","index":68.43119582012145,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"omniscienceAccuracy","benchmarkName":"Artificial Analysis Omniscience Accuracy","category":"knowledge","score":42,"unit":"percent","family":"omniscienceAccuracy","index":63.62070179712511,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:legal_research:overall","benchmarkName":"Legal Research Bench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":38.462,"unit":"percent","family":"vals-ai:legal_research:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_research","measuredAt":"2026-11-09","stderr":3.381,"costPerTest":1.559385,"latency":1763.023},{"benchmarkId":"livebench:livebench-browser:overall","benchmarkName":"LiveBench Browser Use","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":36,"unit":"percent","family":"livebench-2-agentic","index":44.91739099950665,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Browser","measuredAt":"2026-10-14","stderr":null,"costPerTest":2.6433679999999997,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tier-4-v2-private:overall","benchmarkName":"FrontierMath-Tier-4-v2-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":31.70731707317073,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tier-4-v2-private:overall","index":60.94452523945129,"aboveFit":false,"run":"xhigh effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":7.3576112824382225,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:tax_agent_bench:overall","benchmarkName":"Tax Agent Bench","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":26.321,"unit":"percent","family":"vals-ai:tax_agent_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench","measuredAt":"2026-11-09","stderr":2.653,"costPerTest":0.883559,"latency":701.825},{"benchmarkId":"vals-ai:hlab:overall","benchmarkName":"HLAB-Vals-1.1","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":25,"unit":"percent","family":"vals-ai:hlab:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/hlab","measuredAt":"2026-11-09","stderr":3.599,"costPerTest":13.6316,"latency":14355.684},{"benchmarkId":"vals-ai:code-migration:overall","benchmarkName":"Code Migration","category":"coding","domain":null,"eligible":true,"ranked":true,"score":21.171,"unit":"percent","family":"vals-ai:code-migration:overall","index":56.43602146838739,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/code-migration","measuredAt":"2026-10-29","stderr":3.472,"costPerTest":9.042387,"latency":6790.697},{"benchmarkId":"livebench:livebench-long-horizon:overall","benchmarkName":"LiveBench Long Agentic","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":18.9,"unit":"percent","family":"livebench-2-agentic","index":32.41162927636684,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Long%20Agentic","measuredAt":"2026-10-14","stderr":null,"costPerTest":4.24356551724138,"latency":null},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":14.9,"unit":"percent","family":"critpt","index":65.69377743158633,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:terminal-bench-4:overall","benchmarkName":"Terminal-Bench 4.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":6.566,"unit":"percent","family":"terminal-bench-4","index":57.63380391975194,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-4","measuredAt":"2026-11-09","stderr":0.505,"costPerTest":7.264062,"latency":7466.403},{"benchmarkId":"frontierCode2","benchmarkName":"FrontierCode 2.0","category":"coding","score":6.2,"unit":"percent","family":"frontierCode2","index":56.62400245918239,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"arc-prize:v3-semi-private:overall","benchmarkName":"ARC-AGI-3 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":0.13621973920554636,"unit":"percent","family":"arc-prize:v3-semi-private:overall","index":65.36346207146008,"aboveFit":false,"run":"xhigh effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mazeBench","benchmarkName":"MazeBench: visual spatial reasoning in a 3D open world","category":"agentic","score":0,"unit":"percent","family":"mazeBench","index":61.97866734834783,"aboveFit":false,"run":null,"source":"Jonathan Pappas and David Pappas","sourceUrl":"https://mazebench.com/blog?post=maze-bench-results","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":1.25,"outputPerMillion":4.25,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":86,"timeToFirstToken":0.5,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":90,"benchmarkCount":31,"categoryCount":6,"confidence":"medium","state":"available"},"hosts":[{"host":"fireworks-ai","name":"Fireworks AI","kind":"host","url":"https://fireworks.ai/docs/","inputPerMillion":1.25,"outputPerMillion":4.25,"cacheReadPerMillion":0.15,"contextTokens":1048576,"maxOutputTokens":131072,"deprecated":false},{"host":"modal","name":"Modal","kind":"host","url":"https://modal.com/docs/guide/endpoints","inputPerMillion":1.25,"outputPerMillion":4.25,"cacheReadPerMillion":0.15,"contextTokens":1048576,"maxOutputTokens":131072,"deprecated":false},{"host":"nebius","name":"Nebius","kind":"host","url":"https://docs.tokenfactory.nebius.com/","inputPerMillion":1.25,"outputPerMillion":4.25,"cacheReadPerMillion":0.15,"contextTokens":1048576,"maxOutputTokens":1048576,"deprecated":false},{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":1.25,"outputPerMillion":4.25,"cacheReadPerMillion":0.15,"contextTokens":1048576,"maxOutputTokens":262144,"deprecated":false},{"host":"together-ai","name":"Together AI","kind":"host","url":"https://docs.together.ai/docs/serverless-models","inputPerMillion":1.25,"outputPerMillion":4.25,"cacheReadPerMillion":0.15,"contextTokens":1048576,"maxOutputTokens":131072,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:arc-prize-foundation","name":"ARC Prize Foundation","url":"https://arcprize.org","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:livebench","name":"LiveBench","url":"https://livebench.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"muse-spark-1-2","providerId":"meta","slug":"muse-spark-1-2","name":"Muse Spark 1.2","provider":"Meta","description":"Muse Spark 1.2 is a reasoning model from Meta in the Muse Spark family. 34 benchmarks count toward its index, in 7 categories.","logoUrl":null,"releaseDate":"2026-08-05","contextWindowTokens":1048576,"modalities":["text","image","video","file"],"openWeights":false,"details":{"apiId":"meta/muse-spark-1.2","maxOutputTokens":943718,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.15,"weightsUrl":null},"reasoning":true,"overallScore":64.89414087037775,"overallScoreRange":{"low":61.62100874862339,"high":68.16727299213213},"badges":[],"capabilityScores":{"agentic":62.67084181874665,"coding":64.31409114698553,"reasoning":64.13285941377015,"multimodal":67.23954918575402,"knowledge":65.22124515620291,"multilingual":null,"instruction":76.97883457319489,"math":64.4445222850834},"capabilityCounts":{"math":2,"knowledge":6,"reasoning":4,"coding":11,"multimodal":1,"agentic":9,"instruction":1},"benchmarks":[{"benchmarkId":"livebench:livebench-math:overall","benchmarkName":"LiveBench Mathematics","category":"math","domain":null,"eligible":true,"ranked":true,"score":91.20349999999999,"unit":"percent","family":"livebench:livebench-math:overall","index":66.88690183685223,"aboveFit":false,"run":"xhigh effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Mathematics","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.0625485488126649,"latency":null},{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":90.4,"unit":"percent","family":"gpqa-diamond","index":60.1215105505981,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:medscribe:overall","benchmarkName":"MedScribe","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":90.062,"unit":"percent","family":"vals-ai:medscribe:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medscribe","measuredAt":"2026-10-29","stderr":1.958,"costPerTest":0.037779,"latency":61.458},{"benchmarkId":"livebench:livebench-reasoning:overall","benchmarkName":"LiveBench Reasoning","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":90.00475,"unit":"percent","family":"livebench","index":78.21493430009119,"aboveFit":false,"run":"xhigh effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Reasoning","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.09936831683168318,"latency":null},{"benchmarkId":"vals-ai:mmlu_pro:overall","benchmarkName":"MMLU Pro","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":88.275,"unit":"percent","family":"mmlu-pro","index":55.69666218105877,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","measuredAt":"2026-09-01","stderr":0.318,"costPerTest":0.010668,"latency":44.234},{"benchmarkId":"vulcanBench","benchmarkName":"VulcanBench v3","category":"coding","score":87,"unit":"percent","family":"vulcanBench","index":68.4175418120868,"aboveFit":false,"run":null,"source":"VulcanBench contributors","sourceUrl":"https://github.com/morganlinton/VulcanBench/tree/main","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:swebench:overall","benchmarkName":"SWE-bench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":86.6,"unit":"percent","family":"swe-bench-verified","index":65.10041376553936,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/swebench","measuredAt":"2026-09-01","stderr":1.525,"costPerTest":0.554231,"latency":487.088},{"benchmarkId":"vals-ai:mmmu:overall","benchmarkName":"MMMU Pro","category":"multimodal","domain":null,"eligible":true,"ranked":true,"score":86.127,"unit":"percent","family":"mmmu-pro","index":67.23954918575402,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmmu","measuredAt":"2026-09-01","stderr":0.831,"costPerTest":0.018022,"latency":73.839},{"benchmarkId":"vals-ai:legal_bench:overall","benchmarkName":"LegalBench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":85.262,"unit":"percent","family":"vals-ai:legal_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","measuredAt":"2026-10-13","stderr":0.446,"costPerTest":0.00637,"latency":30.102},{"benchmarkId":"terminalBench21","benchmarkName":"Terminal-Bench 2.1","category":"agentic","score":82.9,"unit":"percent","family":"terminal-bench-2-1","index":69.62563092216486,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:tax_eval_v2:overall","benchmarkName":"TaxEval v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":80.376,"unit":"percent","family":"vals-ai:tax_eval_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","measuredAt":"2026-09-01","stderr":0.763,"costPerTest":0.014199,"latency":48.313},{"benchmarkId":"aaTerminalBench21","benchmarkName":"Artificial Analysis Terminal-Bench v2.1","category":"agentic","score":80.1,"unit":"percent","family":"aaTerminalBench21","index":62.63559690622473,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/terminalbench-v2-1","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:vibe-code:overall","benchmarkName":"Vibe Code Bench v1.1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":79.1,"unit":"percent","family":"vals-ai:vibe-code:overall","index":70.79004534516737,"aboveFit":false,"run":"OpenHands","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","measuredAt":"2026-11-09","stderr":3.305,"costPerTest":1.529642,"latency":1195.221},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":79,"unit":"percent","family":"lcr","index":61.49841098043786,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-language:overall","benchmarkName":"LiveBench Language","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":78.57466666666666,"unit":"percent","family":"livebench:livebench-language:overall","index":63.41173044847135,"aboveFit":false,"run":"xhigh effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Language","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.05595333333333334,"latency":null},{"benchmarkId":"livebench:livebench-coding-2:overall","benchmarkName":"LiveBench Coding 2.0","category":"coding","domain":null,"eligible":true,"ranked":true,"score":78.2,"unit":"percent","family":"livebench:livebench-coding-2:overall","index":64.81413982218434,"aboveFit":false,"run":"xhigh effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Coding","measuredAt":"2026-10-14","stderr":null,"costPerTest":1.4657206896551722,"latency":null},{"benchmarkId":"livebench:livebench-coding:overall","benchmarkName":"LiveBench Coding","category":"coding","domain":null,"eligible":true,"ranked":true,"score":77.5415,"unit":"percent","family":"livebench:livebench-coding:overall","index":63.879281285626774,"aboveFit":false,"run":"xhigh effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Coding","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.050888888888888886,"latency":null},{"benchmarkId":"livebench:livebench-data-analysis:overall","benchmarkName":"LiveBench Data Analysis","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":76.45833333333333,"unit":"percent","family":"livebench","index":60.66644562412486,"aboveFit":false,"run":"xhigh effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Data%20Analysis","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.10944733333333334,"latency":null},{"benchmarkId":"livebench:livebench-instruction-following:overall","benchmarkName":"LiveBench Instruction Following","category":"instruction","domain":null,"eligible":true,"ranked":true,"score":74.325,"unit":"percent","family":"livebench:livebench-instruction-following:overall","index":76.97883457319489,"aboveFit":false,"run":"xhigh effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=IF","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.07215700000000001,"latency":null},{"benchmarkId":"vals-ai:corp_fin_v2:overall","benchmarkName":"CorpFin v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":70.94,"unit":"percent","family":"vals-ai:corp_fin_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","measuredAt":"2026-08-12","stderr":0.892,"costPerTest":0.108223,"latency":30.047},{"benchmarkId":"vals-ai:terminal-bench-2-1:overall","benchmarkName":"Terminal-Bench 2.1","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":69.663,"unit":"percent","family":"terminal-bench-2-1","index":61.88390046263605,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2-1","measuredAt":"2026-10-01","stderr":0.649,"costPerTest":0.499472,"latency":879.273},{"benchmarkId":"vals-ai:cyber:overall","benchmarkName":"CyberBench v1.1","category":null,"domain":"Other","eligible":false,"ranked":true,"score":69.524,"unit":"percent","family":"vals-ai:cyber:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/cyber","measuredAt":"2026-11-05","stderr":5.558,"costPerTest":2.149077,"latency":1220.583},{"benchmarkId":"vals-ai:public-benefits-bench:overall","benchmarkName":"Public Benefits Bench v1.1","category":null,"domain":"Public benefits","eligible":false,"ranked":true,"score":68.471,"unit":"percent","family":"vals-ai:public-benefits-bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/public-benefits-bench","measuredAt":"2026-10-12","stderr":1.209,"costPerTest":0.587608,"latency":535.162},{"benchmarkId":"livebench:livebench-research:overall","benchmarkName":"LiveBench Research","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":67.7,"unit":"percent","family":"livebench-2-agentic","index":68.10058670848514,"aboveFit":false,"run":"xhigh effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Research","measuredAt":"2026-10-14","stderr":null,"costPerTest":1.783181818181818,"latency":null},{"benchmarkId":"livebench:livebench-design:overall","benchmarkName":"LiveBench Design","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":67.1,"unit":"percent","family":"livebench-2-agentic","index":67.66178805153287,"aboveFit":false,"run":"xhigh effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Design","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.856232,"latency":null},{"benchmarkId":"vals-ai:mortgage_tax:overall","benchmarkName":"MortgageTax","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":65.422,"unit":"percent","family":"vals-ai:mortgage_tax:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mortgage_tax","measuredAt":"2026-09-01","stderr":0.934,"costPerTest":0.026128,"latency":44.623},{"benchmarkId":"vals-ai:biomysterybench:overall","benchmarkName":"BioMysteryBench","category":null,"domain":"Science","eligible":false,"ranked":true,"score":64.815,"unit":"percent","family":"vals-ai:biomysterybench:overall","index":null,"aboveFit":false,"run":"Terminus 2","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/biomysterybench","measuredAt":"2026-11-09","stderr":1.335,"costPerTest":0.994513,"latency":1417.288},{"benchmarkId":"vals-ai:fabv2:overall","benchmarkName":"Finance Agent (v2)","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":60.599,"unit":"percent","family":"vals-ai:fabv2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/fabv2","measuredAt":"2026-10-29","stderr":0.279,"costPerTest":0.76972,"latency":304.205},{"benchmarkId":"epoch-ai:epoch-simpleqa-verified:overall","benchmarkName":"SimpleQA Verified","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":60.30150753768844,"unit":"percent","family":"epoch-ai:epoch-simpleqa-verified:overall","index":74.15553140199256,"aboveFit":false,"run":"xhigh effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.5518791563606453,"costPerTest":null,"latency":null},{"benchmarkId":"deepSwe","benchmarkName":"DeepSWE","category":"agentic","score":59.3,"unit":"percent","family":"deepSwe","index":64.18616521370767,"aboveFit":false,"run":null,"source":"Datacurve AI","sourceUrl":"https://deepswe.datacurve.ai/blog","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-agentic-coding:overall","benchmarkName":"LiveBench Agentic Coding","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":57.57566666666667,"unit":"percent","family":"livebench:livebench-agentic-coding:overall","index":68.53949478559763,"aboveFit":false,"run":"xhigh effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Agentic%20Coding","measuredAt":"2026-06-25","stderr":null,"costPerTest":3.9246180555555554,"latency":null},{"benchmarkId":"aaSciCode","benchmarkName":"Artificial Analysis SciCode","category":"coding","score":57.4,"unit":"percent","family":"scicode-family","index":70.67167024671694,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/scicode","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:emb:overall","benchmarkName":"EMB","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":56.978,"unit":"percent","family":"vals-ai:emb:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/emb","measuredAt":"2026-10-20","stderr":3.127,"costPerTest":2.313636,"latency":1027.476},{"benchmarkId":"vals-ai:skillsbench:overall","benchmarkName":"SkillsBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":53.036,"unit":"percent","family":"vals-ai:skillsbench:overall","index":64.61831593132307,"aboveFit":false,"run":"OpenHands","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/skillsbench","measuredAt":"2026-10-29","stderr":4.42,"costPerTest":0.648998,"latency":780.011},{"benchmarkId":"vals-ai:ioi-v1:overall","benchmarkName":"IOI v1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":49.5,"unit":"percent","family":"vals-ai:ioi-v1:overall","index":64.76946375910009,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/ioi-v1","measuredAt":"2026-08-09","stderr":3.667,"costPerTest":6.100817,"latency":4771.52},{"benchmarkId":"vals-ai:medcode:overall","benchmarkName":"MedCode","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":49.346,"unit":"percent","family":"vals-ai:medcode:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medcode","measuredAt":"2026-10-29","stderr":2.187,"costPerTest":0.039023,"latency":60.17},{"benchmarkId":"gdpvalAaNormalized","benchmarkName":"GDPval-AA normalized","category":"agentic","score":48.6,"unit":"percent","family":"gdpvalAaNormalized","index":68.78164553541582,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaEnterpriseOpsGym","benchmarkName":"Artificial Analysis EnterpriseOps-Gym","category":"agentic","score":47.3,"unit":"percent","family":"aaEnterpriseOpsGym","index":null,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/enterprise-ops-gym-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":45.5,"unit":"percent","family":"hle","index":70.03646421216912,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"omniscienceAccuracy","benchmarkName":"Artificial Analysis Omniscience Accuracy","category":"knowledge","score":45.4,"unit":"percent","family":"omniscienceAccuracy","index":67.90557214292753,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:legal_research:overall","benchmarkName":"Legal Research Bench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":43.75,"unit":"percent","family":"vals-ai:legal_research:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_research","measuredAt":"2026-11-09","stderr":3.448,"costPerTest":0.508097,"latency":306.893},{"benchmarkId":"vals-ai:proof_bench:overall","benchmarkName":"ProofBench v1.1","category":"math","domain":null,"eligible":true,"ranked":true,"score":43,"unit":"percent","family":"vals-ai:proof_bench:overall","index":62.002142733314564,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/proof_bench","measuredAt":"2026-11-09","stderr":4.976,"costPerTest":0.442007,"latency":469.146},{"benchmarkId":"livebench:livebench-browser:overall","benchmarkName":"LiveBench Browser Use","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":36,"unit":"percent","family":"livebench-2-agentic","index":44.91739099950665,"aboveFit":false,"run":"xhigh effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Browser","measuredAt":"2026-10-14","stderr":null,"costPerTest":2.494524,"latency":null},{"benchmarkId":"vals-ai:tax_agent_bench:overall","benchmarkName":"Tax Agent Bench","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":32.798,"unit":"percent","family":"vals-ai:tax_agent_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench","measuredAt":"2026-11-09","stderr":2.751,"costPerTest":0.269212,"latency":119.905},{"benchmarkId":"vals-ai:code-migration:overall","benchmarkName":"Code Migration","category":"coding","domain":null,"eligible":true,"ranked":true,"score":29.954,"unit":"percent","family":"vals-ai:code-migration:overall","index":61.10190321136686,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/code-migration","measuredAt":"2026-10-29","stderr":4.023,"costPerTest":3.784593,"latency":1866.928},{"benchmarkId":"epoch-ai:epoch-chess-puzzles:overall","benchmarkName":"Chess Puzzles","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":27,"unit":"percent","family":"epoch-ai:epoch-chess-puzzles:overall","index":54.47808599419153,"aboveFit":false,"run":"xhigh effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":4.461960433384737,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:ioi:overall","benchmarkName":"IOI","category":"coding","domain":null,"eligible":true,"ranked":true,"score":21.778,"unit":"percent","family":"vals-ai:ioi:overall","index":54.292583217875034,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/ioi","measuredAt":"2026-10-29","stderr":0.873,"costPerTest":2.640493,"latency":1534.626},{"benchmarkId":"livebench:livebench-long-horizon:overall","benchmarkName":"LiveBench Long Agentic","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":21.3,"unit":"percent","family":"livebench-2-agentic","index":34.166823904175935,"aboveFit":false,"run":"xhigh effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Long%20Agentic","measuredAt":"2026-10-14","stderr":null,"costPerTest":3.744231818181818,"latency":null},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":17.7,"unit":"percent","family":"critpt","index":71.11425071834321,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"frontierSweV2","benchmarkName":"FrontierSWE v2","category":"coding","score":12,"unit":"percent","family":"frontierSweV2","index":58.999644219854275,"aboveFit":false,"run":null,"source":"Proximal","sourceUrl":"https://www.frontierswe.com/blog/v2","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:terminal-bench-4:overall","benchmarkName":"Terminal-Bench 4.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":6.061,"unit":"percent","family":"terminal-bench-4","index":57.394010410885414,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-4","measuredAt":"2026-11-09","stderr":1.515,"costPerTest":4.135537,"latency":4705.164},{"benchmarkId":"lmarena:agent-bash-recovery:overall","benchmarkName":"Agent Arena command recovery","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":5.790054455603921,"unit":"index","family":"agent-arena","index":71.29113378766792,"aboveFit":false,"run":"xhigh effort","source":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset","measuredAt":"2026-10-08","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:programbench:overall","benchmarkName":"ProgramBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":0.5,"unit":"percent","family":"vals-ai:programbench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/programbench","measuredAt":"2026-11-09","stderr":0.5,"costPerTest":2.467822,"latency":1598.124},{"benchmarkId":"mazeBench","benchmarkName":"MazeBench: visual spatial reasoning in a 3D open world","category":"agentic","score":0,"unit":"percent","family":"mazeBench","index":61.97866734834783,"aboveFit":false,"run":null,"source":"Jonathan Pappas and David Pappas","sourceUrl":"https://mazebench.com/blog?post=maze-bench-results","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:srebench:overall","benchmarkName":"SRE Bench","category":null,"domain":"Other","eligible":false,"ranked":true,"score":0,"unit":"percent","family":"vals-ai:srebench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/srebench","measuredAt":"2026-11-05","stderr":0,"costPerTest":23.427477,"latency":5986.803},{"benchmarkId":"lmarena:agent-task-outcome:overall","benchmarkName":"Agent Arena task outcome","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":-9.795163940584406,"unit":"index","family":"agent-arena","index":58.47764095062057,"aboveFit":false,"run":"xhigh effort","source":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset","measuredAt":"2026-11-11","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lmarena:agent-steerability:overall","benchmarkName":"Agent Arena steerability","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":-12.531056306003642,"unit":"index","family":"agent-arena","index":56.22830851676738,"aboveFit":false,"run":"xhigh effort","source":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset","measuredAt":"2026-11-11","stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":1.25,"outputPerMillion":4.25,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":123,"timeToFirstToken":3.7295,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":{"elo":1484.0008497377926,"eloUpper":1489.864946517766,"eloLower":1478.1367529578192,"votes":15004,"rank":17,"rankUpper":null,"rankLower":null,"arenaName":"Muse Spark 1.2-xhigh","organization":"meta","updatedAt":"2026-11-09"},"quality":{"coveragePercent":95,"benchmarkCount":34,"categoryCount":7,"confidence":"high","state":"available"},"hosts":[{"host":"meta","name":"Meta","kind":"maker","url":"https://dev.meta.ai/docs","inputPerMillion":1.25,"outputPerMillion":4.25,"cacheReadPerMillion":0.15,"contextTokens":1048576,"maxOutputTokens":131072,"deprecated":false},{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":1.25,"outputPerMillion":4.25,"cacheReadPerMillion":0.15,"contextTokens":1048576,"maxOutputTokens":943718,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:livebench","name":"LiveBench","url":"https://livebench.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:lmarena","name":"LMArena","url":"https://huggingface.co","license":"CC BY 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"muse-ember","providerId":"meta","slug":"muse-ember","name":"Muse Ember","provider":"Meta","description":"Muse Ember is a reasoning model from Meta. 9 benchmarks count toward its index, in 4 categories.","logoUrl":null,"releaseDate":"2026-11-09","contextWindowTokens":1048576,"modalities":["text","image","video","file","audio"],"openWeights":false,"details":{"apiId":"meta/muse-ember","maxOutputTokens":943718,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.004,"weightsUrl":null},"reasoning":true,"overallScore":64.80945554008542,"overallScoreRange":{"low":56.41945554008542,"high":73.19945554008542},"badges":[],"capabilityScores":{"agentic":71.44040647706044,"coding":72.45088006372123,"reasoning":63.37568797578815,"multimodal":null,"knowledge":37.281351730280804,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"coding":1,"agentic":6,"knowledge":1,"reasoning":1},"benchmarks":[{"benchmarkId":"vals-ai:vibe-code:overall","benchmarkName":"Vibe Code Bench v1.1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":83.703,"unit":"percent","family":"vals-ai:vibe-code:overall","index":72.45088006372123,"aboveFit":false,"run":"OpenHands","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","measuredAt":"2026-11-09","stderr":2.756,"costPerTest":0.054457,"latency":167.856},{"benchmarkId":"perceptionBenchTools","benchmarkName":"PerceptionBench with tools","category":"multimodal","score":80.6,"unit":"percent","family":"perceptionBenchTools","index":null,"aboveFit":false,"run":null,"source":"Meta (reported run)","sourceUrl":"https://research.meta.ai/blog/muse-ember","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"osWorld21","benchmarkName":"OSWorld 2.1 (all 108 tasks)","category":"agentic","score":69.5,"unit":"percent","family":"osWorld21","index":69.25766742815041,"aboveFit":false,"run":null,"source":"Mengqi Yuan, Zilong Zhou, Xinzhuang Xiong, Weiming Wu, Jiayang Sun, Jiamin Song, Kaiqian Cui, Bowen Wang, Haoyuan Wu, Yitong Li, Dunjie Lu, Haikong Lu, Qi Zhen, Xinyuan Wang, Jiaqi Deng, Yuhao Yang, Cheng Chen, Boyuan Zheng, Alex Su, Xiao Yu, Hao Zou, Saaket Agashe, Xing Han Lu, Manpreet Kaur, Zhengyang Qi, Vincent Sunn Chen, Frederic Sala, Dayiheng Liu, Junyang Lin, Zhou Yu, Yu Su, Siva Reddy, Xin Eric Wang, Peng Qi, Tianbao Xie, Tao Yu","sourceUrl":"https://github.com/xlang-ai/OSWorld-V2/releases","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaAutomationBench","benchmarkName":"Artificial Analysis AutomationBench","category":"agentic","score":68.1,"unit":"percent","family":"aaAutomationBench","index":77.76475908020012,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/automationbench-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"agentsLastExamLinux","benchmarkName":"Agents' Last Exam, Linux subset","category":"agentic","score":61.6,"unit":"percent","family":"agentsLastExamLinux","index":null,"aboveFit":false,"run":null,"source":"Meta (reported run)","sourceUrl":"https://research.meta.ai/blog/muse-ember","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:legal_research:overall","benchmarkName":"Legal Research Bench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":49.038,"unit":"percent","family":"vals-ai:legal_research:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_research","measuredAt":"2026-11-09","stderr":3.475,"costPerTest":0.056581,"latency":207.736},{"benchmarkId":"aaTerminalBench4","benchmarkName":"Artificial Analysis Terminal-Bench v4.0","category":"agentic","score":40.9,"unit":"percent","family":"aaTerminalBench4","index":70.49849315750521,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/terminalbench-v4-0","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:terminal-bench-4:overall","benchmarkName":"Terminal-Bench 4.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":39.394,"unit":"percent","family":"terminal-bench-4","index":73.22180651197013,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-4","measuredAt":"2026-11-09","stderr":2.314,"costPerTest":0.218381,"latency":1581.448},{"benchmarkId":"vals-ai:tax_agent_bench:overall","benchmarkName":"Tax Agent Bench","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":38.233,"unit":"percent","family":"vals-ai:tax_agent_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench","measuredAt":"2026-11-09","stderr":3.499,"costPerTest":0.031788,"latency":90.131},{"benchmarkId":"aaAutomationBenchStrict","benchmarkName":"Artificial Analysis AutomationBench, strict pass@1","category":"agentic","score":37.8,"unit":"percent","family":"aaAutomationBenchStrict","index":null,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/automationbench-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaFrontierCode2","benchmarkName":"FrontierCode 2.0, Artificial Analysis evaluation","category":"coding","score":27.2,"unit":"percent","family":"aaFrontierCode2","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:hlab:overall","benchmarkName":"HLAB-Vals-1.1","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":25,"unit":"percent","family":"vals-ai:hlab:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/hlab","measuredAt":"2026-11-09","stderr":3.695,"costPerTest":0.118448,"latency":353.605},{"benchmarkId":"aaGdpPdf","benchmarkName":"Artificial Analysis GDP.pdf","category":"agentic","score":22.8,"unit":"percent","family":"aaGdpPdf","index":69.55188743098628,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdp-pdf","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"omniscienceAccuracy","benchmarkName":"Artificial Analysis Omniscience Accuracy","category":"knowledge","score":21.1,"unit":"percent","family":"omniscienceAccuracy","index":37.281351730280804,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaMlcr","benchmarkName":"Medical Long Context Reasoning (MLCR-AA)","category":"reasoning","score":18.3,"unit":"percent","family":"aaMlcr","index":63.37568797578815,"aboveFit":false,"run":null,"source":"Wisedocs and Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mlcr-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lmarena:agent-steerability:overall","benchmarkName":"Agent Arena steerability","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":11.991782869518214,"unit":"index","family":"agent-arena","index":76.38992679710296,"aboveFit":false,"run":"max effort","source":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset","measuredAt":"2026-11-11","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lmarena:agent-task-outcome:overall","benchmarkName":"Agent Arena task outcome","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":-7.571643046357456,"unit":"index","family":"agent-arena","index":60.30572370999789,"aboveFit":false,"run":"max effort","source":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset","measuredAt":"2026-11-11","stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":0.09999999999999999,"outputPerMillion":0.5,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":158,"timeToFirstToken":2.1445,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":70,"benchmarkCount":9,"categoryCount":4,"confidence":"low","state":"available"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:lmarena","name":"LMArena","url":"https://huggingface.co","license":"CC BY 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gpt-5-4","providerId":"openai","slug":"gpt-5-4","name":"GPT-5.4","provider":"OpenAI","description":"GPT-5.4 is a reasoning model from OpenAI. 67 benchmarks count toward its index, in 7 categories.","logoUrl":null,"releaseDate":"2026-03-05","contextWindowTokens":1050000,"modalities":["text","image","file"],"openWeights":false,"details":{"apiId":"openai/gpt-5.4","maxOutputTokens":128000,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.25,"weightsUrl":null},"reasoning":true,"overallScore":64.77495681097328,"overallScoreRange":{"low":62.29947689573182,"high":67.25043672621473},"badges":[],"capabilityScores":{"agentic":62.09743481661368,"coding":61.80174958433039,"reasoning":69.03377995353796,"multimodal":62.11464147696071,"knowledge":66.37351588607899,"multilingual":null,"instruction":63.374959693185424,"math":67.96700533463483},"capabilityCounts":{"agentic":21,"math":7,"reasoning":10,"knowledge":8,"multimodal":8,"coding":11,"instruction":2},"benchmarks":[{"benchmarkId":"tau2Bench","benchmarkName":"τ²-Bench Tool-Agent-User Evaluation","category":"agentic","score":98.9,"unit":"percent","family":"tau2-bench","index":68.4168209143857,"aboveFit":false,"run":null,"source":"Victor Barres, Honghua Dong, Soham Ray, Xujie Si, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2506.07982","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:aime:overall","benchmarkName":"AIME","category":"math","domain":null,"eligible":true,"ranked":true,"score":96.667,"unit":"percent","family":"vals-ai:aime:overall","index":57.75046851414084,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/aime","measuredAt":"2026-04-16","stderr":0.534,"costPerTest":0.138186,"latency":121.466},{"benchmarkId":"arc-prize:v1-public-eval:overall","benchmarkName":"ARC-AGI-1 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":96.38,"unit":"percent","family":"arc-prize:v1-public-eval:overall","index":null,"aboveFit":false,"run":"xhigh effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.43,"latency":null},{"benchmarkId":"vals-ai:medqa:overall","benchmarkName":"MedQA","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":96.092,"unit":"percent","family":"vals-ai:medqa:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medqa","measuredAt":"2026-04-16","stderr":0.177,"costPerTest":0.043618,"latency":22.784},{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":95.3,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":64.36406343492784,"aboveFit":false,"run":"xhigh effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":3.2,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-math:overall","benchmarkName":"LiveBench Mathematics","category":"math","domain":null,"eligible":true,"ranked":true,"score":94.148,"unit":"percent","family":"livebench:livebench-math:overall","index":70.99070829654414,"aboveFit":false,"run":"xhigh effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Mathematics","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.16482005277044853,"latency":null},{"benchmarkId":"arc-prize:v1-semi-private:overall","benchmarkName":"ARC-AGI-1 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":93.67,"unit":"percent","family":"arc-prize:v1-semi-private:overall","index":70.0508396258025,"aboveFit":false,"run":"xhigh effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.62,"latency":null},{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":93.30000000000001,"unit":"percent","family":"gpqa-diamond","index":62.78131130786548,"aboveFit":false,"run":"xhigh effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.7999999999999998,"costPerTest":null,"latency":null},{"benchmarkId":"gpqa","benchmarkName":"Graduate-Level Google-Proof Q&A","category":"knowledge","score":92.8,"unit":"percent","family":"gpqa-diamond","index":62.322724970405574,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gpqaDiamond","benchmarkName":"GPQA Diamond","category":"knowledge","score":92.8,"unit":"percent","family":"gpqa-diamond","index":62.322724970405574,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":92,"unit":"percent","family":"gpqa-diamond","index":61.58898683046975,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:gpqa:overall","benchmarkName":"GPQA Diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":91.666,"unit":"percent","family":"gpqa-diamond","index":61.282651157046544,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","measuredAt":"2026-09-01","stderr":1.908,"costPerTest":0.20934,"latency":202.953},{"benchmarkId":"livebench:livebench-reasoning:overall","benchmarkName":"LiveBench Reasoning","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":88.1155,"unit":"percent","family":"livebench","index":75.7675356228713,"aboveFit":false,"run":"xhigh effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Reasoning","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.212619801980198,"latency":null},{"benchmarkId":"vals-ai:mmmu:overall","benchmarkName":"MMMU Pro","category":"multimodal","domain":null,"eligible":true,"ranked":true,"score":87.514,"unit":"percent","family":"mmmu-pro","index":69.04460799747429,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmmu","measuredAt":"2026-09-01","stderr":0.795,"costPerTest":0.047083,"latency":65.565},{"benchmarkId":"liveCodeBenchPro","benchmarkName":"LiveCodeBench Pro","category":"coding","score":87.5,"unit":"percent","family":"liveCodeBenchPro","index":null,"aboveFit":false,"run":null,"source":"LiveCodeBench Pro authors","sourceUrl":"https://arxiv.org/abs/2506.11928","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:mmlu_pro:overall","benchmarkName":"MMLU Pro","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":87.482,"unit":"percent","family":"mmlu-pro","index":54.6975771007188,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","measuredAt":"2026-09-01","stderr":0.421,"costPerTest":0.050888,"latency":28.875},{"benchmarkId":"vals-ai:legal_bench:overall","benchmarkName":"LegalBench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":86.044,"unit":"percent","family":"vals-ai:legal_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","measuredAt":"2026-10-13","stderr":0.414,"costPerTest":0.023053,"latency":27.787},{"benchmarkId":"screenSpotPro","benchmarkName":"ScreenSpot Pro","category":"multimodal","score":85.4,"unit":"percent","family":"screenSpotPro","index":65.210035435268,"aboveFit":false,"run":null,"source":"Kaixin Li, Ziyang Meng, Hongzhan Lin, Ziyang Luo, Yuchen Tian, Jing Ma, Zhiyong Huang, Tat-Seng Chua","sourceUrl":"https://arxiv.org/abs/2504.07981","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"reactNativeEvals","benchmarkName":"React Native Evals","category":"coding","score":85.3,"unit":"percent","family":"reactNativeEvals","index":62.50830214135037,"aboveFit":false,"run":null,"source":"Callstack","sourceUrl":"https://rn-evals.vercel.app/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"arc-prize:v2-public-eval:overall","benchmarkName":"ARC-AGI-2 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":84.17,"unit":"percent","family":"arc-prize:v2-public-eval:overall","index":null,"aboveFit":false,"run":"xhigh effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":1.57,"latency":null},{"benchmarkId":"vals-ai:lcb:overall","benchmarkName":"LiveCodeBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":84.141,"unit":"percent","family":"livecodebench","index":59.15902325301826,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","measuredAt":"2026-09-01","stderr":1.04,"costPerTest":0.329246,"latency":209.424},{"benchmarkId":"charxiv","benchmarkName":"CharXiv Reasoning","category":"multimodal","score":82.8,"unit":"percent","family":"charxiv","index":58.451870668319444,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"browseComp","benchmarkName":"BrowseComp","category":"agentic","score":82.7,"unit":"percent","family":"browseComp","index":65.24363200674136,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-language:overall","benchmarkName":"LiveBench Language","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":82.63366666666667,"unit":"percent","family":"livebench:livebench-language:overall","index":68.00168676727567,"aboveFit":false,"run":"xhigh effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Language","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.18986466666666668,"latency":null},{"benchmarkId":"mmmuProPython","benchmarkName":"MMMU-Pro with Python","category":"multimodal","score":82.1,"unit":"percent","family":"mmmuProPython","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":82,"unit":"percent","family":"lcr","index":63.55665255719484,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmmuPro","benchmarkName":"Massive Multi-discipline Multimodal Understanding Pro","category":"multimodal","score":81.2,"unit":"percent","family":"mmmu-pro","index":60.82749095551189,"aboveFit":false,"run":null,"source":"MMMU-Pro authors","sourceUrl":"https://arxiv.org/abs/2409.02813","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-2025-02-28-public:overall","benchmarkName":"FrontierMath-2025-02-28-Public","category":"math","domain":null,"eligible":false,"ranked":true,"score":80,"unit":"percent","family":"epoch-ai:epoch-frontiermath-2025-02-28-public:overall","index":null,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":13.333333333333334,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-data-analysis:overall","benchmarkName":"LiveBench Data Analysis","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":79.31333333333333,"unit":"percent","family":"livebench","index":64.36490968312329,"aboveFit":false,"run":"xhigh effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Data%20Analysis","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.2817753333333333,"latency":null},{"benchmarkId":"cyberGym","benchmarkName":"CyberGym","category":"agentic","score":79,"unit":"percent","family":"cyberGym","index":66.08809822097658,"aboveFit":false,"run":null,"source":"Zhun Wang, Tianneng Shi, Jingxuan He, Matthew Cai, Jialin Zhang, Dawn Song","sourceUrl":"https://www.cybergym.io/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tiers-1-3-v2-private:overall","benchmarkName":"FrontierMath-Tiers-1-3-v2-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":78.59649122807018,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tiers-1-3-v2-private:overall","index":72.2640939652923,"aboveFit":false,"run":"xhigh effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.4338000553262393,"costPerTest":null,"latency":null},{"benchmarkId":"aaMmmuPro","benchmarkName":"Artificial Analysis MMMU-Pro","category":"multimodal","score":78.4,"unit":"percent","family":"mmmu-pro","index":57.183536613178234,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmmu-pro","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:swebench:overall","benchmarkName":"SWE-bench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":78.2,"unit":"percent","family":"swe-bench-verified","index":58.90493011215636,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/swebench","measuredAt":"2026-09-01","stderr":1.848,"costPerTest":0.801134,"latency":307.118},{"benchmarkId":"vals-ai:medscribe:overall","benchmarkName":"MedScribe","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":77.549,"unit":"percent","family":"vals-ai:medscribe:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medscribe","measuredAt":"2026-10-29","stderr":3.316,"costPerTest":0.639282,"latency":282.914},{"benchmarkId":"livebench:livebench-coding:overall","benchmarkName":"LiveBench Coding","category":"coding","domain":null,"eligible":true,"ranked":true,"score":77.5415,"unit":"percent","family":"livebench:livebench-coding:overall","index":63.879281285626774,"aboveFit":false,"run":"xhigh effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Coding","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.2666008547008547,"latency":null},{"benchmarkId":"medXpertQaMm","benchmarkName":"MedXpertQA Multimodal","category":"multimodal","score":77.1,"unit":"percent","family":"medXpertQaMm","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-swe-bench-verified:overall","benchmarkName":"SWE-Bench verified","category":"coding","domain":null,"eligible":true,"ranked":true,"score":76.85950413223141,"unit":"percent","family":"swe-bench-verified","index":57.9162372268959,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.9189407003691976,"costPerTest":null,"latency":null},{"benchmarkId":"terminalBench2","benchmarkName":"Terminal-Bench 2.0","category":"agentic","score":75.1,"unit":"percent","family":"terminalBench2","index":67.31968536124259,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"osWorldVerified","benchmarkName":"OSWorld-Verified","category":"agentic","score":75,"unit":"percent","family":"osWorldVerified","index":61.0252901850093,"aboveFit":false,"run":null,"source":"Tianbao Xie, Danyang Zhang, Jixuan Chen, Xiaochuan Li, Siheng Zhao, Ruisheng Cao, Toh Jing Hua, Zhoujun Cheng, Dongchan Shin, Fangyu Lei, Yitao Liu, Yiheng Xu, Shuyan Zhou, Silvio Savarese, Caiming Xiong, Victor Zhong, Tao Yu","sourceUrl":"https://os-world.github.io/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:tax_eval_v2:overall","benchmarkName":"TaxEval v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":73.958,"unit":"percent","family":"vals-ai:tax_eval_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","measuredAt":"2026-09-01","stderr":0.868,"costPerTest":0.157344,"latency":78.717},{"benchmarkId":"arc-prize:v2-semi-private:overall","benchmarkName":"ARC-AGI-2 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":73.95,"unit":"percent","family":"arc-prize:v2-semi-private:overall","index":73.1415265394293,"aboveFit":false,"run":"xhigh effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":1.52,"latency":null},{"benchmarkId":"aaIfBench","benchmarkName":"Artificial Analysis IFBench","category":"instruction","score":73.9,"unit":"percent","family":"ifbench-family","index":58.083963827965185,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"deepSearchQa","benchmarkName":"DeepSearchQA","category":"agentic","score":73.6,"unit":"percent","family":"deepSearchQa","index":54.421159830214584,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lifeVal","benchmarkName":"LifeVal: everyday life tasks","category":"agentic","score":73,"unit":"percent","family":"lifeVal","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mcpAtlas","benchmarkName":"MCP Atlas","category":"agentic","score":70.6,"unit":"percent","family":"mcpAtlas","index":58.60401239427679,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-instruction-following:overall","benchmarkName":"LiveBench Instruction Following","category":"instruction","domain":null,"eligible":true,"ranked":true,"score":70.21674999999999,"unit":"percent","family":"livebench:livebench-instruction-following:overall","index":68.66595555840566,"aboveFit":false,"run":"xhigh effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=IF","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.25874699999999995,"latency":null},{"benchmarkId":"vals-ai:mortgage_tax:overall","benchmarkName":"MortgageTax","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":68.323,"unit":"percent","family":"vals-ai:mortgage_tax:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mortgage_tax","measuredAt":"2026-09-01","stderr":0.915,"costPerTest":0.103445,"latency":45.002},{"benchmarkId":"vals-ai:ioi-v1:overall","benchmarkName":"IOI v1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":67.834,"unit":"percent","family":"vals-ai:ioi-v1:overall","index":73.89167590050296,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/ioi-v1","measuredAt":"2026-08-09","stderr":9.867,"costPerTest":3.471332,"latency":2660.886},{"benchmarkId":"erqa","benchmarkName":"ERQA","category":"multimodal","score":65.4,"unit":"percent","family":"erqa","index":58.727346920881025,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:corp_fin_v2:overall","benchmarkName":"CorpFin v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":65.268,"unit":"percent","family":"vals-ai:corp_fin_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","measuredAt":"2026-08-12","stderr":0.938,"costPerTest":0.235266,"latency":82.34},{"benchmarkId":"gertLabs","benchmarkName":"Gert Labs Composite Game Benchmark","category":"agentic","score":64.89,"unit":"percent","family":"gertLabs","index":68.08927916510882,"aboveFit":false,"run":null,"source":"Gert Labs","sourceUrl":"https://gertlabs.com/rankings","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:case_law_v2:overall","benchmarkName":"CaseLaw v2","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":63.773,"unit":"percent","family":"vals-ai:case_law_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/case_law_v2","measuredAt":"2026-05-04","stderr":2.238,"costPerTest":0.100765,"latency":56.797},{"benchmarkId":"parseBenchLlamaIndex","benchmarkName":"ParseBench: document parsing for AI agents (LlamaIndex leaderboard)","category":"multimodal","score":61.72,"unit":"percent","family":"parseBenchLlamaIndex","index":59.83330100372034,"aboveFit":false,"run":null,"source":"Boyang Zhang, Sebastián G. Acosta, Preston Carlson, Sacha Bron, Pierre-Loïc Doulcet, Daniel B. Ospina and Simon Suo (LlamaIndex)","sourceUrl":"https://arxiv.org/abs/2604.08538","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"simpleVqa","benchmarkName":"SimpleVQA","category":"multimodal","score":61.1,"unit":"percent","family":"simpleVqa","index":52.03034625142607,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"clawEval","benchmarkName":"Claw-Eval","category":"agentic","score":60.3,"unit":"percent","family":"clawEval","index":52.312306658994416,"aboveFit":false,"run":null,"source":"Bowen Ye, Rang Li, Qibin Yang, Yuanxin Liu, Linli Yao, Hanglong Lv, Zhihui Xie, Chenxin An, Lei Li, Lingpeng Kong, Qi Liu, Zhifang Sui, Tong Yang","sourceUrl":"https://arxiv.org/abs/2604.06132","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"medXpertQaText","benchmarkName":"MedXpertQA Text","category":"knowledge","score":59.6,"unit":"percent","family":"medXpertQaText","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:terminal-bench-2:overall","benchmarkName":"Terminal-Bench 2.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":58.427,"unit":"percent","family":"vals-ai:terminal-bench-2:overall","index":61.730818847292845,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2","measuredAt":"2026-06-04","stderr":5.254,"costPerTest":0.511959,"latency":1027.517},{"benchmarkId":"swePro","benchmarkName":"SWE-bench Pro","category":"coding","score":57.7,"unit":"percent","family":"swePro","index":57.97727958846014,"aboveFit":false,"run":null,"source":"Xiang Deng, Jeff Da, Edwin Pan, Yannis Yiming He, Charles Ide, Kanak Garg, Niklas Lauffer, Andrew Park, Nitin Pasari, Chetan Rane, Karmini Sampath, Maya Krishnan, Srivatsa Kundurthy, Sean Hendryx, Zifan Wang, Chen Bo Calvin Zhang, Noah Jacobson, Bing Liu, Brad Kenstler","sourceUrl":"https://arxiv.org/abs/2509.16941","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"terminalBenchHard","benchmarkName":"Terminal-Bench Hard","category":"agentic","score":57.6,"unit":"percent","family":"terminalBenchHard","index":66.36780473916211,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/terminalbench-hard","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:finance_agent:overall","benchmarkName":"Finance Agent v1.1","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":57.152,"unit":"percent","family":"vals-ai:finance_agent:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/finance_agent","measuredAt":"2026-06-04","stderr":2.853,"costPerTest":1.412332,"latency":656.674},{"benchmarkId":"toolathlon","benchmarkName":"Toolathlon","category":"agentic","score":54.6,"unit":"percent","family":"toolathlon","index":66.37128649545055,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-agentic-coding:overall","benchmarkName":"LiveBench Agentic Coding","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":53.83833333333333,"unit":"percent","family":"livebench:livebench-agentic-coding:overall","index":64.68524657199846,"aboveFit":false,"run":"xhigh effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Agentic%20Coding","measuredAt":"2026-06-25","stderr":null,"costPerTest":1.728751388888889,"latency":null},{"benchmarkId":"officeQaPro","benchmarkName":"OfficeQA Pro","category":"multimodal","score":53.2,"unit":"percent","family":"officeQaPro","index":63.249935267920534,"aboveFit":false,"run":null,"source":"OfficeQA Pro authors","sourceUrl":"https://arxiv.org/abs/2603.08655","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hle","benchmarkName":"Humanity's Last Exam","category":"knowledge","score":52.1,"unit":"percent","family":"hle","index":76.65819632936577,"aboveFit":false,"run":null,"source":"Center for AI Safety, Scale AI, and thousands of expert contributors","sourceUrl":"https://lastexam.ai/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:skillsbench:overall","benchmarkName":"SkillsBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":51.711,"unit":"percent","family":"vals-ai:skillsbench:overall","index":63.63215530920876,"aboveFit":false,"run":"OpenHands","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/skillsbench","measuredAt":"2026-10-29","stderr":4.721,"costPerTest":2.044198,"latency":542.281},{"benchmarkId":"omniscienceAccuracy","benchmarkName":"Artificial Analysis Omniscience Accuracy","category":"knowledge","score":50.8,"unit":"percent","family":"omniscienceAccuracy","index":74.71095445684902,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-public:overall","benchmarkName":"FrontierMath-Tier-4-2025-07-01-Public","category":"math","domain":null,"eligible":false,"ranked":true,"score":50,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-public:overall","index":null,"aboveFit":false,"run":"xhigh effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":50,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tier-4-v2-private:overall","benchmarkName":"FrontierMath-Tier-4-v2-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":49,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tier-4-v2-private:overall","index":67.73795639808229,"aboveFit":false,"run":"xhigh effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":8,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:vibe-code:overall","benchmarkName":"Vibe Code Bench v1.1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":48.474,"unit":"percent","family":"vals-ai:vibe-code:overall","index":59.73970337461984,"aboveFit":false,"run":"Codex","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","measuredAt":"2026-11-09","stderr":5.07,"costPerTest":7.50518,"latency":1075.396},{"benchmarkId":"healthBenchProfessional","benchmarkName":"HealthBench Professional","category":"knowledge","score":48.1,"unit":"percent","family":"healthBenchProfessional","index":60.34142431175391,"aboveFit":false,"run":null,"source":"Rebecca Soskin Hicks, Mikhail Trofimov, Dominick Lim, Rahul K. Arora, Foivos Tsimpourlas, Preston Bowman, Michael Sharman, Chi Tong, Kavin Karthik, Arnav Dugar, Akshay Jagadeesh, Khaled Saab, Johannes Heidecke, Ashley Alexander, Nate Gross, Karan Singhal","sourceUrl":"https://arxiv.org/abs/2604.27470","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-chess-puzzles:overall","benchmarkName":"Chess Puzzles","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":48,"unit":"percent","family":"epoch-ai:epoch-chess-puzzles:overall","index":70.41740509234356,"aboveFit":false,"run":"xhigh effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":5.021167315686783,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-2025-02-28-private:overall","benchmarkName":"FrontierMath-2025-02-28-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":47.599999999999994,"unit":"percent","family":"epoch-ai:epoch-frontiermath-2025-02-28-private:overall","index":71.93438837256073,"aboveFit":false,"run":"xhigh effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.9000000000000004,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-simpleqa-verified:overall","benchmarkName":"SimpleQA Verified","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":45.1,"unit":"percent","family":"epoch-ai:epoch-simpleqa-verified:overall","index":61.491333659501024,"aboveFit":false,"run":"xhigh effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.5743152379585443,"costPerTest":null,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":43.7,"unit":"percent","family":"hle","index":68.2305372711155,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:medcode:overall","benchmarkName":"MedCode","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":41.292,"unit":"percent","family":"vals-ai:medcode:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medcode","measuredAt":"2026-10-29","stderr":2.148,"costPerTest":0.212108,"latency":187.265},{"benchmarkId":"zeroBench","benchmarkName":"ZeroBench","category":"multimodal","score":41,"unit":"percent","family":"zeroBench","index":78.58680531263838,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"healthBenchHard","benchmarkName":"HealthBench Hard","category":"knowledge","score":40.1,"unit":"percent","family":"healthBenchHard","index":81.19188855101247,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hleNoTools","benchmarkName":"Humanity's Last Exam without tools","category":"knowledge","score":39.8,"unit":"percent","family":"hle","index":64.31769556549929,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaTau3Banking","benchmarkName":"Artificial Analysis Tau3-Banking","category":"agentic","score":39.6,"unit":"percent","family":"aaTau3Banking","index":63.51087786376911,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/tau3-banking","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"tau2-bench:banking-knowledge:overall","benchmarkName":"τ²-bench Banking","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":39.43,"unit":"percent","family":"tau2-bench","index":28.95412794840779,"aboveFit":false,"run":"xhigh effort · Sierra","source":"Sierra Research","sourceUrl":"https://github.com/sierra-research/tau2-bench","measuredAt":"2026-10-20","stderr":null,"costPerTest":1.8423,"latency":null},{"benchmarkId":"jobBench","benchmarkName":"JobBench","category":"agentic","score":38.9,"unit":"percent","family":"jobBench","index":58.009148459656814,"aboveFit":false,"run":null,"source":"Yuetai Li, Yichen Feng, Zhangchen Xu, Zixian Ma, Kaiyuan Zheng, Fengqing Jiang, Xinghua Sun, Rulin Shao, Zichen Chen, Yue Huang, Xinyang Han, Brian Lee, Kayla Xu, Shenglai Zeng, Hang Hua, Xiangliang Zhang, Basel Alomair, Ranjay Krishna, Luke Zettlemoyer, Pang Wei Koh, Bhaskar Ramasubramanian, Luyao Niu, Xiang Yue, Radha Poovendran","sourceUrl":"https://arxiv.org/abs/2605.26329","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-furniture-assembly:overall","benchmarkName":"Furniture Assembly","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":37.5,"unit":"percent","family":"epoch-ai:epoch-furniture-assembly:overall","index":64.14514389375583,"aboveFit":false,"run":"xhigh effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":6.246467928511352,"costPerTest":null,"latency":null},{"benchmarkId":"gdpvalAaNormalized","benchmarkName":"GDPval-AA normalized","category":"agentic","score":37.3,"unit":"percent","family":"gdpvalAaNormalized","index":60.70240812829812,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-mystery-game-puzzles:overall","benchmarkName":"Mystery Game Puzzles","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":37,"unit":"percent","family":"epoch-ai:epoch-mystery-game-puzzles:overall","index":67.33434938016154,"aboveFit":false,"run":"xhigh effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":4.8523658709390975,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:code-migration:overall","benchmarkName":"Code Migration","category":"coding","domain":null,"eligible":true,"ranked":true,"score":34.984,"unit":"percent","family":"vals-ai:code-migration:overall","index":63.774040882684936,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/code-migration","measuredAt":"2026-10-29","stderr":4.113,"costPerTest":7.756561,"latency":2146.253},{"benchmarkId":"apexAgentsAa","benchmarkName":"APEX-Agents-AA","category":"agentic","score":33.3,"unit":"percent","family":"apexAgentsAa","index":64.9599144322905,"aboveFit":false,"run":null,"source":"Artificial Analysis / Mercor","sourceUrl":"https://artificialanalysis.ai/evaluations/apex-agents-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"bioBasicsBench","benchmarkName":"BioBasicsBench: everyday wet-lab science","category":"reasoning","score":33.2,"unit":"percent","family":"bioBasicsBench","index":null,"aboveFit":false,"run":null,"source":"OpenAI, Ligo","sourceUrl":"https://openai.com/index/measuring-how-ai-agents-handle-wet-lab-science/","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-private:overall","benchmarkName":"FrontierMath-Tier-4-2025-07-01-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":27.1,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-private:overall","index":70.72735836089568,"aboveFit":false,"run":"xhigh effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":6.4,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-ebr-bench:overall","benchmarkName":"EBR-bench","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":25.396800000000002,"unit":"percent","family":"epoch-ai:epoch-ebr-bench:overall","index":64.082065614699,"aboveFit":false,"run":"xhigh effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":23.4,"unit":"percent","family":"critpt","index":82.14878562352686,"aboveFit":true,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-mirrorcode:overall","benchmarkName":"MirrorCode","category":"coding","domain":null,"eligible":true,"ranked":true,"score":23.333333333333332,"unit":"percent","family":"epoch-ai:epoch-mirrorcode:overall","index":59.81297261588808,"aboveFit":false,"run":"xhigh effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":8.099905283919412,"costPerTest":null,"latency":null},{"benchmarkId":"postTrainBench11","benchmarkName":"PostTrainBench v1.1","category":"coding","score":19,"unit":"percent","family":"postTrainBench11","index":57.03422740674802,"aboveFit":false,"run":null,"source":"PostTrainBench team","sourceUrl":"https://posttrainbench.com/?version=v1.1","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"researchClawBench","benchmarkName":"ResearchClawBench","category":"agentic","score":15.3,"unit":"percent","family":"researchClawBench","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"apprenticeBench","benchmarkName":"ApprenticeBench: end-to-end computer use, continual learning, and long-horizon agency on a real accounts-payable job","category":"agentic","score":11,"unit":"percent","family":"apprenticeBench","index":63.78501614476527,"aboveFit":false,"run":null,"source":"NeoCognition","sourceUrl":"https://neocognition.io/blog/apprentice-bench/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"exploitGym","benchmarkName":"ExploitGym","category":"agentic","score":6,"unit":"percent","family":"exploitGym","index":62.62665909724513,"aboveFit":false,"run":null,"source":"Zhun Wang, Nico Schiller, Hongwei Li, Srijiith Sesha Narayana, Milad Nasr, Nicholas Carlini, Xiangyu Qi, Eric Wallace, Elie Bursztein, Luca Invernizzi, Kurt Thomas, Yan Shoshitaishvili, Wenbo Guo, Jingxuan He, Thorsten Holz, Dawn Song","sourceUrl":"https://arxiv.org/abs/2605.11086","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lmarena:agent-bash-recovery:overall","benchmarkName":"Agent Arena command recovery","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":1.2145414242588082,"unit":"index","family":"agent-arena","index":67.52934476664937,"aboveFit":false,"run":"high effort","source":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset","measuredAt":"2026-10-08","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:programbench:overall","benchmarkName":"ProgramBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":0.5,"unit":"percent","family":"vals-ai:programbench:overall","index":null,"aboveFit":false,"run":"high effort","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/programbench","measuredAt":"2026-11-09","stderr":0.5,"costPerTest":2.27429,"latency":958.495},{"benchmarkId":"arc-prize:v3-semi-private:overall","benchmarkName":"ARC-AGI-3 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":0.21,"unit":"percent","family":"arc-prize:v3-semi-private:overall","index":65.39480855546893,"aboveFit":false,"run":"high effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mazeBench","benchmarkName":"MazeBench: visual spatial reasoning in a 3D open world","category":"agentic","score":0,"unit":"percent","family":"mazeBench","index":61.97866734834783,"aboveFit":false,"run":null,"source":"Jonathan Pappas and David Pappas","sourceUrl":"https://mazebench.com/blog?post=maze-bench-results","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:programbench:overall","benchmarkName":"ProgramBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":0,"unit":"percent","family":"vals-ai:programbench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/programbench","measuredAt":"2026-11-09","stderr":0,"costPerTest":null,"latency":2386.411}],"price":{"inputPerMillion":2.5,"outputPerMillion":15,"batch":{"inputPerMillion":1.25,"outputPerMillion":7.5},"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":38,"timeToFirstToken":1.731,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":{"elo":1452.4739525653604,"eloUpper":1456.1397873392034,"eloLower":1448.808117791517,"votes":67105,"rank":49,"rankUpper":null,"rankLower":null,"arenaName":"GPT-5.4","organization":"openai","updatedAt":"2026-11-09"},"quality":{"coveragePercent":95,"benchmarkCount":67,"categoryCount":7,"confidence":"high","state":"available"},"hosts":[{"host":"databricks","name":"Databricks","kind":"cloud","url":"https://docs.databricks.com/aws/en/machine-learning/foundation-models/","inputPerMillion":2.5,"outputPerMillion":15,"cacheReadPerMillion":0.25,"contextTokens":1050000,"maxOutputTokens":128000,"deprecated":false},{"host":"digitalocean","name":"DigitalOcean","kind":"cloud","url":"https://docs.digitalocean.com/products/gradient-ai-platform/details/models/","inputPerMillion":2.5,"outputPerMillion":15,"cacheReadPerMillion":0.25,"contextTokens":400000,"maxOutputTokens":128000,"deprecated":false},{"host":"azure","name":"Microsoft Azure","kind":"cloud","url":"https://learn.microsoft.com/en-us/azure/ai-services/openai/concepts/models","inputPerMillion":2.5,"outputPerMillion":15,"cacheReadPerMillion":0.25,"contextTokens":1050000,"maxOutputTokens":128000,"deprecated":false},{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":2.5,"outputPerMillion":15,"cacheReadPerMillion":0.25,"contextTokens":1050000,"maxOutputTokens":128000,"deprecated":false},{"host":"sap-ai-core","name":"SAP AI Core","kind":"cloud","url":"https://help.sap.com/docs/sap-ai-core","inputPerMillion":2.5,"outputPerMillion":15,"cacheReadPerMillion":0.25,"contextTokens":1050000,"maxOutputTokens":128000,"deprecated":false},{"host":"snowflake","name":"Snowflake Cortex","kind":"cloud","url":"https://docs.snowflake.com/en/user-guide/snowflake-cortex/cortex-rest-api","inputPerMillion":2.5,"outputPerMillion":15,"cacheReadPerMillion":0.25,"contextTokens":1050000,"maxOutputTokens":128000,"deprecated":false},{"host":"amazon-bedrock","name":"Amazon Bedrock","kind":"cloud","url":"https://docs.aws.amazon.com/bedrock/latest/userguide/models-supported.html","inputPerMillion":2.75,"outputPerMillion":16.5,"cacheReadPerMillion":0.275,"contextTokens":1000000,"maxOutputTokens":128000,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:livebench","name":"LiveBench","url":"https://livebench.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:arc-prize-foundation","name":"ARC Prize Foundation","url":"https://arcprize.org","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:sierra-research","name":"Sierra Research","url":"https://github.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:lmarena","name":"LMArena","url":"https://huggingface.co","license":"CC BY 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"muse-spark-1-1","providerId":"meta","slug":"muse-spark-1-1","name":"Muse Spark 1.1","provider":"Meta","description":"Muse Spark 1.1 is a reasoning model from Meta in the Muse Spark family. 38 benchmarks count toward its index, in 7 categories.","logoUrl":null,"releaseDate":"2026-07-09","contextWindowTokens":1048576,"modalities":["text","image","video","file"],"openWeights":false,"details":{"apiId":"meta/muse-spark-1.1","maxOutputTokens":943718,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.15,"weightsUrl":null},"reasoning":true,"overallScore":64.61205410855652,"overallScoreRange":{"low":61.48893372887209,"high":67.73517448824096},"badges":[],"capabilityScores":{"agentic":61.853434082607805,"coding":64.82297423143984,"reasoning":62.978411223175414,"multimodal":66.46624090262156,"knowledge":67.41745559021427,"multilingual":null,"instruction":67.4938664281637,"math":61.230143654385095},"capabilityCounts":{"knowledge":7,"multimodal":3,"agentic":14,"reasoning":4,"math":1,"coding":8,"instruction":1},"benchmarks":[{"benchmarkId":"cybench","benchmarkName":"Cybench","category":"agentic","score":92.9,"unit":"percent","family":"cybench","index":null,"aboveFit":false,"run":null,"source":"Andy K. Zhang, Neil Perry, Riya Dulepet, Joey Ji, Celeste Menders, Justin W. Lin, Eliot Jones, Gashon Hussein, Samantha Liu, Donovan Jasper, Pura Peetathawatchai, Ari Glenn, Vikram Sivashankar, Daniel Zamoshchin, Leo Glikbarg, Derek Askaryar, Mike Yang, Teddy Zhang, Rishi Alluri, Nathan Tran, Rinnara Sangpisit, Polycarpos Yiorkadjis, Kenny Osele, Gautham Raghupathi, Dan Boneh, Daniel E. Ho, Percy Liang","sourceUrl":"https://arxiv.org/abs/2408.08926","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:gpqa:overall","benchmarkName":"GPQA Diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":91.162,"unit":"percent","family":"gpqa-diamond","index":60.820396128886976,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","measuredAt":"2026-09-01","stderr":2.104,"costPerTest":0.034304,"latency":75.198},{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":89.8,"unit":"percent","family":"gpqa-diamond","index":59.571206945646225,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:medscribe:overall","benchmarkName":"MedScribe","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":88.888,"unit":"percent","family":"vals-ai:medscribe:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medscribe","measuredAt":"2026-10-29","stderr":1.95,"costPerTest":0.034628,"latency":63.335},{"benchmarkId":"vals-ai:mmlu_pro:overall","benchmarkName":"MMLU Pro","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":88.726,"unit":"percent","family":"mmlu-pro","index":56.26486819774644,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","measuredAt":"2026-09-01","stderr":0.318,"costPerTest":0.010704,"latency":21.03},{"benchmarkId":"charxiv","benchmarkName":"CharXiv Reasoning","category":"multimodal","score":88.4,"unit":"percent","family":"charxiv","index":64.82746185149361,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mcpAtlas","benchmarkName":"MCP Atlas","category":"agentic","score":88.1,"unit":"percent","family":"mcpAtlas","index":70.38843532553298,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-reasoning:overall","benchmarkName":"LiveBench Reasoning","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":87.73075,"unit":"percent","family":"livebench","index":75.26911739285391,"aboveFit":false,"run":"xhigh effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Reasoning","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.07460346534653466,"latency":null},{"benchmarkId":"livebench:livebench-math:overall","benchmarkName":"LiveBench Mathematics","category":"math","domain":null,"eligible":true,"ranked":true,"score":87.14474999999999,"unit":"percent","family":"livebench:livebench-math:overall","index":61.230143654385095,"aboveFit":false,"run":"xhigh effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Mathematics","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.045640897097625324,"latency":null},{"benchmarkId":"vals-ai:mmmu:overall","benchmarkName":"MMMU Pro","category":"multimodal","domain":null,"eligible":true,"ranked":true,"score":86.59,"unit":"percent","family":"mmmu-pro","index":67.84210306450419,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmmu","measuredAt":"2026-09-01","stderr":0.819,"costPerTest":0.01361,"latency":31.715},{"benchmarkId":"vals-ai:lcb:overall","benchmarkName":"LiveCodeBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":85.906,"unit":"percent","family":"livecodebench","index":60.62351652220809,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","measuredAt":"2026-09-01","stderr":0.992,"costPerTest":0.046497,"latency":113.253},{"benchmarkId":"vals-ai:legal_bench:overall","benchmarkName":"LegalBench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":84.981,"unit":"percent","family":"vals-ai:legal_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","measuredAt":"2026-10-13","stderr":0.425,"costPerTest":0.008535,"latency":17.148},{"benchmarkId":"deepSearchQa","benchmarkName":"DeepSearchQA","category":"agentic","score":84.9,"unit":"percent","family":"deepSearchQa","index":62.35584052361233,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:swebench:overall","benchmarkName":"SWE-bench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":82,"unit":"percent","family":"swe-bench-verified","index":61.70764890773438,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/swebench","measuredAt":"2026-09-01","stderr":1.72,"costPerTest":0.352022,"latency":209.115},{"benchmarkId":"osWorldVerified","benchmarkName":"OSWorld-Verified","category":"agentic","score":80.8,"unit":"percent","family":"osWorldVerified","index":65.70738113113318,"aboveFit":false,"run":null,"source":"Tianbao Xie, Danyang Zhang, Jixuan Chen, Xiaochuan Li, Siheng Zhao, Ruisheng Cao, Toh Jing Hua, Zhoujun Cheng, Dongchan Shin, Fangyu Lei, Yitao Liu, Yiheng Xu, Shuyan Zhou, Silvio Savarese, Caiming Xiong, Victor Zhong, Tao Yu","sourceUrl":"https://os-world.github.io/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"terminalBench21","benchmarkName":"Terminal-Bench 2.1","category":"agentic","score":80,"unit":"percent","family":"terminal-bench-2-1","index":67.92955036519322,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:tax_eval_v2:overall","benchmarkName":"TaxEval v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":79.722,"unit":"percent","family":"vals-ai:tax_eval_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","measuredAt":"2026-09-01","stderr":0.77,"costPerTest":0.014921,"latency":32.816},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":77.7,"unit":"percent","family":"lcr","index":60.6065062971765,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-coding:overall","benchmarkName":"LiveBench Coding","category":"coding","domain":null,"eligible":true,"ranked":true,"score":77.1585,"unit":"percent","family":"livebench:livebench-coding:overall","index":63.340634835811045,"aboveFit":false,"run":"xhigh effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Coding","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.024137606837606837,"latency":null},{"benchmarkId":"babyVision","benchmarkName":"BabyVision","category":"multimodal","score":76.3,"unit":"percent","family":"babyVision","index":66.72915779186692,"aboveFit":false,"run":null,"source":"Meta AI","sourceUrl":"https://ai.meta.com/static-resource/muse-spark-1-1-evaluation-report","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"toolathlon","benchmarkName":"Toolathlon","category":"agentic","score":75.6,"unit":"percent","family":"toolathlon","index":83.27004042670833,"aboveFit":true,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-language:overall","benchmarkName":"LiveBench Language","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":74.342,"unit":"percent","family":"livebench:livebench-language:overall","index":58.62539018927968,"aboveFit":false,"run":"xhigh effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Language","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.04532200000000001,"latency":null},{"benchmarkId":"livebench:livebench-data-analysis:overall","benchmarkName":"LiveBench Data Analysis","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":72.548,"unit":"percent","family":"livebench","index":55.60086688785979,"aboveFit":false,"run":"xhigh effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Data%20Analysis","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.087326,"latency":null},{"benchmarkId":"vals-ai:vibe-code:overall","benchmarkName":"Vibe Code Bench v1.1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":72.158,"unit":"percent","family":"vals-ai:vibe-code:overall","index":68.28526267816741,"aboveFit":false,"run":"OpenHands","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","measuredAt":"2026-11-09","stderr":3.327,"costPerTest":0.923283,"latency":615.428},{"benchmarkId":"vals-ai:corp_fin_v2:overall","benchmarkName":"CorpFin v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":71.29,"unit":"percent","family":"vals-ai:corp_fin_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","measuredAt":"2026-08-12","stderr":0.889,"costPerTest":0.10923,"latency":27.823},{"benchmarkId":"livebench:livebench-instruction-following:overall","benchmarkName":"LiveBench Instruction Following","category":"instruction","domain":null,"eligible":true,"ranked":true,"score":69.6375,"unit":"percent","family":"livebench:livebench-instruction-following:overall","index":67.4938664281637,"aboveFit":false,"run":"xhigh effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=IF","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.046748000000000005,"latency":null},{"benchmarkId":"vals-ai:terminal-bench-2-1:overall","benchmarkName":"Terminal-Bench 2.1","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":69.288,"unit":"percent","family":"terminal-bench-2-1","index":61.66457970095868,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2-1","measuredAt":"2026-10-01","stderr":0.991,"costPerTest":0.417396,"latency":624.772},{"benchmarkId":"webArenaVerified","benchmarkName":"WebArena-Verified Browser Agent Benchmark","category":"agentic","score":69,"unit":"percent","family":"webArenaVerified","index":null,"aboveFit":false,"run":null,"source":"Amine El Hattami, Megh Thakkar, Nicolas Chapados, Christopher Pal","sourceUrl":"https://openreview.net/forum?id=94tlGxmqkN","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:mortgage_tax:overall","benchmarkName":"MortgageTax","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":66.136,"unit":"percent","family":"vals-ai:mortgage_tax:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mortgage_tax","measuredAt":"2026-09-01","stderr":0.928,"costPerTest":0.022312,"latency":24.818},{"benchmarkId":"hle","benchmarkName":"Humanity's Last Exam","category":"knowledge","score":62.1,"unit":"percent","family":"hle","index":86.69112377966371,"aboveFit":true,"run":null,"source":"Center for AI Safety, Scale AI, and thousands of expert contributors","sourceUrl":"https://lastexam.ai/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"swePro","benchmarkName":"SWE-bench Pro","category":"coding","score":61.5,"unit":"percent","family":"swePro","index":61.213397516974474,"aboveFit":false,"run":null,"source":"Xiang Deng, Jeff Da, Edwin Pan, Yannis Yiming He, Charles Ide, Kanak Garg, Niklas Lauffer, Andrew Park, Nitin Pasari, Chetan Rane, Karmini Sampath, Maya Krishnan, Srivatsa Kundurthy, Sean Hendryx, Zifan Wang, Chen Bo Calvin Zhang, Noah Jacobson, Bing Liu, Brad Kenstler","sourceUrl":"https://arxiv.org/abs/2509.16941","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:public-benefits-bench:overall","benchmarkName":"Public Benefits Bench v1.1","category":null,"domain":"Public benefits","eligible":false,"ranked":true,"score":61.299,"unit":"percent","family":"vals-ai:public-benefits-bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/public-benefits-bench","measuredAt":"2026-10-12","stderr":1.267,"costPerTest":0.278428,"latency":395.054},{"benchmarkId":"healthBenchProfessional","benchmarkName":"HealthBench Professional","category":"knowledge","score":59.3,"unit":"percent","family":"healthBenchProfessional","index":71.66597159417246,"aboveFit":false,"run":null,"source":"Rebecca Soskin Hicks, Mikhail Trofimov, Dominick Lim, Rahul K. Arora, Foivos Tsimpourlas, Preston Bowman, Michael Sharman, Chi Tong, Kavin Karthik, Arnav Dugar, Akshay Jagadeesh, Khaled Saab, Johannes Heidecke, Ashley Alexander, Nate Gross, Karan Singhal","sourceUrl":"https://arxiv.org/abs/2604.27470","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:skillsbench:overall","benchmarkName":"SkillsBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":59.189,"unit":"percent","family":"vals-ai:skillsbench:overall","index":69.19782257877162,"aboveFit":false,"run":"OpenHands","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/skillsbench","measuredAt":"2026-10-29","stderr":4.444,"costPerTest":0.449432,"latency":376.268},{"benchmarkId":"cyberGym","benchmarkName":"CyberGym","category":"agentic","score":59,"unit":"percent","family":"cyberGym","index":53.72374206261905,"aboveFit":false,"run":null,"source":"Zhun Wang, Tianneng Shi, Jingxuan He, Matthew Cai, Jialin Zhang, Dawn Song","sourceUrl":"https://www.cybergym.io/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaSciCode","benchmarkName":"Artificial Analysis SciCode","category":"coding","score":58.8,"unit":"percent","family":"scicode-family","index":72.49683785169756,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/scicode","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-agentic-coding:overall","benchmarkName":"LiveBench Agentic Coding","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":58.535333333333334,"unit":"percent","family":"livebench:livebench-agentic-coding:overall","index":69.52918267419484,"aboveFit":false,"run":"xhigh effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Agentic%20Coding","measuredAt":"2026-06-25","stderr":null,"costPerTest":1.7314416666666665,"latency":null},{"benchmarkId":"epoch-ai:epoch-simpleqa-verified:overall","benchmarkName":"SimpleQA Verified","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":57.78894472361809,"unit":"percent","family":"epoch-ai:epoch-simpleqa-verified:overall","index":72.06234477303954,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.5665429072238852,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:fabv2:overall","benchmarkName":"Finance Agent (v2)","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":57.207,"unit":"percent","family":"vals-ai:fabv2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/fabv2","measuredAt":"2026-10-29","stderr":0.789,"costPerTest":0.723391,"latency":414.77},{"benchmarkId":"vals-ai:emb:overall","benchmarkName":"EMB","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":56.361,"unit":"percent","family":"vals-ai:emb:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/emb","measuredAt":"2026-10-20","stderr":3.153,"costPerTest":1.112498,"latency":891.512},{"benchmarkId":"jobBench","benchmarkName":"JobBench","category":"agentic","score":54.7,"unit":"percent","family":"jobBench","index":66.74811693223737,"aboveFit":false,"run":null,"source":"Yuetai Li, Yichen Feng, Zhangchen Xu, Zixian Ma, Kaiyuan Zheng, Fengqing Jiang, Xinghua Sun, Rulin Shao, Zichen Chen, Yue Huang, Xinyang Han, Brian Lee, Kayla Xu, Shenglai Zeng, Hang Hua, Xiangliang Zhang, Basel Alomair, Ranjay Krishna, Luke Zettlemoyer, Pang Wei Koh, Bhaskar Ramasubramanian, Luyao Niu, Xiang Yue, Radha Poovendran","sourceUrl":"https://arxiv.org/abs/2605.26329","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mrcr1m","benchmarkName":"MRCR 1M","category":"reasoning","score":54.1,"unit":"percent","family":"mrcr1m","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"deepSwe","benchmarkName":"DeepSWE","category":"agentic","score":53.3,"unit":"percent","family":"deepSwe","index":59.824332709743935,"aboveFit":false,"run":null,"source":"Datacurve AI","sourceUrl":"https://deepswe.datacurve.ai/blog","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hleNoTools","benchmarkName":"Humanity's Last Exam without tools","category":"knowledge","score":52.2,"unit":"percent","family":"hle","index":76.75852560386875,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"omniscienceAccuracy","benchmarkName":"Artificial Analysis Omniscience Accuracy","category":"knowledge","score":52.1,"unit":"percent","family":"omniscienceAccuracy","index":76.34928723612643,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":46.2,"unit":"percent","family":"hle","index":70.73876913368997,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"tau2-bench:banking-knowledge:overall","benchmarkName":"τ²-bench Banking","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":40.4639175257732,"unit":"percent","family":"tau2-bench","index":29.6402078184554,"aboveFit":false,"run":"xhigh effort · Sierra","source":"Sierra Research","sourceUrl":"https://github.com/sierra-research/tau2-bench","measuredAt":"2026-10-20","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:legal_research:overall","benchmarkName":"Legal Research Bench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":37.981,"unit":"percent","family":"vals-ai:legal_research:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_research","measuredAt":"2026-11-09","stderr":3.373,"costPerTest":0.377379,"latency":312.994},{"benchmarkId":"gdpvalAaNormalized","benchmarkName":"GDPval-AA normalized","category":"agentic","score":35.6,"unit":"percent","family":"gdpvalAaNormalized","index":59.486947633422005,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-chess-puzzles:overall","benchmarkName":"Chess Puzzles","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":34,"unit":"percent","family":"epoch-ai:epoch-chess-puzzles:overall","index":59.791192360242206,"aboveFit":false,"run":"xhigh effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":4.760952285695233,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:code-migration:overall","benchmarkName":"Code Migration","category":"coding","domain":null,"eligible":true,"ranked":true,"score":31.115,"unit":"percent","family":"vals-ai:code-migration:overall","index":61.71867296015419,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/code-migration","measuredAt":"2026-10-29","stderr":4.068,"costPerTest":4.560329,"latency":2351.314},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":15.1,"unit":"percent","family":"critpt","index":66.08095409492611,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"osWorld2","benchmarkName":"OSWorld 2.0","category":"agentic","score":14.2,"unit":"percent","family":"osWorld2","index":59.81265109419971,"aboveFit":false,"run":null,"source":"Mengqi Yuan, Zilong Zhou, Xinzhuang Xiong, Weiming Wu, Jiayang Sun, Jiamin Song, Kaiqian Cui, Bowen Wang, Haoyuan Wu, Yitong Li, Dunjie Lu, Haikong Lu, Qi Zhen, Xinyuan Wang, Jiaqi Deng, Yuhao Yang, Cheng Chen, Boyuan Zheng, Alex Su, Xiao Yu, Hao Zou, Saaket Agashe, Xing Han Lu, Manpreet Kaur, Zhengyang Qi, Vincent Sunn Chen, Frederic Sala, Dayiheng Liu, Junyang Lin, Zhou Yu, Yu Su, Siva Reddy, Xin Eric Wang, Peng Qi, Tianbao Xie, Tao Yu","sourceUrl":"https://arxiv.org/abs/2606.29537","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:time_horizon_index:overall","benchmarkName":"Time Horizon Index: KSP","category":null,"domain":"Other","eligible":false,"ranked":true,"score":1.667,"unit":"percent","family":"vals-ai:time_horizon_index:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/time_horizon_index","measuredAt":"2026-10-07","stderr":0,"costPerTest":144.921195,"latency":0},{"benchmarkId":"exploitGym","benchmarkName":"ExploitGym","category":"agentic","score":0.8,"unit":"percent","family":"exploitGym","index":59.47121024507398,"aboveFit":false,"run":null,"source":"Zhun Wang, Nico Schiller, Hongwei Li, Srijiith Sesha Narayana, Milad Nasr, Nicholas Carlini, Xiangyu Qi, Eric Wallace, Elie Bursztein, Luca Invernizzi, Kurt Thomas, Yan Shoshitaishvili, Wenbo Guo, Jingxuan He, Thorsten Holz, Dawn Song","sourceUrl":"https://arxiv.org/abs/2605.11086","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:programbench:overall","benchmarkName":"ProgramBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":0,"unit":"percent","family":"vals-ai:programbench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/programbench","measuredAt":"2026-11-09","stderr":0,"costPerTest":13.063498,"latency":20592.047},{"benchmarkId":"lmarena:agent-bash-recovery:overall","benchmarkName":"Agent Arena command recovery","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":-2.258230890923607,"unit":"index","family":"agent-arena","index":64.67418149244276,"aboveFit":false,"run":null,"source":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset","measuredAt":"2026-10-08","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lmarena:agent-task-outcome:overall","benchmarkName":"Agent Arena task outcome","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":-6.492530318173529,"unit":"index","family":"agent-arena","index":61.19292354650022,"aboveFit":false,"run":null,"source":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset","measuredAt":"2026-11-11","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lmarena:agent-steerability:overall","benchmarkName":"Agent Arena steerability","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":-15.448231684794367,"unit":"index","family":"agent-arena","index":53.82993302594412,"aboveFit":false,"run":null,"source":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset","measuredAt":"2026-11-11","stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":1.25,"outputPerMillion":4.25,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":152,"timeToFirstToken":2.079,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":{"elo":1478.7277946321105,"eloUpper":1482.8738304477579,"eloLower":1474.5817588164632,"votes":45562,"rank":23,"rankUpper":null,"rankLower":null,"arenaName":"Muse Spark 1.1","organization":"meta","updatedAt":"2026-11-09"},"quality":{"coveragePercent":95,"benchmarkCount":38,"categoryCount":7,"confidence":"high","state":"available"},"hosts":[{"host":"meta","name":"Meta","kind":"maker","url":"https://dev.meta.ai/docs","inputPerMillion":1.25,"outputPerMillion":4.25,"cacheReadPerMillion":0.15,"contextTokens":1048576,"maxOutputTokens":131072,"deprecated":false},{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":1.25,"outputPerMillion":4.25,"cacheReadPerMillion":0.15,"contextTokens":1048576,"maxOutputTokens":943718,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:livebench","name":"LiveBench","url":"https://livebench.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:sierra-research","name":"Sierra Research","url":"https://github.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:lmarena","name":"LMArena","url":"https://huggingface.co","license":"CC BY 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"qwen3-8-flash-next","providerId":"alibaba","slug":"qwen3-8-flash-next","name":"Qwen3.8-Flash-Next","provider":"Alibaba","description":"Qwen3.8-Flash-Next is a reasoning model from Alibaba. 35 benchmarks count toward its index, in 7 categories.","logoUrl":null,"releaseDate":"2026-08-26","contextWindowTokens":262000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":true,"overallScore":64.28058633546587,"overallScoreRange":{"low":61.04735747567385,"high":67.51381519525789},"badges":[],"capabilityScores":{"agentic":68.5245281699125,"coding":62.129496968998325,"reasoning":62.20655627041717,"multimodal":64.9753630389963,"knowledge":55.2476047175205,"multilingual":null,"instruction":73.8052387508299,"math":59.38520773962297},"capabilityCounts":{"knowledge":4,"coding":6,"multimodal":7,"reasoning":3,"agentic":12,"math":1,"instruction":2},"benchmarks":[{"benchmarkId":"mathVisionPython","benchmarkName":"MathVision with Python","category":"multimodal","score":95.7,"unit":"percent","family":"mathVisionPython","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":92.3,"unit":"percent","family":"gpqa-diamond","index":61.86413863294568,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"liveCodeBenchV6","benchmarkName":"LiveCodeBench v6","category":"coding","score":91.9,"unit":"percent","family":"liveCodeBenchV6","index":59.87222292401802,"aboveFit":false,"run":null,"source":"LiveCodeBench maintainers","sourceUrl":"https://github.com/LiveCodeBench/LiveCodeBench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gpqa","benchmarkName":"Graduate-Level Google-Proof Q&A","category":"knowledge","score":91.7,"unit":"percent","family":"gpqa-diamond","index":61.31383502799382,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gpqaDiamond","benchmarkName":"GPQA Diamond","category":"knowledge","score":91.7,"unit":"percent","family":"gpqa-diamond","index":61.31383502799382,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mathVision","benchmarkName":"MathVision","category":"multimodal","score":90.6,"unit":"percent","family":"mathVision","index":61.46112542496196,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"charxiv","benchmarkName":"CharXiv Reasoning","category":"multimodal","score":90.6,"unit":"percent","family":"charxiv","index":67.33215838774059,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"extractBenchLlamaIndex","benchmarkName":"ExtractBench: schema-guided enterprise document extraction (LlamaIndex leaderboard)","category":"multimodal","score":89.88,"unit":"percent","family":"extractBenchLlamaIndex","index":71.33363944332514,"aboveFit":false,"run":null,"source":"Boyang Zhang, Adrian Lyjak, Eli Stewart, Zhaoqi Li and Simon Suo (LlamaIndex)","sourceUrl":"https://arxiv.org/abs/2607.29677","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"realWorldQa","benchmarkName":"RealWorldQA","category":"multimodal","score":88.5,"unit":"percent","family":"realWorldQa","index":65.2165039138496,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-coding-2:overall","benchmarkName":"LiveBench Coding 2.0","category":"coding","domain":null,"eligible":true,"ranked":true,"score":87.4,"unit":"percent","family":"livebench:livebench-coding-2:overall","index":71.47793052151347,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Coding","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.10613793103448275,"latency":null},{"benchmarkId":"livebench:livebench-reasoning:overall","benchmarkName":"LiveBench Reasoning","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":87.37975,"unit":"percent","family":"livebench","index":74.81442006020649,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Reasoning","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.02200049504950495,"latency":null},{"benchmarkId":"aaTerminalBench21","benchmarkName":"Artificial Analysis Terminal-Bench v2.1","category":"agentic","score":86.1,"unit":"percent","family":"aaTerminalBench21","index":68.72207388186816,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/terminalbench-v2-1","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-math:overall","benchmarkName":"LiveBench Mathematics","category":"math","domain":null,"eligible":true,"ranked":true,"score":85.821,"unit":"percent","family":"livebench:livebench-math:overall","index":59.38520773962297,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Mathematics","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.021559366754617414,"latency":null},{"benchmarkId":"charxivNoTools","benchmarkName":"CharXiv Reasoning without tools","category":"multimodal","score":84.6,"unit":"percent","family":"charxivNoTools","index":65.36354651907244,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"androidWorld","benchmarkName":"AndroidWorld","category":"agentic","score":84.5,"unit":"percent","family":"androidWorld","index":63.82071958166035,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"ifBench","benchmarkName":"Instruction Following Benchmark","category":"instruction","score":81.3,"unit":"percent","family":"ifbench-family","index":64.9993423777035,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaMmmuPro","benchmarkName":"Artificial Analysis MMMU-Pro","category":"multimodal","score":79.8,"unit":"percent","family":"mmmu-pro","index":59.005513784345055,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmmu-pro","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":79.7,"unit":"percent","family":"lcr","index":61.97866734834783,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-instruction-following:overall","benchmarkName":"LiveBench Instruction Following","category":"instruction","domain":null,"eligible":true,"ranked":true,"score":77.10849999999999,"unit":"percent","family":"livebench:livebench-instruction-following:overall","index":82.6111351239563,"aboveFit":true,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=IF","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.017534499999999998,"latency":null},{"benchmarkId":"lvBench","benchmarkName":"LVBench","category":"multimodal","score":76.6,"unit":"percent","family":"lvBench","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-language:overall","benchmarkName":"LiveBench Language","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":74.643,"unit":"percent","family":"livebench:livebench-language:overall","index":58.96576389018141,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Language","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.025377999999999998,"latency":null},{"benchmarkId":"livebench:livebench-research:overall","benchmarkName":"LiveBench Research","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":74.6,"unit":"percent","family":"livebench-2-agentic","index":73.14677126343629,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Research","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.31680454545454545,"latency":null},{"benchmarkId":"livebench:livebench-data-analysis:overall","benchmarkName":"LiveBench Data Analysis","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":74.24,"unit":"percent","family":"livebench","index":57.79274120933975,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Data%20Analysis","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.04350733333333333,"latency":null},{"benchmarkId":"coworkBench","benchmarkName":"CoWorkBench","category":"agentic","score":73.9,"unit":"percent","family":"coworkBench","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"toolathlonVerified","benchmarkName":"Toolathlon-Verified","category":"agentic","score":73.5,"unit":"percent","family":"toolathlonVerified","index":68.28301609189548,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-design:overall","benchmarkName":"LiveBench Design","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":72.8,"unit":"percent","family":"livebench-2-agentic","index":71.83037529257948,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Design","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.128072,"latency":null},{"benchmarkId":"livebench:livebench-coding:overall","benchmarkName":"LiveBench Coding","category":"coding","domain":null,"eligible":true,"ranked":true,"score":72.5505,"unit":"percent","family":"livebench:livebench-coding:overall","index":56.86000078685302,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Coding","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.028460683760683763,"latency":null},{"benchmarkId":"erqa","benchmarkName":"ERQA","category":"multimodal","score":72.3,"unit":"percent","family":"erqa","index":65.11505379967929,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vision2Web","benchmarkName":"Vision2Web","category":"multimodal","score":64,"unit":"percent","family":"vision2Web","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"swePro","benchmarkName":"SWE-bench Pro","category":"coding","score":62.5,"unit":"percent","family":"swePro","index":62.06500749816246,"aboveFit":false,"run":null,"source":"Xiang Deng, Jeff Da, Edwin Pan, Yannis Yiming He, Charles Ide, Kanak Garg, Niklas Lauffer, Andrew Park, Nitin Pasari, Chetan Rane, Karmini Sampath, Maya Krishnan, Srivatsa Kundurthy, Sean Hendryx, Zifan Wang, Chen Bo Calvin Zhang, Noah Jacobson, Bing Liu, Brad Kenstler","sourceUrl":"https://arxiv.org/abs/2509.16941","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-agentic-coding:overall","benchmarkName":"LiveBench Agentic Coding","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":61.61633333333333,"unit":"percent","family":"livebench:livebench-agentic-coding:overall","index":72.70656549958701,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Agentic%20Coding","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.15427916666666666,"latency":null},{"benchmarkId":"deepSwe","benchmarkName":"DeepSWE","category":"agentic","score":58.7,"unit":"percent","family":"deepSwe","index":63.7499819633113,"aboveFit":false,"run":null,"source":"Datacurve AI","sourceUrl":"https://deepswe.datacurve.ai/blog","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gdpvalAaNormalized","benchmarkName":"GDPval-AA normalized","category":"agentic","score":56.4,"unit":"percent","family":"gdpvalAaNormalized","index":74.35846427661211,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"jobBench","benchmarkName":"JobBench","category":"agentic","score":55.7,"unit":"percent","family":"jobBench","index":67.30121620265385,"aboveFit":false,"run":null,"source":"Yuetai Li, Yichen Feng, Zhangchen Xu, Zixian Ma, Kaiyuan Zheng, Fengqing Jiang, Xinghua Sun, Rulin Shao, Zichen Chen, Yue Huang, Xinyang Han, Brian Lee, Kayla Xu, Shenglai Zeng, Hang Hua, Xiangliang Zhang, Basel Alomair, Ranjay Krishna, Luke Zettlemoyer, Pang Wei Koh, Bhaskar Ramasubramanian, Luyao Niu, Xiang Yue, Radha Poovendran","sourceUrl":"https://arxiv.org/abs/2605.26329","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"agentsLastExam","benchmarkName":"Agents' Last Exam","category":"agentic","score":51.2,"unit":"percent","family":"agentsLastExam","index":83.56283412697758,"aboveFit":true,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaSciCode","benchmarkName":"Artificial Analysis SciCode","category":"coding","score":50.6,"unit":"percent","family":"scicode-family","index":61.80657045109676,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/scicode","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"nl2Repo","benchmarkName":"NL2Repo","category":"coding","score":48.1,"unit":"percent","family":"nl2Repo","index":60.69524963234622,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-browser:overall","benchmarkName":"LiveBench Browser Use","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":48,"unit":"percent","family":"livebench-2-agentic","index":53.693364138552134,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Browser","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.224024,"latency":null},{"benchmarkId":"aaTau3Banking","benchmarkName":"Artificial Analysis Tau3-Banking","category":"agentic","score":45.4,"unit":"percent","family":"aaTau3Banking","index":70.68761382039311,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/tau3-banking","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":38,"unit":"percent","family":"hle","index":62.511768624445665,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-long-horizon:overall","benchmarkName":"LiveBench Long Agentic","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":37.1,"unit":"percent","family":"livebench-2-agentic","index":45.721855203919155,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Long%20Agentic","measuredAt":"2026-10-14","stderr":null,"costPerTest":1.9914260869565217,"latency":null},{"benchmarkId":"hle","benchmarkName":"Humanity's Last Exam","category":"knowledge","score":35.9,"unit":"percent","family":"hle","index":60.40485385988309,"aboveFit":false,"run":null,"source":"Center for AI Safety, Scale AI, and thousands of expert contributors","sourceUrl":"https://lastexam.ai/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hleNoTools","benchmarkName":"Humanity's Last Exam without tools","category":"knowledge","score":35.9,"unit":"percent","family":"hle","index":60.40485385988309,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"omniscienceAccuracy","benchmarkName":"Artificial Analysis Omniscience Accuracy","category":"knowledge","score":23.5,"unit":"percent","family":"omniscienceAccuracy","index":40.30596609202369,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"osWorld2","benchmarkName":"OSWorld 2.0","category":"agentic","score":19.4,"unit":"percent","family":"osWorld2","index":61.91517562603979,"aboveFit":false,"run":null,"source":"Mengqi Yuan, Zilong Zhou, Xinzhuang Xiong, Weiming Wu, Jiayang Sun, Jiamin Song, Kaiqian Cui, Bowen Wang, Haoyuan Wu, Yitong Li, Dunjie Lu, Haikong Lu, Qi Zhen, Xinyuan Wang, Jiaqi Deng, Yuhao Yang, Cheng Chen, Boyuan Zheng, Alex Su, Xiao Yu, Hao Zou, Saaket Agashe, Xing Han Lu, Manpreet Kaur, Zhengyang Qi, Vincent Sunn Chen, Frederic Sala, Dayiheng Liu, Junyang Lin, Zhou Yu, Yu Su, Siva Reddy, Xin Eric Wang, Peng Qi, Tianbao Xie, Tao Yu","sourceUrl":"https://arxiv.org/abs/2606.29537","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":11.1,"unit":"percent","family":"critpt","index":58.33742082813056,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lmarena:agent-task-outcome:overall","benchmarkName":"Agent Arena task outcome","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":0.2524291630047526,"unit":"index","family":"agent-arena","index":66.73833765628194,"aboveFit":false,"run":null,"source":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset","measuredAt":"2026-11-11","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lmarena:agent-steerability:overall","benchmarkName":"Agent Arena steerability","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":-2.561548030372107,"unit":"index","family":"agent-arena","index":64.42480725238542,"aboveFit":false,"run":null,"source":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset","measuredAt":"2026-11-11","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lmarena:agent-bash-recovery:overall","benchmarkName":"Agent Arena command recovery","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":-2.5728882638138995,"unit":"index","family":"agent-arena","index":64.41548380267282,"aboveFit":false,"run":null,"source":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset","measuredAt":"2026-10-08","stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":55,"timeToFirstToken":38.14,"source":"BenchLM","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":95,"benchmarkCount":35,"categoryCount":7,"confidence":"high","state":"available"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:livebench","name":"LiveBench","url":"https://livebench.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:lmarena","name":"LMArena","url":"https://huggingface.co","license":"CC BY 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"mimo-v2-6-flash","providerId":"xiaomi","slug":"mimo-v2-6-flash","name":"MiMo-V2.6-Flash","provider":"Xiaomi","description":"MiMo-V2.6-Flash is a reasoning model from Xiaomi in the MiMo-V2.6 family. 23 benchmarks count toward its index, in 6 categories.","logoUrl":null,"releaseDate":"2026-09-22","contextWindowTokens":1050000,"modalities":["text","image","video","audio"],"openWeights":true,"details":{"apiId":"xiaomi/mimo-v2.6-flash","maxOutputTokens":131072,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.0028,"weightsUrl":"https://huggingface.co/XiaomiMiMo/MiMo-V2.6-Flash-RL"},"reasoning":true,"overallScore":64.27436758813963,"overallScoreRange":{"low":59.90268951283673,"high":68.64604566344252},"badges":[],"capabilityScores":{"agentic":69.72506548782472,"coding":64.12211240403536,"reasoning":59.176774161672405,"multimodal":50.28605160804664,"knowledge":52.28556644845162,"multilingual":null,"instruction":null,"math":68.90703675314599},"capabilityCounts":{"agentic":12,"coding":5,"reasoning":2,"multimodal":1,"math":1,"knowledge":2},"benchmarks":[{"benchmarkId":"cyberGym","benchmarkName":"CyberGym","category":"agentic","score":95.1,"unit":"percent","family":"cyberGym","index":76.04140492845437,"aboveFit":false,"run":null,"source":"Zhun Wang, Tianneng Shi, Jingxuan He, Matthew Cai, Jialin Zhang, Dawn Song","sourceUrl":"https://www.cybergym.io/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"terminalBench21","benchmarkName":"Terminal-Bench 2.1","category":"agentic","score":87.6,"unit":"percent","family":"terminal-bench-2-1","index":72.37445113518785,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:medscribe:overall","benchmarkName":"MedScribe","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":85.275,"unit":"percent","family":"vals-ai:medscribe:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medscribe","measuredAt":"2026-10-29","stderr":1.986,"costPerTest":0.002184,"latency":74.481},{"benchmarkId":"vals-ai:sage:overall","benchmarkName":"SAGE","category":null,"domain":"Education","eligible":false,"ranked":true,"score":83.256,"unit":"percent","family":"vals-ai:sage:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/sage","measuredAt":"2026-11-05","stderr":2.142,"costPerTest":0.007384,"latency":166.54},{"benchmarkId":"osWorldVerified","benchmarkName":"OSWorld-Verified","category":"agentic","score":80.8,"unit":"percent","family":"osWorldVerified","index":65.70738113113318,"aboveFit":false,"run":null,"source":"Tianbao Xie, Danyang Zhang, Jixuan Chen, Xiaochuan Li, Siheng Zhao, Ruisheng Cao, Toh Jing Hua, Zhoujun Cheng, Dongchan Shin, Fangyu Lei, Yitao Liu, Yiheng Xu, Shuyan Zhou, Silvio Savarese, Caiming Xiong, Victor Zhong, Tao Yu","sourceUrl":"https://os-world.github.io/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:vibe-code:overall","benchmarkName":"Vibe Code Bench v1.1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":78.96,"unit":"percent","family":"vals-ai:vibe-code:overall","index":70.7395311456024,"aboveFit":false,"run":"OpenHands","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","measuredAt":"2026-11-09","stderr":4.045,"costPerTest":0.563041,"latency":3013.012},{"benchmarkId":"vals-ai:terminal-bench-2-1:overall","benchmarkName":"Terminal-Bench 2.1","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":76.404,"unit":"percent","family":"terminal-bench-2-1","index":65.8264104745484,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2-1","measuredAt":"2026-10-01","stderr":1.716,"costPerTest":0.02332,"latency":809.733},{"benchmarkId":"vals-ai:cyber:overall","benchmarkName":"CyberBench v1.1","category":null,"domain":"Other","eligible":false,"ranked":true,"score":75.357,"unit":"percent","family":"vals-ai:cyber:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/cyber","measuredAt":"2026-11-05","stderr":5.417,"costPerTest":0.046276,"latency":1701.995},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":74.3,"unit":"percent","family":"lcr","index":58.27383251018526,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"toolathlonVerified","benchmarkName":"Toolathlon-Verified","category":"agentic","score":73.6,"unit":"percent","family":"toolathlonVerified","index":68.36038698074329,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaMmmuPro","benchmarkName":"Artificial Analysis MMMU-Pro","category":"multimodal","score":73.1,"unit":"percent","family":"mmmu-pro","index":50.28605160804664,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmmu-pro","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:biomysterybench:overall","benchmarkName":"BioMysteryBench","category":null,"domain":"Science","eligible":false,"ranked":true,"score":69.259,"unit":"percent","family":"vals-ai:biomysterybench:overall","index":null,"aboveFit":false,"run":"Terminus 2","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/biomysterybench","measuredAt":"2026-11-09","stderr":0.98,"costPerTest":0.050119,"latency":1707.953},{"benchmarkId":"deepSwe","benchmarkName":"DeepSWE","category":"agentic","score":67.9,"unit":"percent","family":"deepSwe","index":70.43812513605569,"aboveFit":false,"run":null,"source":"Datacurve AI","sourceUrl":"https://deepswe.datacurve.ai/blog","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:public-benefits-bench:overall","benchmarkName":"Public Benefits Bench v1.1","category":null,"domain":"Public benefits","eligible":false,"ranked":true,"score":67.591,"unit":"percent","family":"vals-ai:public-benefits-bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/public-benefits-bench","measuredAt":"2026-10-12","stderr":1.217,"costPerTest":0.031798,"latency":1255.522},{"benchmarkId":"vals-ai:emb:overall","benchmarkName":"EMB","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":65.459,"unit":"percent","family":"vals-ai:emb:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/emb","measuredAt":"2026-10-20","stderr":2.709,"costPerTest":0.108634,"latency":1521.906},{"benchmarkId":"vals-ai:proof_bench:overall","benchmarkName":"ProofBench v1.1","category":"math","domain":null,"eligible":true,"ranked":true,"score":63,"unit":"percent","family":"vals-ai:proof_bench:overall","index":68.90703675314599,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/proof_bench","measuredAt":"2026-11-09","stderr":4.852,"costPerTest":0.123277,"latency":3769.808},{"benchmarkId":"jobBench","benchmarkName":"JobBench","category":"agentic","score":61.2,"unit":"percent","family":"jobBench","index":70.34326218994455,"aboveFit":false,"run":null,"source":"Yuetai Li, Yichen Feng, Zhangchen Xu, Zixian Ma, Kaiyuan Zheng, Fengqing Jiang, Xinghua Sun, Rulin Shao, Zichen Chen, Yue Huang, Xinyang Han, Brian Lee, Kayla Xu, Shenglai Zeng, Hang Hua, Xiangliang Zhang, Basel Alomair, Ranjay Krishna, Luke Zettlemoyer, Pang Wei Koh, Bhaskar Ramasubramanian, Luyao Niu, Xiang Yue, Radha Poovendran","sourceUrl":"https://arxiv.org/abs/2605.26329","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:fabv2:overall","benchmarkName":"Finance Agent (v2)","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":56.277,"unit":"percent","family":"vals-ai:fabv2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/fabv2","measuredAt":"2026-10-29","stderr":0.462,"costPerTest":0.071536,"latency":421.99},{"benchmarkId":"gdpvalAaNormalized","benchmarkName":"GDPval-AA normalized","category":"agentic","score":55.4,"unit":"percent","family":"gdpvalAaNormalized","index":73.64348751492028,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"automationBench","benchmarkName":"AutomationBench","category":"agentic","score":52.3,"unit":"percent","family":"automationBench","index":81.56981753005891,"aboveFit":true,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaSciCode","benchmarkName":"Artificial Analysis SciCode","category":"coding","score":51.3,"unit":"percent","family":"scicode-family","index":62.71915425358707,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/scicode","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:ioi:overall","benchmarkName":"IOI","category":"coding","domain":null,"eligible":true,"ranked":true,"score":47.667,"unit":"percent","family":"vals-ai:ioi:overall","index":64.01189375215877,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/ioi","measuredAt":"2026-10-29","stderr":4.785,"costPerTest":0.215104,"latency":6521.606},{"benchmarkId":"vals-ai:medcode:overall","benchmarkName":"MedCode","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":41.057,"unit":"percent","family":"vals-ai:medcode:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medcode","measuredAt":"2026-10-29","stderr":2.032,"costPerTest":0.002865,"latency":93.373},{"benchmarkId":"vals-ai:code-migration:overall","benchmarkName":"Code Migration","category":"coding","domain":null,"eligible":true,"ranked":true,"score":40.927,"unit":"percent","family":"vals-ai:code-migration:overall","index":66.93120075955241,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/code-migration","measuredAt":"2026-10-29","stderr":4.351,"costPerTest":0.493563,"latency":11538.747},{"benchmarkId":"vals-ai:legal_research:overall","benchmarkName":"Legal Research Bench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":37.981,"unit":"percent","family":"vals-ai:legal_research:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_research","measuredAt":"2026-11-09","stderr":3.373,"costPerTest":0.070424,"latency":919.689},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":35.1,"unit":"percent","family":"hle","index":59.60221966385926,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:tax_agent_bench:overall","benchmarkName":"Tax Agent Bench","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":28.507,"unit":"percent","family":"vals-ai:tax_agent_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench","measuredAt":"2026-11-09","stderr":3.14,"costPerTest":0.048982,"latency":1038.928},{"benchmarkId":"agentsLastExam","benchmarkName":"Agents' Last Exam","category":"agentic","score":27.6,"unit":"percent","family":"agentsLastExam","index":63.97105197406749,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"omniscienceAccuracy","benchmarkName":"Artificial Analysis Omniscience Accuracy","category":"knowledge","score":27.2,"unit":"percent","family":"omniscienceAccuracy","index":44.96891323304397,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"programBench","benchmarkName":"ProgramBench: Can Language Models Rebuild Programs From Scratch?","category":"coding","score":26,"unit":"percent","family":"programBench","index":56.208782109276186,"aboveFit":false,"run":null,"source":"John Yang, Kilian Lieret, Jeffrey Ma, Parth Thakkar, Dmitrii Pedchenko, Sten Sootla, Emily McMilin, Pengcheng Yin, Rui Hou, Gabriel Synnaeve, Diyi Yang, Ofir Press","sourceUrl":"https://programbench.com/static/paper.pdf","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:terminal-bench-4:overall","benchmarkName":"Terminal-Bench 4.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":24.242,"unit":"percent","family":"terminal-bench-4","index":66.02705156871163,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-4","measuredAt":"2026-11-09","stderr":1.515,"costPerTest":0.211804,"latency":8712.24},{"benchmarkId":"vals-ai:mysterymechanism:overall","benchmarkName":"MysteryMechanism","category":null,"domain":"Science","eligible":false,"ranked":true,"score":21.622,"unit":"percent","family":"vals-ai:mysterymechanism:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mysterymechanism","measuredAt":"2026-11-11","stderr":2.769,"costPerTest":0.057497,"latency":2107.543},{"benchmarkId":"vals-ai:hlab:overall","benchmarkName":"HLAB-Vals-1.1","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":13.333,"unit":"percent","family":"vals-ai:hlab:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/hlab","measuredAt":"2026-11-09","stderr":2.821,"costPerTest":0.085646,"latency":1342.563},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":12,"unit":"percent","family":"critpt","index":60.07971581315955,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"exploitGym","benchmarkName":"ExploitGym","category":"agentic","score":6,"unit":"percent","family":"exploitGym","index":62.62665909724513,"aboveFit":false,"run":null,"source":"Zhun Wang, Nico Schiller, Hongwei Li, Srijiith Sesha Narayana, Milad Nasr, Nicholas Carlini, Xiangyu Qi, Eric Wallace, Elie Bursztein, Luca Invernizzi, Kurt Thomas, Yan Shoshitaishvili, Wenbo Guo, Jingxuan He, Thorsten Holz, Dawn Song","sourceUrl":"https://arxiv.org/abs/2605.11086","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:terminal-bench-science:overall","benchmarkName":"Terminal-Bench-Science","category":null,"domain":"Science","eligible":false,"ranked":true,"score":4.286,"unit":"percent","family":"vals-ai:terminal-bench-science:overall","index":null,"aboveFit":false,"run":"Mini-SWE-agent","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-science","measuredAt":"2026-11-09","stderr":2.438,"costPerTest":0.244346,"latency":14326.966},{"benchmarkId":"lmarena:agent-task-outcome:overall","benchmarkName":"Agent Arena task outcome","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":4.225807601561982,"unit":"index","family":"agent-arena","index":70.00507764458928,"aboveFit":false,"run":null,"source":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset","measuredAt":"2026-11-11","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:srebench:overall","benchmarkName":"SRE Bench","category":null,"domain":"Other","eligible":false,"ranked":true,"score":4.198,"unit":"percent","family":"vals-ai:srebench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/srebench","measuredAt":"2026-11-05","stderr":1.241,"costPerTest":0.395481,"latency":12121.727},{"benchmarkId":"lmarena:agent-steerability:overall","benchmarkName":"Agent Arena steerability","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":2.8472984334565132,"unit":"index","family":"agent-arena","index":68.87172699769404,"aboveFit":false,"run":null,"source":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset","measuredAt":"2026-11-11","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:programbench:overall","benchmarkName":"ProgramBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":0.5,"unit":"percent","family":"vals-ai:programbench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/programbench","measuredAt":"2026-11-09","stderr":0.5,"costPerTest":0.470526,"latency":12091.441},{"benchmarkId":"lmarena:agent-bash-recovery:overall","benchmarkName":"Agent Arena command recovery","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":-1.0597459793076691,"unit":"index","family":"agent-arena","index":65.6595239778619,"aboveFit":false,"run":null,"source":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset","measuredAt":"2026-10-08","stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":0.14,"outputPerMillion":0.28,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":28,"timeToFirstToken":3.269,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":{"elo":1450.3208744557305,"eloUpper":1455.5339533851757,"eloLower":1445.1077955262854,"votes":22354,"rank":54,"rankUpper":null,"rankLower":null,"arenaName":"MiMo-V2.6-Flash","organization":"xiaomi","updatedAt":"2026-11-09"},"quality":{"coveragePercent":90,"benchmarkCount":23,"categoryCount":6,"confidence":"medium","state":"available"},"hosts":[{"host":"vultr","name":"Vultr","kind":"cloud","url":"https://api.vultrinference.com/","inputPerMillion":0.1,"outputPerMillion":0.25,"cacheReadPerMillion":null,"contextTokens":1048576,"maxOutputTokens":131072,"deprecated":false},{"host":"deepinfra","name":"Deep Infra","kind":"host","url":"https://deepinfra.com/models","inputPerMillion":0.14,"outputPerMillion":0.28,"cacheReadPerMillion":0.0028,"contextTokens":1048576,"maxOutputTokens":131072,"deprecated":false},{"host":"novita-ai","name":"Novita AI","kind":"host","url":"https://novita.ai/docs/guides/introduction","inputPerMillion":0.14,"outputPerMillion":0.28,"cacheReadPerMillion":0.0028,"contextTokens":1048576,"maxOutputTokens":131072,"deprecated":false},{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":0.14,"outputPerMillion":0.28,"cacheReadPerMillion":0.0028,"contextTokens":1050000,"maxOutputTokens":131072,"deprecated":false},{"host":"xiaomi","name":"Xiaomi","kind":"maker","url":"https://platform.xiaomimimo.com/#/docs","inputPerMillion":0.14,"outputPerMillion":0.28,"cacheReadPerMillion":0.0028,"contextTokens":1048576,"maxOutputTokens":131072,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:lmarena","name":"LMArena","url":"https://huggingface.co","license":"CC BY 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"claude-sonnet-5","providerId":"anthropic","slug":"claude-sonnet-5","name":"Claude Sonnet 5","provider":"Anthropic","description":"Claude Sonnet 5 is a reasoning model from Anthropic. 53 benchmarks count toward its index, in 7 categories.","logoUrl":null,"releaseDate":"2026-06-30","contextWindowTokens":1000000,"modalities":["text","image","file"],"openWeights":false,"details":{"apiId":"anthropic/claude-sonnet-5","maxOutputTokens":128000,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.19999999999999998,"weightsUrl":null},"reasoning":true,"overallScore":64.2003448895518,"overallScoreRange":{"low":61.4792245705015,"high":66.9214652086021},"badges":[],"capabilityScores":{"agentic":65.26779131373317,"coding":65.26544120269136,"reasoning":66.51658484534332,"multimodal":59.640205707142826,"knowledge":58.96629356490669,"multilingual":null,"instruction":55.80029183400153,"math":64.78608374654802},"capabilityCounts":{"math":5,"knowledge":6,"reasoning":5,"multimodal":4,"coding":15,"agentic":17,"instruction":1},"benchmarks":[{"benchmarkId":"livebench:livebench-math:overall","benchmarkName":"LiveBench Mathematics","category":"math","domain":null,"eligible":true,"ranked":true,"score":92.94225,"unit":"percent","family":"livebench:livebench-math:overall","index":69.31023125841423,"aboveFit":false,"run":"xhigh effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Mathematics","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.29504459102902375,"latency":null},{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":91.1,"unit":"percent","family":"gpqa-diamond","index":60.76353142304194,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vulcanCiiV1","benchmarkName":"VulcanBench Coding Intelligence Index v1","category":"coding","score":89.2,"unit":"percent","family":"vulcanCiiV1","index":null,"aboveFit":false,"run":null,"source":"VulcanBench contributors","sourceUrl":"https://github.com/morganlinton/VulcanBench/blob/main/docs/results/cii-v1-2026-08/README.md","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:gpqa:overall","benchmarkName":"GPQA Diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":88.889,"unit":"percent","family":"gpqa-diamond","index":58.735662638794295,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","measuredAt":"2026-09-01","stderr":2.224,"costPerTest":0.077709,"latency":63.234},{"benchmarkId":"livebench:livebench-reasoning:overall","benchmarkName":"LiveBench Reasoning","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":88.69225,"unit":"percent","family":"livebench","index":76.51467718014884,"aboveFit":false,"run":"xhigh effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Reasoning","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.3827668316831682,"latency":null},{"benchmarkId":"charxiv","benchmarkName":"CharXiv Reasoning","category":"multimodal","score":88.3,"unit":"percent","family":"charxiv","index":64.71361200893692,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:mmlu_pro:overall","benchmarkName":"MMLU Pro","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":87.545,"unit":"percent","family":"mmlu-pro","index":54.77694955981265,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","measuredAt":"2026-09-01","stderr":0.368,"costPerTest":0.042996,"latency":25.191},{"benchmarkId":"sweVerified","benchmarkName":"Software Engineering Benchmark Verified","category":"coding","score":85.2,"unit":"percent","family":"swe-bench-verified","index":64.0678331566422,"aboveFit":false,"run":null,"source":"Carlos E. Jimenez, John Yang, Alexander Wettig, Shunyu Yao, Kexin Pei, Ofir Press, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2310.06770","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"browseComp","benchmarkName":"BrowseComp","category":"agentic","score":84.7,"unit":"percent","family":"browseComp","index":66.82367161874001,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:legal_bench:overall","benchmarkName":"LegalBench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":83.922,"unit":"percent","family":"vals-ai:legal_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","measuredAt":"2026-10-13","stderr":0.458,"costPerTest":0.005759,"latency":4.901},{"benchmarkId":"vals-ai:mmmu:overall","benchmarkName":"MMMU Pro","category":"multimodal","domain":null,"eligible":true,"ranked":true,"score":83.006,"unit":"percent","family":"mmmu-pro","index":63.1778415063171,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmmu","measuredAt":"2026-09-01","stderr":0.903,"costPerTest":0.033133,"latency":18.817},{"benchmarkId":"livebench:livebench-research:overall","benchmarkName":"LiveBench Research","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":82.8,"unit":"percent","family":"livebench-2-agentic","index":79.14368624178404,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Research","measuredAt":"2026-10-14","stderr":null,"costPerTest":5.9611409090909095,"latency":null},{"benchmarkId":"vals-ai:lcb:overall","benchmarkName":"LiveCodeBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":82.429,"unit":"percent","family":"livecodebench","index":57.738506268965565,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","measuredAt":"2026-09-01","stderr":1.088,"costPerTest":0.094655,"latency":77.021},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":82,"unit":"percent","family":"lcr","index":63.55665255719484,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-design:overall","benchmarkName":"LiveBench Design","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":82,"unit":"percent","family":"livebench-2-agentic","index":78.55862136584767,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Design","measuredAt":"2026-10-14","stderr":null,"costPerTest":5.4721280000000005,"latency":null},{"benchmarkId":"vals-ai:vibe-code:overall","benchmarkName":"Vibe Code Bench v1.1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":81.325,"unit":"percent","family":"vals-ai:vibe-code:overall","index":71.59286030253917,"aboveFit":false,"run":"OpenHands","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","measuredAt":"2026-11-09","stderr":3.046,"costPerTest":25.38542,"latency":4001.424},{"benchmarkId":"osWorldVerified","benchmarkName":"OSWorld-Verified","category":"agentic","score":81.2,"unit":"percent","family":"osWorldVerified","index":66.0302839550038,"aboveFit":false,"run":null,"source":"Tianbao Xie, Danyang Zhang, Jixuan Chen, Xiaochuan Li, Siheng Zhao, Ruisheng Cao, Toh Jing Hua, Zhoujun Cheng, Dongchan Shin, Fangyu Lei, Yitao Liu, Yiheng Xu, Shuyan Zhou, Silvio Savarese, Caiming Xiong, Victor Zhong, Tao Yu","sourceUrl":"https://os-world.github.io/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-coding:overall","benchmarkName":"LiveBench Coding","category":"coding","domain":null,"eligible":true,"ranked":true,"score":80.68,"unit":"percent","family":"livebench:livebench-coding:overall","index":68.29322876016123,"aboveFit":false,"run":"xhigh effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Coding","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.03546239316239316,"latency":null},{"benchmarkId":"aaTerminalBench21","benchmarkName":"Artificial Analysis Terminal-Bench v2.1","category":"agentic","score":80.5,"unit":"percent","family":"aaTerminalBench21","index":63.04136203793429,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/terminalbench-v2-1","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"terminalBench21","benchmarkName":"Terminal-Bench 2.1","category":"agentic","score":80.4,"unit":"percent","family":"terminal-bench-2-1","index":68.16349251098242,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":80.3030303030303,"unit":"percent","family":"gpqa-diamond","index":50.86084584504438,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.8335609732463336,"costPerTest":null,"latency":null},{"benchmarkId":"labBench2","benchmarkName":"LABBench2: An Improved Benchmark for AI Systems Performing Biology Research","category":"knowledge","score":80.1,"unit":"percent","family":"labBench2","index":null,"aboveFit":false,"run":null,"source":"Jon M. Laurent, Albert Bou, Michael Pieler, Conor Igoe, Alex Andonian, Siddharth Narayanan, James Braza, Alexandros Sanchez Vassopoulos, Jacob L. Steenwyk, Blake Lash, Andrew D. White, Samuel G. Rodriques","sourceUrl":"https://arxiv.org/abs/2604.09554","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":80,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":55.46261453521162,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":6.030226891555273,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:swebench:overall","benchmarkName":"SWE-bench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":79.6,"unit":"percent","family":"swe-bench-verified","index":59.93751072105352,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/swebench","measuredAt":"2026-09-01","stderr":1.804,"costPerTest":1.489899,"latency":962.366},{"benchmarkId":"aaMmmuPro","benchmarkName":"Artificial Analysis MMMU-Pro","category":"multimodal","score":77.3,"unit":"percent","family":"mmmu-pro","index":55.75198312154714,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmmu-pro","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"charxivNoTools","benchmarkName":"CharXiv Reasoning without tools","category":"multimodal","score":77,"unit":"percent","family":"charxivNoTools","index":55.330393155578555,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:proof_bench:overall","benchmarkName":"ProofBench v1.1","category":"math","domain":null,"eligible":true,"ranked":true,"score":77,"unit":"percent","family":"vals-ai:proof_bench:overall","index":73.74046256702798,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/proof_bench","measuredAt":"2026-11-09","stderr":4.23,"costPerTest":1.373684,"latency":898.079},{"benchmarkId":"vals-ai:medscribe:overall","benchmarkName":"MedScribe","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":76.054,"unit":"percent","family":"vals-ai:medscribe:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medscribe","measuredAt":"2026-10-29","stderr":3.05,"costPerTest":0.433684,"latency":251.703},{"benchmarkId":"vals-ai:tax_eval_v2:overall","benchmarkName":"TaxEval v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":75.634,"unit":"percent","family":"vals-ai:tax_eval_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","measuredAt":"2026-09-01","stderr":0.836,"costPerTest":0.301619,"latency":202.488},{"benchmarkId":"livebench:livebench-language:overall","benchmarkName":"LiveBench Language","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":74.97033333333333,"unit":"percent","family":"livebench:livebench-language:overall","index":59.3359155782052,"aboveFit":false,"run":"xhigh effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Language","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.12741133333333335,"latency":null},{"benchmarkId":"livebench:livebench-coding-2:overall","benchmarkName":"LiveBench Coding 2.0","category":"coding","domain":null,"eligible":true,"ranked":true,"score":74.7,"unit":"percent","family":"livebench:livebench-coding-2:overall","index":62.279002056135226,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Coding","measuredAt":"2026-10-14","stderr":null,"costPerTest":3.483062068965517,"latency":null},{"benchmarkId":"vals-ai:terminal-bench-2-1:overall","benchmarkName":"Terminal-Bench 2.1","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":74.532,"unit":"percent","family":"terminal-bench-2-1","index":64.73156123225498,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2-1","measuredAt":"2026-10-01","stderr":2.085,"costPerTest":0.534595,"latency":635.043},{"benchmarkId":"livebench:livebench-browser:overall","benchmarkName":"LiveBench Browser Use","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":72,"unit":"percent","family":"livebench-2-agentic","index":71.24531041664311,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Browser","measuredAt":"2026-10-14","stderr":null,"costPerTest":6.289836,"latency":null},{"benchmarkId":"livebench:livebench-data-analysis:overall","benchmarkName":"LiveBench Data Analysis","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":71.74066666666666,"unit":"percent","family":"livebench","index":54.55501984163745,"aboveFit":false,"run":"xhigh effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Data%20Analysis","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.4650986666666666,"latency":null},{"benchmarkId":"vals-ai:mortgage_tax:overall","benchmarkName":"MortgageTax","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":70.032,"unit":"percent","family":"vals-ai:mortgage_tax:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mortgage_tax","measuredAt":"2026-09-01","stderr":0.897,"costPerTest":0.080204,"latency":28.668},{"benchmarkId":"vals-ai:corp_fin_v2:overall","benchmarkName":"CorpFin v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":66.977,"unit":"percent","family":"vals-ai:corp_fin_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","measuredAt":"2026-08-12","stderr":0.926,"costPerTest":0.516119,"latency":18.628},{"benchmarkId":"vals-ai:emb:overall","benchmarkName":"EMB","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":66.316,"unit":"percent","family":"vals-ai:emb:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/emb","measuredAt":"2026-10-20","stderr":3.006,"costPerTest":10.293794,"latency":2985.704},{"benchmarkId":"vals-ai:public-benefits-bench:overall","benchmarkName":"Public Benefits Bench v1.1","category":null,"domain":"Public benefits","eligible":false,"ranked":true,"score":66.035,"unit":"percent","family":"vals-ai:public-benefits-bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/public-benefits-bench","measuredAt":"2026-10-12","stderr":1.232,"costPerTest":1.290695,"latency":1268.852},{"benchmarkId":"epoch-ai:epoch-frontiermath-tiers-1-3-v2-private:overall","benchmarkName":"FrontierMath-Tiers-1-3-v2-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":65.6140350877193,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tiers-1-3-v2-private:overall","index":65.43075595895804,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.818576398907258,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-instruction-following:overall","benchmarkName":"LiveBench Instruction Following","category":"instruction","domain":null,"eligible":true,"ranked":true,"score":63.85850000000001,"unit":"percent","family":"livebench:livebench-instruction-following:overall","index":55.80029183400153,"aboveFit":false,"run":"xhigh effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=IF","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.2594285,"latency":null},{"benchmarkId":"epoch-ai:epoch-mirrorcode:overall","benchmarkName":"MirrorCode","category":"coding","domain":null,"eligible":true,"ranked":true,"score":63.33333333333333,"unit":"percent","family":"epoch-ai:epoch-mirrorcode:overall","index":77.37981916833517,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":10.690449676496975,"costPerTest":null,"latency":null},{"benchmarkId":"swePro","benchmarkName":"SWE-bench Pro","category":"coding","score":63.2,"unit":"percent","family":"swePro","index":62.66113448499405,"aboveFit":false,"run":null,"source":"Xiang Deng, Jeff Da, Edwin Pan, Yannis Yiming He, Charles Ide, Kanak Garg, Niklas Lauffer, Andrew Park, Nitin Pasari, Chetan Rane, Karmini Sampath, Maya Krishnan, Srivatsa Kundurthy, Sean Hendryx, Zifan Wang, Chen Bo Calvin Zhang, Noah Jacobson, Bing Liu, Brad Kenstler","sourceUrl":"https://arxiv.org/abs/2509.16941","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:cyber:overall","benchmarkName":"CyberBench v1.1","category":null,"domain":"Other","eligible":false,"ranked":true,"score":61.905,"unit":"percent","family":"vals-ai:cyber:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/cyber","measuredAt":"2026-11-05","stderr":5.742,"costPerTest":1.888947,"latency":1017.033},{"benchmarkId":"cursorBench32","benchmarkName":"cursorBench32","category":"coding","score":61.5,"unit":"percent","family":"cursorBench32","index":66.36891146058365,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"parseBenchLlamaIndex","benchmarkName":"ParseBench: document parsing for AI agents (LlamaIndex leaderboard)","category":"multimodal","score":61.16,"unit":"percent","family":"parseBenchLlamaIndex","index":59.05190535012372,"aboveFit":false,"run":null,"source":"Boyang Zhang, Sebastián G. Acosta, Preston Carlson, Sacha Bron, Pierre-Loïc Doulcet, Daniel B. Ospina and Simon Suo (LlamaIndex)","sourceUrl":"https://arxiv.org/abs/2604.08538","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-agentic-coding:overall","benchmarkName":"LiveBench Agentic Coding","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":59.394000000000005,"unit":"percent","family":"livebench:livebench-agentic-coding:overall","index":70.4147109829918,"aboveFit":false,"run":"xhigh effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Agentic%20Coding","measuredAt":"2026-06-25","stderr":null,"costPerTest":2.137570833333333,"latency":null},{"benchmarkId":"hleWithTools","benchmarkName":"Humanity's Last Exam with tools","category":"agentic","score":57.4,"unit":"percent","family":"hleWithTools","index":66.34692379451198,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hle","benchmarkName":"Humanity's Last Exam","category":"knowledge","score":57.4,"unit":"percent","family":"hle","index":81.97564787802366,"aboveFit":true,"run":null,"source":"Center for AI Safety, Scale AI, and thousands of expert contributors","sourceUrl":"https://lastexam.ai/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"osWorld21","benchmarkName":"OSWorld 2.1 (all 108 tasks)","category":"agentic","score":57,"unit":"percent","family":"osWorld21","index":59.42924603677173,"aboveFit":false,"run":null,"source":"Mengqi Yuan, Zilong Zhou, Xinzhuang Xiong, Weiming Wu, Jiayang Sun, Jiamin Song, Kaiqian Cui, Bowen Wang, Haoyuan Wu, Yitong Li, Dunjie Lu, Haikong Lu, Qi Zhen, Xinyuan Wang, Jiaqi Deng, Yuhao Yang, Cheng Chen, Boyuan Zheng, Alex Su, Xiao Yu, Hao Zou, Saaket Agashe, Xing Han Lu, Manpreet Kaur, Zhengyang Qi, Vincent Sunn Chen, Frederic Sala, Dayiheng Liu, Junyang Lin, Zhou Yu, Yu Su, Siva Reddy, Xin Eric Wang, Peng Qi, Tianbao Xie, Tao Yu","sourceUrl":"https://github.com/xlang-ai/OSWorld-V2/releases","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaSciCode","benchmarkName":"Artificial Analysis SciCode","category":"coding","score":54.3,"unit":"percent","family":"scicode-family","index":66.63022769283127,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/scicode","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:fabv2:overall","benchmarkName":"Finance Agent (v2)","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":53.909,"unit":"percent","family":"vals-ai:fabv2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/fabv2","measuredAt":"2026-10-29","stderr":0.518,"costPerTest":0.750759,"latency":792.086},{"benchmarkId":"gdpvalAaNormalized","benchmarkName":"GDPval-AA normalized","category":"agentic","score":48.2,"unit":"percent","family":"gdpvalAaNormalized","index":68.49565483073908,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:medcode:overall","benchmarkName":"MedCode","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":47.54,"unit":"percent","family":"vals-ai:medcode:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medcode","measuredAt":"2026-10-29","stderr":2.274,"costPerTest":0.278799,"latency":134.934},{"benchmarkId":"vals-ai:skillsbench:overall","benchmarkName":"SkillsBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":46.476,"unit":"percent","family":"vals-ai:skillsbench:overall","index":59.73589051164771,"aboveFit":false,"run":"OpenHands","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/skillsbench","measuredAt":"2026-10-29","stderr":4.491,"costPerTest":2.89733,"latency":911.674},{"benchmarkId":"aaAnalystAgent","benchmarkName":"Artificial Analysis AnalystAgent","category":"agentic","score":46.3,"unit":"percent","family":"aaAnalystAgent","index":67.82972161858362,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/aa-analyst-agent","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:ioi:overall","benchmarkName":"IOI","category":"coding","domain":null,"eligible":true,"ranked":true,"score":45,"unit":"percent","family":"vals-ai:ioi:overall","index":63.010642209227996,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/ioi","measuredAt":"2026-10-29","stderr":2.754,"costPerTest":12.889951,"latency":3583.884},{"benchmarkId":"epoch-ai:epoch-chess-puzzles:overall","benchmarkName":"Chess Puzzles","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":45,"unit":"percent","family":"epoch-ai:epoch-chess-puzzles:overall","index":68.14035950689328,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":5,"costPerTest":null,"latency":null},{"benchmarkId":"aaEnterpriseOpsGym","benchmarkName":"Artificial Analysis EnterpriseOps-Gym","category":"agentic","score":44.7,"unit":"percent","family":"aaEnterpriseOpsGym","index":null,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/enterprise-ops-gym-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:code-migration:overall","benchmarkName":"Code Migration","category":"coding","domain":null,"eligible":true,"ranked":true,"score":44.392,"unit":"percent","family":"vals-ai:code-migration:overall","index":68.77194768422778,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/code-migration","measuredAt":"2026-10-29","stderr":4.246,"costPerTest":35.310438,"latency":7026.544},{"benchmarkId":"hleNoTools","benchmarkName":"Humanity's Last Exam without tools","category":"knowledge","score":43.2,"unit":"percent","family":"hle","index":67.7288908986006,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"frontierCode","benchmarkName":"FrontierCode 1.1 Main","category":"coding","score":42.7,"unit":"percent","family":"frontierCode","index":67.73877718289793,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:legal_research:overall","benchmarkName":"Legal Research Bench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":41.827,"unit":"percent","family":"vals-ai:legal_research:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_research","measuredAt":"2026-11-09","stderr":3.428,"costPerTest":2.719446,"latency":1545.997},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":41.3,"unit":"percent","family":"hle","index":65.82263468304399,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"omniscienceAccuracy","benchmarkName":"Artificial Analysis Omniscience Accuracy","category":"knowledge","score":40.1,"unit":"percent","family":"omniscienceAccuracy","index":61.226215427411994,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-long-horizon:overall","benchmarkName":"LiveBench Long Agentic","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":39,"unit":"percent","family":"livebench-2-agentic","index":47.11138428426802,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Long%20Agentic","measuredAt":"2026-10-14","stderr":null,"costPerTest":33.21113888888889,"latency":null},{"benchmarkId":"aaTau3Banking","benchmarkName":"Artificial Analysis Tau3-Banking","category":"agentic","score":37.3,"unit":"percent","family":"aaTau3Banking","index":60.66493084648717,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/tau3-banking","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-mystery-game-puzzles:overall","benchmarkName":"Mystery Game Puzzles","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":35,"unit":"percent","family":"epoch-ai:epoch-mystery-game-puzzles:overall","index":65.7855195867512,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":4.793724854411023,"costPerTest":null,"latency":null},{"benchmarkId":"cursorBench40","benchmarkName":"cursorBench40","category":"coding","score":34.1,"unit":"percent","family":"cursorBench40","index":62.22350788143778,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-simpleqa-verified:overall","benchmarkName":"SimpleQA Verified","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":33.7,"unit":"percent","family":"epoch-ai:epoch-simpleqa-verified:overall","index":51.9941272866154,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.495508791865346,"costPerTest":null,"latency":null},{"benchmarkId":"hleVerified","benchmarkName":"HLE-Verified","category":"knowledge","score":31,"unit":"percent","family":"hleVerified","index":null,"aboveFit":false,"run":null,"source":"Weiqi Zhai, Zhihai Wang, Jinghang Wang, Boyu Yang, Xiaogang Li, Xiang Xu, Bohan Wang, Peng Wang, Xingzhe Wu, Anfeng Li, Qiyuan Feng, Yuhao Zhou, Shoulin Han, Wenjie Luo, Yiyuan Li, Yaxuan Wang, Ruixian Luo, Guojie Lin, Peiyao Xiao, Chengliang Xu, Ben Wang, Zeyu Wang, Zichao Chen, Jianan Ye, Yijie Hu, Jialong Chen, Zongwen Shen, Yuliang Xu, An Yang, Bowen Yu, Dayiheng Liu, Junyang Lin, Hu Wei, Que Shen, Bing Zhao","sourceUrl":"https://arxiv.org/abs/2602.13964","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:tax_agent_bench:overall","benchmarkName":"Tax Agent Bench","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":29.374,"unit":"percent","family":"vals-ai:tax_agent_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench","measuredAt":"2026-11-09","stderr":3.248,"costPerTest":1.794774,"latency":1158.894},{"benchmarkId":"epoch-ai:epoch-frontiermath-tier-4-v2-private:overall","benchmarkName":"FrontierMath-Tier-4-v2-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":29.268292682926827,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tier-4-v2-private:overall","index":59.986354413128154,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":7.194088392074452,"costPerTest":null,"latency":null},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":16.9,"unit":"percent","family":"critpt","index":69.5655440649841,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"apprenticeBench","benchmarkName":"ApprenticeBench: end-to-end computer use, continual learning, and long-horizon agency on a real accounts-payable job","category":"agentic","score":16,"unit":"percent","family":"apprenticeBench","index":66.3197592285453,"aboveFit":false,"run":null,"source":"NeoCognition","sourceUrl":"https://neocognition.io/blog/apprentice-bench/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"frontierBench","benchmarkName":"Terminal-Bench 3.0","category":"agentic","score":14.59,"unit":"percent","family":"frontierBench","index":63.41092167730846,"aboveFit":false,"run":null,"source":"Ryan Marten, Alex Shaw, Andy Konwinski, Harbor, and the Laude Institute","sourceUrl":"https://www.frontierbench.ai/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:vcb-1-100:overall","benchmarkName":"Vibe Code Bench 1-100","category":"coding","domain":null,"eligible":true,"ranked":true,"score":13.822,"unit":"percent","family":"vals-ai:vcb-1-100:overall","index":62.55449043753809,"aboveFit":false,"run":"OpenHands","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vcb-1-100","measuredAt":"2026-10-29","stderr":2.985,"costPerTest":71.148517,"latency":15369.858},{"benchmarkId":"terminal-bench:terminal-bench-4-0-0:overall","benchmarkName":"Terminal-Bench 4.0.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":12.42,"unit":"percent","family":"terminal-bench-4","index":60.413509268077924,"aboveFit":false,"run":"max effort · Claude Code","source":"Terminal-Bench","sourceUrl":"https://www.tbench.ai/leaderboard","measuredAt":"2026-09-29","stderr":1.5612244897959184,"costPerTest":29.102606060606064,"latency":6510.3},{"benchmarkId":"terminal-bench:terminal-bench-4-1-0:overall","benchmarkName":"Terminal-Bench 4.1.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":12.12,"unit":"percent","family":"terminal-bench:terminal-bench-4-1-0:overall","index":58.633912934308704,"aboveFit":false,"run":"max effort · Claude Code","source":"Terminal-Bench","sourceUrl":"https://www.tbench.ai/leaderboard","measuredAt":"2026-09-29","stderr":1.530612244897959,"costPerTest":29.102606060606064,"latency":6510.3},{"benchmarkId":"vals-ai:terminal-bench-4:overall","benchmarkName":"Terminal-Bench 4.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":10.606,"unit":"percent","family":"terminal-bench-4","index":59.552151990684116,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-4","measuredAt":"2026-11-09","stderr":1.515,"costPerTest":26.04996,"latency":6297.51},{"benchmarkId":"lmarena:agent-bash-recovery:overall","benchmarkName":"Agent Arena command recovery","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":7.376754221324799,"unit":"index","family":"agent-arena","index":72.59564974411941,"aboveFit":false,"run":"high effort","source":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset","measuredAt":"2026-10-08","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:terminal-bench-science:overall","benchmarkName":"Terminal-Bench-Science","category":null,"domain":"Science","eligible":false,"ranked":true,"score":5.714,"unit":"percent","family":"vals-ai:terminal-bench-science:overall","index":null,"aboveFit":false,"run":"Mini-SWE-agent","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-science","measuredAt":"2026-11-09","stderr":2.794,"costPerTest":30.299862,"latency":11157.924},{"benchmarkId":"lmarena:agent-steerability:overall","benchmarkName":"Agent Arena steerability","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":2.7604962939647484,"unit":"index","family":"agent-arena","index":68.80036203094515,"aboveFit":false,"run":"high effort","source":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset","measuredAt":"2026-11-11","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:srebench:overall","benchmarkName":"SRE Bench","category":null,"domain":"Other","eligible":false,"ranked":true,"score":0.382,"unit":"percent","family":"vals-ai:srebench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/srebench","measuredAt":"2026-11-05","stderr":0.382,"costPerTest":18.493954,"latency":13984.551},{"benchmarkId":"mazeBench","benchmarkName":"MazeBench: visual spatial reasoning in a 3D open world","category":"agentic","score":0,"unit":"percent","family":"mazeBench","index":61.97866734834783,"aboveFit":false,"run":null,"source":"Jonathan Pappas and David Pappas","sourceUrl":"https://mazebench.com/blog?post=maze-bench-results","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:programbench:overall","benchmarkName":"ProgramBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":0,"unit":"percent","family":"vals-ai:programbench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/programbench","measuredAt":"2026-11-09","stderr":0,"costPerTest":24.358317,"latency":5471.639},{"benchmarkId":"lmarena:agent-task-outcome:overall","benchmarkName":"Agent Arena task outcome","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":-0.573920237996306,"unit":"index","family":"agent-arena","index":66.05894890076632,"aboveFit":false,"run":"high effort","source":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset","measuredAt":"2026-11-11","stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":2,"outputPerMillion":10,"batch":{"inputPerMillion":1,"outputPerMillion":5},"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":75,"timeToFirstToken":1.306,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":{"elo":1441.891103707937,"eloUpper":1445.926342964849,"eloLower":1437.8558644510244,"votes":53013,"rank":75,"rankUpper":null,"rankLower":null,"arenaName":"Claude Sonnet 5-high","organization":"anthropic","updatedAt":"2026-11-09"},"quality":{"coveragePercent":95,"benchmarkCount":53,"categoryCount":7,"confidence":"high","state":"available"},"hosts":[{"host":"amazon-bedrock","name":"Amazon Bedrock","kind":"cloud","url":"https://docs.aws.amazon.com/bedrock/latest/userguide/models-supported.html","inputPerMillion":2,"outputPerMillion":10,"cacheReadPerMillion":0.2,"contextTokens":1000000,"maxOutputTokens":128000,"deprecated":false},{"host":"digitalocean","name":"DigitalOcean","kind":"cloud","url":"https://docs.digitalocean.com/products/gradient-ai-platform/details/models/","inputPerMillion":2,"outputPerMillion":10,"cacheReadPerMillion":0.2,"contextTokens":1000000,"maxOutputTokens":128000,"deprecated":false},{"host":"google-vertex","name":"Google Vertex AI","kind":"cloud","url":"https://cloud.google.com/vertex-ai/generative-ai/docs/models","inputPerMillion":2,"outputPerMillion":10,"cacheReadPerMillion":0.2,"contextTokens":1000000,"maxOutputTokens":128000,"deprecated":false},{"host":"azure","name":"Microsoft Azure","kind":"cloud","url":"https://learn.microsoft.com/en-us/azure/ai-services/openai/concepts/models","inputPerMillion":2,"outputPerMillion":10,"cacheReadPerMillion":0.2,"contextTokens":1000000,"maxOutputTokens":128000,"deprecated":false},{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":2,"outputPerMillion":10,"cacheReadPerMillion":0.2,"contextTokens":1000000,"maxOutputTokens":128000,"deprecated":false},{"host":"snowflake","name":"Snowflake Cortex","kind":"cloud","url":"https://docs.snowflake.com/en/user-guide/snowflake-cortex/cortex-rest-api","inputPerMillion":2,"outputPerMillion":10,"cacheReadPerMillion":0.2,"contextTokens":1000000,"maxOutputTokens":64000,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:livebench","name":"LiveBench","url":"https://livebench.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:terminal-bench","name":"Terminal-Bench","url":"https://www.tbench.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:lmarena","name":"LMArena","url":"https://huggingface.co","license":"CC BY 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gemini-3-5-flash","providerId":"google","slug":"gemini-3-5-flash","name":"Gemini 3.5 Flash","provider":"Google","description":"Gemini 3.5 Flash is a reasoning model from Google. 51 benchmarks count toward its index, in 7 categories.","logoUrl":null,"releaseDate":"2026-05-19","contextWindowTokens":1048576,"modalities":["text","image","video","file","audio"],"openWeights":false,"details":{"apiId":"google/gemini-3.5-flash","maxOutputTokens":65536,"knowledgeCutoff":"2025-01-01","retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.15,"weightsUrl":null},"reasoning":true,"overallScore":64.19182361535108,"overallScoreRange":{"low":61.42757071947779,"high":66.95607651122438},"badges":[],"capabilityScores":{"agentic":64.68136999798323,"coding":60.78375702576335,"reasoning":62.21659022872048,"multimodal":64.68292766006164,"knowledge":68.4692149525683,"multilingual":null,"instruction":69.94276760609966,"math":61.491672809178056},"capabilityCounts":{"math":7,"agentic":14,"knowledge":6,"reasoning":9,"multimodal":4,"coding":9,"instruction":2},"benchmarks":[{"benchmarkId":"arc-prize:v1-public-eval:overall","benchmarkName":"ARC-AGI-1 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":96,"unit":"percent","family":"arc-prize:v1-public-eval:overall","index":null,"aboveFit":false,"run":"high effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.3201,"latency":null},{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":95.55555555555556,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":64.51274413550408,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.6696111689702042,"costPerTest":null,"latency":null},{"benchmarkId":"tau2Bench","benchmarkName":"τ²-Bench Tool-Agent-User Evaluation","category":"agentic","score":95.3,"unit":"percent","family":"tau2-bench","index":66.0279577114679,"aboveFit":false,"run":null,"source":"Victor Barres, Honghua Dong, Soham Ray, Xujie Si, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2506.07982","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":92.8030303030303,"unit":"percent","family":"gpqa-diamond","index":62.3255042815417,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.6387002835275906,"costPerTest":null,"latency":null},{"benchmarkId":"gpqaDiamond","benchmarkName":"GPQA Diamond","category":"knowledge","score":92.676,"unit":"percent","family":"gpqa-diamond","index":62.20899555871553,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:gpqa:overall","benchmarkName":"GPQA Diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":92.676,"unit":"percent","family":"gpqa-diamond","index":62.20899555871553,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","measuredAt":"2026-09-01","stderr":1.462,"costPerTest":0.063842,"latency":27.655},{"benchmarkId":"arc-prize:v1-semi-private:overall","benchmarkName":"ARC-AGI-1 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":92.5,"unit":"percent","family":"arc-prize:v1-semi-private:overall","index":69.51294838532246,"aboveFit":false,"run":"high effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.4278,"latency":null},{"benchmarkId":"gpqa","benchmarkName":"Graduate-Level Google-Proof Q&A","category":"knowledge","score":92.2,"unit":"percent","family":"gpqa-diamond","index":61.77242136545371,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":92.2,"unit":"percent","family":"gpqa-diamond","index":61.77242136545371,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:mmlu_pro:overall","benchmarkName":"MMLU Pro","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":89.515,"unit":"percent","family":"mmlu-pro","index":57.25891375687409,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","measuredAt":"2026-09-01","stderr":0.306,"costPerTest":0.02498,"latency":26.262},{"benchmarkId":"vals-ai:mmmu:overall","benchmarkName":"MMMU Pro","category":"multimodal","domain":null,"eligible":true,"ranked":true,"score":88.266,"unit":"percent","family":"mmmu-pro","index":70.02327002084391,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmmu","measuredAt":"2026-09-01","stderr":0.774,"costPerTest":0.025155,"latency":12.226},{"benchmarkId":"livebench:livebench-math:overall","benchmarkName":"LiveBench Mathematics","category":"math","domain":null,"eligible":true,"ranked":true,"score":88.24375,"unit":"percent","family":"livebench:livebench-math:overall","index":62.76184115793458,"aboveFit":false,"run":"high effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Mathematics","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.138034036939314,"latency":null},{"benchmarkId":"vals-ai:lcb:overall","benchmarkName":"LiveCodeBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":87.604,"unit":"percent","family":"livecodebench","index":62.032417129054735,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","measuredAt":"2026-09-01","stderr":0.95,"costPerTest":0.085937,"latency":64.371},{"benchmarkId":"livebench:livebench-language:overall","benchmarkName":"LiveBench Language","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":84.58433333333333,"unit":"percent","family":"livebench:livebench-language:overall","index":70.20751943358451,"aboveFit":false,"run":"high effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Language","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.05945,"latency":null},{"benchmarkId":"aaMmmuPro","benchmarkName":"Artificial Analysis MMMU-Pro","category":"multimodal","score":84.3,"unit":"percent","family":"mmmu-pro","index":64.86186897738129,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmmu-pro","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"charxiv","benchmarkName":"CharXiv Reasoning","category":"multimodal","score":84.2,"unit":"percent","family":"charxiv","index":60.04576846411299,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:legal_bench:overall","benchmarkName":"LegalBench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":83.602,"unit":"percent","family":"vals-ai:legal_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","measuredAt":"2026-10-13","stderr":0.849,"costPerTest":0.007553,"latency":34.576},{"benchmarkId":"mcpAtlas","benchmarkName":"MCP Atlas","category":"agentic","score":83.6,"unit":"percent","family":"mcpAtlas","index":67.35815514320996,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmmuPro","benchmarkName":"Massive Multi-discipline Multimodal Understanding Pro","category":"multimodal","score":83.6,"unit":"percent","family":"mmmu-pro","index":63.950880391797874,"aboveFit":false,"run":null,"source":"MMMU-Pro authors","sourceUrl":"https://arxiv.org/abs/2409.02813","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-reasoning:overall","benchmarkName":"LiveBench Reasoning","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":82.00475,"unit":"percent","family":"livebench","index":67.85146233091874,"aboveFit":false,"run":"high effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Reasoning","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.14888465346534654,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-2025-02-28-public:overall","benchmarkName":"FrontierMath-2025-02-28-Public","category":"math","domain":null,"eligible":false,"ranked":true,"score":80,"unit":"percent","family":"epoch-ai:epoch-frontiermath-2025-02-28-public:overall","index":null,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":13.333333333333334,"costPerTest":null,"latency":null},{"benchmarkId":"extractBenchLlamaIndex","benchmarkName":"ExtractBench: schema-guided enterprise document extraction (LlamaIndex leaderboard)","category":"multimodal","score":79.84,"unit":"percent","family":"extractBenchLlamaIndex","index":57.94425430867281,"aboveFit":false,"run":null,"source":"Boyang Zhang, Adrian Lyjak, Eli Stewart, Zhaoqi Li and Simon Suo (LlamaIndex)","sourceUrl":"https://arxiv.org/abs/2607.29677","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-swe-bench-verified:overall","benchmarkName":"SWE-Bench verified","category":"coding","domain":null,"eligible":true,"ranked":true,"score":79.33884297520662,"unit":"percent","family":"swe-bench-verified","index":59.74489237842572,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.8422420250424887,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:swebench:overall","benchmarkName":"SWE-bench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":78.8,"unit":"percent","family":"swe-bench-verified","index":59.34746465882657,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/swebench","measuredAt":"2026-09-01","stderr":1.83,"costPerTest":0.953882,"latency":254.129},{"benchmarkId":"osWorldVerified","benchmarkName":"OSWorld-Verified","category":"agentic","score":78.4,"unit":"percent","family":"osWorldVerified","index":63.76996418790951,"aboveFit":false,"run":null,"source":"Tianbao Xie, Danyang Zhang, Jixuan Chen, Xiaochuan Li, Siheng Zhao, Ruisheng Cao, Toh Jing Hua, Zhoujun Cheng, Dongchan Shin, Fangyu Lei, Yitao Liu, Yiheng Xu, Shuyan Zhou, Silvio Savarese, Caiming Xiong, Victor Zhong, Tao Yu","sourceUrl":"https://os-world.github.io/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-coding:overall","benchmarkName":"LiveBench Coding","category":"coding","domain":null,"eligible":true,"ranked":true,"score":78.1845,"unit":"percent","family":"livebench:livebench-coding:overall","index":64.78358851077434,"aboveFit":false,"run":"high effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Coding","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.06704273504273504,"latency":null},{"benchmarkId":"mrcrv2","benchmarkName":"MRCRv2","category":"reasoning","score":77.3,"unit":"percent","family":"mrcrv2","index":58.1412310968843,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:medscribe:overall","benchmarkName":"MedScribe","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":76.574,"unit":"percent","family":"vals-ai:medscribe:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medscribe","measuredAt":"2026-10-29","stderr":1.923,"costPerTest":0.166341,"latency":57.075},{"benchmarkId":"ifBench","benchmarkName":"Instruction Following Benchmark","category":"instruction","score":76.3,"unit":"percent","family":"ifbench-family","index":60.326789303555984,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaIfBench","benchmarkName":"Artificial Analysis IFBench","category":"instruction","score":76.3,"unit":"percent","family":"ifbench-family","index":60.326789303555984,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"terminalBench21","benchmarkName":"Terminal-Bench 2.1","category":"agentic","score":76.2,"unit":"percent","family":"terminal-bench-2-1","index":65.7070999801959,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-instruction-following:overall","benchmarkName":"LiveBench Instruction Following","category":"instruction","domain":null,"eligible":true,"ranked":true,"score":75.6,"unit":"percent","family":"livebench:livebench-instruction-following:overall","index":79.55874590864335,"aboveFit":false,"run":"high effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=IF","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.133014,"latency":null},{"benchmarkId":"vals-ai:tax_eval_v2:overall","benchmarkName":"TaxEval v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":74.366,"unit":"percent","family":"vals-ai:tax_eval_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","measuredAt":"2026-09-01","stderr":0.846,"costPerTest":0.038136,"latency":19.694},{"benchmarkId":"vals-ai:terminal-bench-2-1:overall","benchmarkName":"Terminal-Bench 2.1","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":74.157,"unit":"percent","family":"terminal-bench-2-1","index":64.51224047057761,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2-1","measuredAt":"2026-10-01","stderr":1.124,"costPerTest":0.868179,"latency":373.223},{"benchmarkId":"parseBenchLlamaIndex","benchmarkName":"ParseBench: document parsing for AI agents (LlamaIndex leaderboard)","category":"multimodal","score":73.22,"unit":"percent","family":"parseBenchLlamaIndex","index":75.87981889007943,"aboveFit":false,"run":null,"source":"Boyang Zhang, Sebastián G. Acosta, Preston Carlson, Sacha Bron, Pierre-Loïc Doulcet, Daniel B. Ospina and Simon Suo (LlamaIndex)","sourceUrl":"https://arxiv.org/abs/2604.08538","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"arc-prize:v2-public-eval:overall","benchmarkName":"ARC-AGI-2 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":72.08,"unit":"percent","family":"arc-prize:v2-public-eval:overall","index":null,"aboveFit":false,"run":"high effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.9001,"latency":null},{"benchmarkId":"arc-prize:v2-semi-private:overall","benchmarkName":"ARC-AGI-2 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":72.08,"unit":"percent","family":"arc-prize:v2-semi-private:overall","index":72.28294464860387,"aboveFit":false,"run":"high effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.8503,"latency":null},{"benchmarkId":"swebench-experiments:swe-bench-verified:overall","benchmarkName":"SWE-bench Verified","category":"coding","domain":null,"eligible":true,"ranked":true,"score":71.8,"unit":"percent","family":"swe-bench-verified","index":54.18456161434072,"aboveFit":false,"run":"medium effort · mini-SWE-agent","source":"SWE-bench team","sourceUrl":"https://github.com/SWE-bench/experiments/tree/main/evaluation/verified","measuredAt":"2026-09-01","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":69.3,"unit":"percent","family":"lcr","index":54.843429882256956,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:mortgage_tax:overall","benchmarkName":"MortgageTax","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":68.124,"unit":"percent","family":"vals-ai:mortgage_tax:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mortgage_tax","measuredAt":"2026-09-01","stderr":0.91,"costPerTest":0.017873,"latency":11.428},{"benchmarkId":"vals-ai:terminal-bench-2:overall","benchmarkName":"Terminal-Bench 2.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":67.416,"unit":"percent","family":"vals-ai:terminal-bench-2:overall","index":67.1431969239635,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2","measuredAt":"2026-06-04","stderr":1.297,"costPerTest":1.192081,"latency":408.048},{"benchmarkId":"swebench-experiments:swe-bench-multilingual:overall","benchmarkName":"SWE-bench Multilingual","category":"coding","domain":null,"eligible":true,"ranked":true,"score":67,"unit":"percent","family":"swe-bench-multilingual","index":null,"aboveFit":false,"run":"medium effort · mini-SWE-agent","source":"SWE-bench team","sourceUrl":"https://github.com/SWE-bench/experiments/tree/main/evaluation/multilingual","measuredAt":"2026-09-02","stderr":null,"costPerTest":0.62704581,"latency":null},{"benchmarkId":"epoch-ai:epoch-simpleqa-verified:overall","benchmarkName":"SimpleQA Verified","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":66.2,"unit":"percent","family":"epoch-ai:epoch-simpleqa-verified:overall","index":79.06949633212267,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.4965960710224384,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-data-analysis:overall","benchmarkName":"LiveBench Data Analysis","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":64.85733333333333,"unit":"percent","family":"livebench","index":45.63811583482866,"aboveFit":false,"run":"high effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Data%20Analysis","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.2456946666666667,"latency":null},{"benchmarkId":"vals-ai:corp_fin_v2:overall","benchmarkName":"CorpFin v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":64.686,"unit":"percent","family":"vals-ai:corp_fin_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","measuredAt":"2026-08-12","stderr":0.942,"costPerTest":0.140208,"latency":8.7},{"benchmarkId":"vals-ai:emb:overall","benchmarkName":"EMB","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":63.548,"unit":"percent","family":"vals-ai:emb:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/emb","measuredAt":"2026-10-20","stderr":2.752,"costPerTest":5.173115,"latency":1123.555},{"benchmarkId":"epoch-ai:epoch-frontiermath-tiers-1-3-v2-private:overall","benchmarkName":"FrontierMath-Tiers-1-3-v2-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":62.807017543859644,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tiers-1-3-v2-private:overall","index":63.953277471101984,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.867975375107462,"costPerTest":null,"latency":null},{"benchmarkId":"gertLabs","benchmarkName":"Gert Labs Composite Game Benchmark","category":"agentic","score":61.85,"unit":"percent","family":"gertLabs","index":65.62774013429453,"aboveFit":false,"run":null,"source":"Gert Labs","sourceUrl":"https://gertlabs.com/rankings","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:public-benefits-bench:overall","benchmarkName":"Public Benefits Bench v1.1","category":null,"domain":"Public benefits","eligible":false,"ranked":true,"score":59.472,"unit":"percent","family":"vals-ai:public-benefits-bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/public-benefits-bench","measuredAt":"2026-10-12","stderr":1.277,"costPerTest":1.113099,"latency":338.872},{"benchmarkId":"vals-ai:public-benefits-bench-v1:overall","benchmarkName":"Public Benefits Bench v1","category":null,"domain":"Public benefits","eligible":false,"ranked":true,"score":57.984,"unit":"percent","family":"vals-ai:public-benefits-bench-v1:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/public-benefits-bench-v1","measuredAt":"2026-06-09","stderr":1.284,"costPerTest":0.775836,"latency":273.985},{"benchmarkId":"vals-ai:fabv2:overall","benchmarkName":"Finance Agent (v2)","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":57.861,"unit":"percent","family":"vals-ai:fabv2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/fabv2","measuredAt":"2026-10-29","stderr":0.231,"costPerTest":2.505488,"latency":322.279},{"benchmarkId":"toolathlon","benchmarkName":"Toolathlon","category":"agentic","score":56.5,"unit":"percent","family":"toolathlon","index":67.90022137494529,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:medcode:overall","benchmarkName":"MedCode","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":55.825,"unit":"percent","family":"vals-ai:medcode:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medcode","measuredAt":"2026-10-29","stderr":2.113,"costPerTest":0.073716,"latency":25.29},{"benchmarkId":"swePro","benchmarkName":"SWE-bench Pro","category":"coding","score":55.1,"unit":"percent","family":"swePro","index":55.763093637371384,"aboveFit":false,"run":null,"source":"Xiang Deng, Jeff Da, Edwin Pan, Yannis Yiming He, Charles Ide, Kanak Garg, Niklas Lauffer, Andrew Park, Nitin Pasari, Chetan Rane, Karmini Sampath, Maya Krishnan, Srivatsa Kundurthy, Sean Hendryx, Zifan Wang, Chen Bo Calvin Zhang, Noah Jacobson, Bing Liu, Brad Kenstler","sourceUrl":"https://arxiv.org/abs/2509.16941","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaSciCode","benchmarkName":"Artificial Analysis SciCode","category":"coding","score":53.9,"unit":"percent","family":"scicode-family","index":66.10875123426538,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/scicode","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"sciCode","benchmarkName":"Scientific Code Benchmark","category":"coding","score":53.1,"unit":"percent","family":"scicode-family","index":65.0657983171336,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:skillsbench:overall","benchmarkName":"SkillsBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":52.737,"unit":"percent","family":"vals-ai:skillsbench:overall","index":64.39577855320067,"aboveFit":false,"run":"OpenHands","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/skillsbench","measuredAt":"2026-10-29","stderr":4.475,"costPerTest":1.253121,"latency":435.011},{"benchmarkId":"omniscienceAccuracy","benchmarkName":"Artificial Analysis Omniscience Accuracy","category":"knowledge","score":51.9,"unit":"percent","family":"omniscienceAccuracy","index":76.09723603931451,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaEnterpriseOpsGym","benchmarkName":"Artificial Analysis EnterpriseOps-Gym","category":"agentic","score":50.1,"unit":"percent","family":"aaEnterpriseOpsGym","index":null,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/enterprise-ops-gym-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"cursorBench31","benchmarkName":"cursorBench31","category":"coding","score":49.8,"unit":"percent","family":"cursorBench31","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-agentic-coding:overall","benchmarkName":"LiveBench Agentic Coding","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":48.989666666666665,"unit":"percent","family":"livebench:livebench-agentic-coding:overall","index":59.684899219606955,"aboveFit":false,"run":"high effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Agentic%20Coding","measuredAt":"2026-06-25","stderr":null,"costPerTest":1.0188416666666666,"latency":null},{"benchmarkId":"cursorBench32","benchmarkName":"cursorBench32","category":"coding","score":48.8,"unit":"percent","family":"cursorBench32","index":55.93192835800779,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:vibe-code:overall","benchmarkName":"Vibe Code Bench v1.1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":48.683,"unit":"percent","family":"vals-ai:vibe-code:overall","index":59.81511385825611,"aboveFit":false,"run":"OpenHands","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","measuredAt":"2026-11-09","stderr":4.727,"costPerTest":2.540385,"latency":893.074},{"benchmarkId":"apexAgentsAa","benchmarkName":"APEX-Agents-AA","category":"agentic","score":47.1,"unit":"percent","family":"apexAgentsAa","index":74.44342099797065,"aboveFit":false,"run":null,"source":"Artificial Analysis / Mercor","sourceUrl":"https://artificialanalysis.ai/evaluations/apex-agents-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaAnalystAgent","benchmarkName":"Artificial Analysis AnalystAgent","category":"agentic","score":45,"unit":"percent","family":"aaAnalystAgent","index":66.3605953841917,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/aa-analyst-agent","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-chess-puzzles:overall","benchmarkName":"Chess Puzzles","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":45,"unit":"percent","family":"epoch-ai:epoch-chess-puzzles:overall","index":68.14035950689328,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":5,"costPerTest":null,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":42.7,"unit":"percent","family":"hle","index":67.2272445260857,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gdpvalAaNormalized","benchmarkName":"GDPval-AA normalized","category":"agentic","score":42.2,"unit":"percent","family":"gdpvalAaNormalized","index":64.2057942605881,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaItbench","benchmarkName":"Artificial Analysis ITBench-AA","category":"agentic","score":40.3,"unit":"percent","family":"aaItbench","index":58.53488664899706,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/itbench-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hle","benchmarkName":"Humanity's Last Exam","category":"knowledge","score":40.2,"unit":"percent","family":"hle","index":64.71901266351121,"aboveFit":false,"run":null,"source":"Center for AI Safety, Scale AI, and thousands of expert contributors","sourceUrl":"https://lastexam.ai/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-2025-02-28-private:overall","benchmarkName":"FrontierMath-2025-02-28-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":38.96551724137931,"unit":"percent","family":"epoch-ai:epoch-frontiermath-2025-02-28-private:overall","index":64.31206017790257,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.868659061400048,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:web_search:overall","benchmarkName":"Vals Web Search Index","category":null,"domain":"Other","eligible":false,"ranked":true,"score":37.040653846153845,"unit":"percent","family":"vals-ai:web_search:overall","index":null,"aboveFit":false,"run":"Native","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/web_search","measuredAt":"2026-11-04","stderr":1.8547389298089656,"costPerTest":0.499017,"latency":249.53663461538463},{"benchmarkId":"blueprintBench2","benchmarkName":"Blueprint-Bench 2","category":"multimodal","score":33.6,"unit":"percent","family":"blueprintBench2","index":null,"aboveFit":false,"run":null,"source":"Google DeepMind","sourceUrl":"https://x.com/GoogleDeepMind","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-mystery-game-puzzles:overall","benchmarkName":"Mystery Game Puzzles","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":32,"unit":"percent","family":"epoch-ai:epoch-mystery-game-puzzles:overall","index":63.46227489663569,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":4.688261722621506,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:proof_bench:overall","benchmarkName":"ProofBench v1.1","category":"math","domain":null,"eligible":true,"ranked":true,"score":31,"unit":"percent","family":"vals-ai:proof_bench:overall","index":57.859206321415705,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/proof_bench","measuredAt":"2026-11-09","stderr":4.648,"costPerTest":0.569863,"latency":394.476},{"benchmarkId":"vals-ai:legal_research:overall","benchmarkName":"Legal Research Bench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":30.769,"unit":"percent","family":"vals-ai:legal_research:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_research","measuredAt":"2026-11-09","stderr":3.208,"costPerTest":1.305281,"latency":277.753},{"benchmarkId":"epoch-ai:epoch-oeis-open-lite:overall","benchmarkName":"OEIS Open Lite","category":"math","domain":null,"eligible":true,"ranked":true,"score":28.999999999999996,"unit":"percent","family":"epoch-ai:epoch-oeis-open-lite:overall","index":null,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":4.560480215720686,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tier-4-v2-private:overall","benchmarkName":"FrontierMath-Tier-4-v2-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":26.82926829268293,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tier-4-v2-private:overall","index":59.02818358680503,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":7.005564203095157,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:code-migration:overall","benchmarkName":"Code Migration","category":"coding","domain":null,"eligible":true,"ranked":true,"score":26.745,"unit":"percent","family":"vals-ai:code-migration:overall","index":59.39715375067904,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/code-migration","measuredAt":"2026-10-29","stderr":4.104,"costPerTest":5.725122,"latency":1168.454},{"benchmarkId":"mrcr1m","benchmarkName":"MRCR 1M","category":"reasoning","score":26.6,"unit":"percent","family":"mrcr1m","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-oeis-open:overall","benchmarkName":"OEIS Open","category":"math","domain":null,"eligible":true,"ranked":true,"score":22.15447154471545,"unit":"percent","family":"epoch-ai:epoch-oeis-open:overall","index":null,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.8741605290061465,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:tax_agent_bench:overall","benchmarkName":"Tax Agent Bench","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":21.652,"unit":"percent","family":"vals-ai:tax_agent_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench","measuredAt":"2026-11-09","stderr":2.963,"costPerTest":0.859679,"latency":233.001},{"benchmarkId":"researchClawBench","benchmarkName":"ResearchClawBench","category":"agentic","score":18,"unit":"percent","family":"researchClawBench","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-private:overall","benchmarkName":"FrontierMath-Tier-4-2025-07-01-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":14.583333333333334,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-private:overall","index":58.014396813582465,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":5.148145136372291,"costPerTest":null,"latency":null},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":13.1,"unit":"percent","family":"critpt","index":62.20918746152833,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:terminal-bench-4:overall","benchmarkName":"Terminal-Bench 4.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":6.061,"unit":"percent","family":"terminal-bench-4","index":57.394010410885414,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-4","measuredAt":"2026-11-09","stderr":1.515,"costPerTest":5.981757,"latency":4286.824},{"benchmarkId":"vals-ai:terminal-bench-science:overall","benchmarkName":"Terminal-Bench-Science","category":null,"domain":"Science","eligible":false,"ranked":true,"score":5.714,"unit":"percent","family":"vals-ai:terminal-bench-science:overall","index":null,"aboveFit":false,"run":"Mini-SWE-agent","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-science","measuredAt":"2026-11-09","stderr":2.794,"costPerTest":3.802646,"latency":3654.61},{"benchmarkId":"epoch-ai:epoch-ebr-bench:overall","benchmarkName":"EBR-bench","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":4.761904761904762,"unit":"percent","family":"epoch-ai:epoch-ebr-bench:overall","index":54.6121470974857,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mazeBench","benchmarkName":"MazeBench: visual spatial reasoning in a 3D open world","category":"agentic","score":0,"unit":"percent","family":"mazeBench","index":61.97866734834783,"aboveFit":false,"run":null,"source":"Jonathan Pappas and David Pappas","sourceUrl":"https://mazebench.com/blog?post=maze-bench-results","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:programbench:overall","benchmarkName":"ProgramBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":0,"unit":"percent","family":"vals-ai:programbench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/programbench","measuredAt":"2026-11-09","stderr":0,"costPerTest":null,"latency":1104.517},{"benchmarkId":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-public:overall","benchmarkName":"FrontierMath-Tier-4-2025-07-01-Public","category":"math","domain":null,"eligible":false,"ranked":true,"score":0,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-public:overall","index":null,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":1.5,"outputPerMillion":9,"batch":{"inputPerMillion":0.75,"outputPerMillion":4.5},"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":137,"timeToFirstToken":1.652,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":{"elo":1482.9362147697643,"eloUpper":1486.82505426353,"eloLower":1479.047375275999,"votes":47642,"rank":19,"rankUpper":null,"rankLower":null,"arenaName":"Gemini 3.5 Flash-high","organization":"google","updatedAt":"2026-11-09"},"quality":{"coveragePercent":95,"benchmarkCount":51,"categoryCount":7,"confidence":"high","state":"available"},"hosts":[{"host":"databricks","name":"Databricks","kind":"cloud","url":"https://docs.databricks.com/aws/en/machine-learning/foundation-models/","inputPerMillion":1.5,"outputPerMillion":9,"cacheReadPerMillion":0.15,"contextTokens":1048576,"maxOutputTokens":65536,"deprecated":false},{"host":"google-vertex","name":"Google Vertex AI","kind":"maker","url":"https://cloud.google.com/vertex-ai/generative-ai/docs/models","inputPerMillion":1.5,"outputPerMillion":9,"cacheReadPerMillion":0.15,"contextTokens":1048576,"maxOutputTokens":65536,"deprecated":false},{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":1.5,"outputPerMillion":9,"cacheReadPerMillion":0.15,"contextTokens":1048576,"maxOutputTokens":65536,"deprecated":false},{"host":"sap-ai-core","name":"SAP AI Core","kind":"cloud","url":"https://help.sap.com/docs/sap-ai-core","inputPerMillion":1.5,"outputPerMillion":9,"cacheReadPerMillion":0.15,"contextTokens":1048576,"maxOutputTokens":65536,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:arc-prize-foundation","name":"ARC Prize Foundation","url":"https://arcprize.org","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:livebench","name":"LiveBench","url":"https://livebench.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:swe-bench-team","name":"SWE-bench team","url":"https://github.com","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"qwen3-8-max-preview","providerId":"alibaba","slug":"qwen3-8-max-preview","name":"Qwen3.8 Max Preview","provider":"Alibaba","description":"Qwen3.8 Max Preview is a reasoning model from Alibaba in the Qwen3.8 Max family. 10 benchmarks count toward its index, in 5 categories.","logoUrl":null,"releaseDate":"2026-07-19","contextWindowTokens":null,"modalities":["text"],"openWeights":false,"details":null,"reasoning":true,"overallScore":64.16995837639051,"overallScoreRange":{"low":57.056555399898244,"high":71.28336135288278},"badges":[],"capabilityScores":{"agentic":73.46522560225019,"coding":63.429830670632455,"reasoning":66.75228319102122,"multimodal":62.909750579702546,"knowledge":59.272929530294704,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"knowledge":3,"multimodal":1,"reasoning":2,"agentic":2,"coding":2},"benchmarks":[{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":92.8,"unit":"percent","family":"gpqa-diamond","index":62.322724970405574,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaMmmuPro","benchmarkName":"Artificial Analysis MMMU-Pro","category":"multimodal","score":82.8,"unit":"percent","family":"mmmu-pro","index":62.909750579702546,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmmu-pro","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":80.3,"unit":"percent","family":"lcr","index":62.39031566369921,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gdpvalAaNormalized","benchmarkName":"GDPval-AA normalized","category":"agentic","score":58.3,"unit":"percent","family":"gdpvalAaNormalized","index":75.71692012382658,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"openHarmonyBench","benchmarkName":"OpenHarmony Bench v1.0","category":"coding","score":56,"unit":"percent","family":"openHarmonyBench","index":63.09755417054605,"aboveFit":false,"run":null,"source":"OpenHarmony Bench authors","sourceUrl":"https://arxiv.org/abs/2608.16022","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaSciCode","benchmarkName":"Artificial Analysis SciCode","category":"coding","score":52.1,"unit":"percent","family":"scicode-family","index":63.76210717071886,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/scicode","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":43.1,"unit":"percent","family":"hle","index":67.62856162409761,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"apexAgentsAa","benchmarkName":"APEX-Agents-AA","category":"agentic","score":42.4,"unit":"percent","family":"apexAgentsAa","index":71.21353108067379,"aboveFit":false,"run":null,"source":"Artificial Analysis / Mercor","sourceUrl":"https://artificialanalysis.ai/evaluations/apex-agents-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"omniscienceAccuracy","benchmarkName":"Artificial Analysis Omniscience Accuracy","category":"knowledge","score":29.5,"unit":"percent","family":"omniscienceAccuracy","index":47.8675019963809,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":17.7,"unit":"percent","family":"critpt","index":71.11425071834321,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":{"elo":1485.2093041935336,"eloUpper":1489.7959141573651,"eloLower":1480.6226942297021,"votes":36324,"rank":16,"rankUpper":null,"rankLower":null,"arenaName":"Qwen3.8-Max","organization":"alibaba","updatedAt":"2026-11-09"},"quality":{"coveragePercent":80,"benchmarkCount":10,"categoryCount":5,"confidence":"low","state":"available"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"sakana-fugu","providerId":"sakana","slug":"sakana-fugu","name":"Sakana Fugu","provider":"Sakana","description":"Sakana Fugu is a reasoning model from Sakana. 8 benchmarks count toward its index, in 5 categories.","logoUrl":null,"releaseDate":"2026-06-22","contextWindowTokens":1000000,"modalities":["text"],"openWeights":false,"details":null,"reasoning":true,"overallScore":63.94132288735705,"overallScoreRange":{"low":56.224364561463815,"high":71.65828121325029},"badges":[],"capabilityScores":{"agentic":68.04652143808782,"coding":64.65642965903767,"reasoning":64.0085145158013,"multimodal":61.070417047123115,"knowledge":68.27057653570438,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"knowledge":2,"coding":3,"reasoning":1,"multimodal":1,"agentic":1},"benchmarks":[{"benchmarkId":"gpqa","benchmarkName":"Graduate-Level Google-Proof Q&A","category":"knowledge","score":95.5,"unit":"percent","family":"gpqa-diamond","index":64.799091192689,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gpqaDiamond","benchmarkName":"GPQA Diamond","category":"knowledge","score":95.5,"unit":"percent","family":"gpqa-diamond","index":64.799091192689,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"liveCodeBenchV6","benchmarkName":"LiveCodeBench v6","category":"coding","score":92.9,"unit":"percent","family":"liveCodeBenchV6","index":60.693280071071754,"aboveFit":false,"run":null,"source":"LiveCodeBench maintainers","sourceUrl":"https://github.com/LiveCodeBench/LiveCodeBench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"liveCodeBenchPro","benchmarkName":"LiveCodeBench Pro","category":"coding","score":87.8,"unit":"percent","family":"liveCodeBenchPro","index":null,"aboveFit":false,"run":null,"source":"LiveCodeBench Pro authors","sourceUrl":"https://arxiv.org/abs/2506.11928","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mrcrv2","benchmarkName":"MRCRv2","category":"reasoning","score":86.6,"unit":"percent","family":"mrcrv2","index":64.0085145158013,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"charxiv","benchmarkName":"CharXiv Reasoning","category":"multimodal","score":85.1,"unit":"percent","family":"charxiv","index":61.070417047123115,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"terminalBench21","benchmarkName":"Terminal-Bench 2.1","category":"agentic","score":80.2,"unit":"percent","family":"terminal-bench-2-1","index":68.04652143808782,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"sciCode","benchmarkName":"Scientific Code Benchmark","category":"coding","score":60.1,"unit":"percent","family":"scicode-family","index":74.19163634203673,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"swePro","benchmarkName":"SWE-bench Pro","category":"coding","score":59,"unit":"percent","family":"swePro","index":59.08437256400452,"aboveFit":false,"run":null,"source":"Xiang Deng, Jeff Da, Edwin Pan, Yannis Yiming He, Charles Ide, Kanak Garg, Niklas Lauffer, Andrew Park, Nitin Pasari, Chetan Rane, Karmini Sampath, Maya Krishnan, Srivatsa Kundurthy, Sean Hendryx, Zifan Wang, Chen Bo Calvin Zhang, Noah Jacobson, Bing Liu, Brad Kenstler","sourceUrl":"https://arxiv.org/abs/2509.16941","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hleNoTools","benchmarkName":"Humanity's Last Exam without tools","category":"knowledge","score":47.2,"unit":"percent","family":"hle","index":71.74206187871977,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":80,"benchmarkCount":8,"categoryCount":5,"confidence":"low","state":"available"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"grok-4-5","providerId":"xai","slug":"grok-4-5","name":"Grok 4.5","provider":"xAI","description":"Grok 4.5 is a reasoning model from xAI. 47 benchmarks count toward its index, in 7 categories.","logoUrl":null,"releaseDate":"2026-07-08","contextWindowTokens":500000,"modalities":["text","image","file"],"openWeights":false,"details":{"apiId":"x-ai/grok-4.5","maxOutputTokens":450000,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.3,"weightsUrl":null},"reasoning":true,"overallScore":63.74549416076884,"overallScoreRange":{"low":60.886857248247885,"high":66.60413107328979},"badges":[],"capabilityScores":{"agentic":61.64550004589555,"coding":65.77730911577316,"reasoning":63.32786494117645,"multimodal":47.67802142874785,"knowledge":65.79470120524434,"multilingual":null,"instruction":71.32174663901442,"math":61.81836831891887},"capabilityCounts":{"math":5,"coding":14,"knowledge":6,"reasoning":8,"agentic":12,"multimodal":1,"instruction":1},"benchmarks":[{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":97.77777777777777,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":65.8056197926887,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.2743145865697103,"costPerTest":null,"latency":null},{"benchmarkId":"arc-prize:v1-public-eval:overall","benchmarkName":"ARC-AGI-1 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":94.75,"unit":"percent","family":"arc-prize:v1-public-eval:overall","index":null,"aboveFit":false,"run":"high effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.24,"latency":null},{"benchmarkId":"livebench:livebench-coding-2:overall","benchmarkName":"LiveBench Coding 2.0","category":"coding","domain":null,"eligible":true,"ranked":true,"score":94.3,"unit":"percent","family":"livebench:livebench-coding-2:overall","index":76.47577354601032,"aboveFit":false,"run":"xhigh effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Coding","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.9591758620689654,"latency":null},{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":93.43434343434343,"unit":"percent","family":"gpqa-diamond","index":62.904527434900146,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.4340395485738813,"costPerTest":null,"latency":null},{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":93.1,"unit":"percent","family":"gpqa-diamond","index":62.59787677288151,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:gpqa:overall","benchmarkName":"GPQA Diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":92.929,"unit":"percent","family":"gpqa-diamond","index":62.441040245470234,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","measuredAt":"2026-09-01","stderr":1.288,"costPerTest":0.044029,"latency":217.858},{"benchmarkId":"livebench:livebench-math:overall","benchmarkName":"LiveBench Mathematics","category":"math","domain":null,"eligible":true,"ranked":true,"score":90.8245,"unit":"percent","family":"livebench:livebench-math:overall","index":66.35868222461818,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Mathematics","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.06592084432717678,"latency":null},{"benchmarkId":"vulcanBench","benchmarkName":"VulcanBench v3","category":"coding","score":89.9,"unit":"percent","family":"vulcanBench","index":72.29949702833147,"aboveFit":false,"run":null,"source":"VulcanBench contributors","sourceUrl":"https://github.com/morganlinton/VulcanBench/tree/main","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:mmlu_pro:overall","benchmarkName":"MMLU Pro","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":89.216,"unit":"percent","family":"mmlu-pro","index":56.882209546254096,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","measuredAt":"2026-09-01","stderr":0.31,"costPerTest":0.014974,"latency":357.751},{"benchmarkId":"vals-ai:lcb:overall","benchmarkName":"LiveCodeBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":87.351,"unit":"percent","family":"livecodebench","index":61.822492598117044,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","measuredAt":"2026-09-01","stderr":0.961,"costPerTest":0.045235,"latency":589.356},{"benchmarkId":"livebench:livebench-reasoning:overall","benchmarkName":"LiveBench Reasoning","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":87.173,"unit":"percent","family":"livebench","index":74.54658908150319,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Reasoning","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.08795297029702971,"latency":null},{"benchmarkId":"vals-ai:medscribe:overall","benchmarkName":"MedScribe","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":86.884,"unit":"percent","family":"vals-ai:medscribe:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medscribe","measuredAt":"2026-10-29","stderr":1.944,"costPerTest":0.033208,"latency":704.513},{"benchmarkId":"vals-ai:swebench:overall","benchmarkName":"SWE-bench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":86.6,"unit":"percent","family":"swe-bench-verified","index":65.10041376553936,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/swebench","measuredAt":"2026-09-01","stderr":1.525,"costPerTest":0.54034,"latency":199.573},{"benchmarkId":"vals-ai:legal_bench:overall","benchmarkName":"LegalBench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":85.965,"unit":"percent","family":"vals-ai:legal_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","measuredAt":"2026-10-13","stderr":0.413,"costPerTest":0.004535,"latency":67.875},{"benchmarkId":"livebench:livebench-research:overall","benchmarkName":"LiveBench Research","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":85.9,"unit":"percent","family":"livebench-2-agentic","index":81.41081263603746,"aboveFit":false,"run":"xhigh effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Research","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.8178727272727273,"latency":null},{"benchmarkId":"arc-prize:v1-semi-private:overall","benchmarkName":"ARC-AGI-1 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":85.67,"unit":"percent","family":"arc-prize:v1-semi-private:overall","index":66.37295080200732,"aboveFit":false,"run":"high effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.33,"latency":null},{"benchmarkId":"terminalBench21","benchmarkName":"Terminal-Bench 2.1","category":"agentic","score":83.3,"unit":"percent","family":"terminal-bench-2-1","index":69.85957306795405,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-language:overall","benchmarkName":"LiveBench Language","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":82.79533333333333,"unit":"percent","family":"livebench:livebench-language:overall","index":68.18450099201247,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Language","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.03625333333333333,"latency":null},{"benchmarkId":"aaTerminalBench21","benchmarkName":"Artificial Analysis Terminal-Bench v2.1","category":"agentic","score":81.6,"unit":"percent","family":"aaTerminalBench21","index":64.15721615013558,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/terminalbench-v2-1","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaMmmuPro","benchmarkName":"Artificial Analysis MMMU-Pro","category":"multimodal","score":80.4,"unit":"percent","family":"mmmu-pro","index":59.78636114341656,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmmu-pro","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":79.3,"unit":"percent","family":"lcr","index":61.70423513811355,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-design:overall","benchmarkName":"LiveBench Design","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":78.1,"unit":"percent","family":"livebench-2-agentic","index":75.7064300956579,"aboveFit":false,"run":"xhigh effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Design","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.38772799999999996,"latency":null},{"benchmarkId":"livebench:livebench-data-analysis:overall","benchmarkName":"LiveBench Data Analysis","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":73.03766666666667,"unit":"percent","family":"livebench","index":56.23519773463955,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Data%20Analysis","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.11452266666666669,"latency":null},{"benchmarkId":"vals-ai:tax_eval_v2:overall","benchmarkName":"TaxEval v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":71.668,"unit":"percent","family":"vals-ai:tax_eval_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","measuredAt":"2026-09-01","stderr":0.892,"costPerTest":0.019211,"latency":195.458},{"benchmarkId":"livebench:livebench-instruction-following:overall","benchmarkName":"LiveBench Instruction Following","category":"instruction","domain":null,"eligible":true,"ranked":true,"score":71.52925,"unit":"percent","family":"livebench:livebench-instruction-following:overall","index":71.32174663901442,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=IF","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.046835,"latency":null},{"benchmarkId":"vals-ai:cyber:overall","benchmarkName":"CyberBench v1.1","category":null,"domain":"Other","eligible":false,"ranked":true,"score":71.012,"unit":"percent","family":"vals-ai:cyber:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/cyber","measuredAt":"2026-11-05","stderr":5.793,"costPerTest":2.949433,"latency":848.118},{"benchmarkId":"vals-ai:vibe-code:overall","benchmarkName":"Vibe Code Bench v1.1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":69.002,"unit":"percent","family":"vals-ai:vibe-code:overall","index":67.14652829368859,"aboveFit":false,"run":"OpenHands","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","measuredAt":"2026-11-09","stderr":4.48,"costPerTest":4.102194,"latency":830.057},{"benchmarkId":"livebench:livebench-coding:overall","benchmarkName":"LiveBench Coding","category":"coding","domain":null,"eligible":true,"ranked":true,"score":68.5855,"unit":"percent","family":"livebench:livebench-coding:overall","index":51.283673963042574,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Coding","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.020555555555555556,"latency":null},{"benchmarkId":"vals-ai:terminal-bench-2-1:overall","benchmarkName":"Terminal-Bench 2.1","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":67.79,"unit":"percent","family":"terminal-bench-2-1","index":60.78846636497817,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2-1","measuredAt":"2026-10-01","stderr":4.416,"costPerTest":0.283943,"latency":624.415},{"benchmarkId":"vals-ai:corp_fin_v2:overall","benchmarkName":"CorpFin v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":67.405,"unit":"percent","family":"vals-ai:corp_fin_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","measuredAt":"2026-08-12","stderr":0.924,"costPerTest":0.221291,"latency":91.881},{"benchmarkId":"cursorBench32","benchmarkName":"cursorBench32","category":"coding","score":66.7,"unit":"percent","family":"cursorBench32","index":70.64232186478793,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:skillsbench:overall","benchmarkName":"SkillsBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":66.034,"unit":"percent","family":"vals-ai:skillsbench:overall","index":74.29236556622251,"aboveFit":false,"run":"OpenHands","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/skillsbench","measuredAt":"2026-10-29","stderr":4.596,"costPerTest":0.578155,"latency":135.652},{"benchmarkId":"swePro","benchmarkName":"SWE-bench Pro","category":"coding","score":64.7,"unit":"percent","family":"swePro","index":63.93854945677602,"aboveFit":false,"run":null,"source":"Xiang Deng, Jeff Da, Edwin Pan, Yannis Yiming He, Charles Ide, Kanak Garg, Niklas Lauffer, Andrew Park, Nitin Pasari, Chetan Rane, Karmini Sampath, Maya Krishnan, Srivatsa Kundurthy, Sean Hendryx, Zifan Wang, Chen Bo Calvin Zhang, Noah Jacobson, Bing Liu, Brad Kenstler","sourceUrl":"https://arxiv.org/abs/2509.16941","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:mortgage_tax:overall","benchmarkName":"MortgageTax","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":61.804,"unit":"percent","family":"vals-ai:mortgage_tax:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mortgage_tax","measuredAt":"2026-09-01","stderr":0.968,"costPerTest":0.014998,"latency":117.461},{"benchmarkId":"vals-ai:mmmu:overall","benchmarkName":"MMMU Pro","category":"multimodal","domain":null,"eligible":true,"ranked":true,"score":61.792,"unit":"percent","family":"mmmu-pro","index":35.56968171407914,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmmu","measuredAt":"2026-09-01","stderr":1.168,"costPerTest":0.030117,"latency":714.648},{"benchmarkId":"arc-prize:v2-public-eval:overall","benchmarkName":"ARC-AGI-2 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":58.19,"unit":"percent","family":"arc-prize:v2-public-eval:overall","index":null,"aboveFit":false,"run":"high effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.81,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tiers-1-3-v2-private:overall","benchmarkName":"FrontierMath-Tiers-1-3-v2-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":57.19298245614035,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tiers-1-3-v2-private:overall","index":60.998320495389876,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.9360921879029935,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-agentic-coding:overall","benchmarkName":"LiveBench Agentic Coding","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":56.464666666666666,"unit":"percent","family":"livebench:livebench-agentic-coding:overall","index":67.39373940779473,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Agentic%20Coding","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.6838319444444445,"latency":null},{"benchmarkId":"aaSciCode","benchmarkName":"Artificial Analysis SciCode","category":"coding","score":55,"unit":"percent","family":"scicode-family","index":67.54281149532159,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/scicode","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"deepSwe","benchmarkName":"DeepSWE","category":"agentic","score":53,"unit":"percent","family":"deepSwe","index":59.60624108454575,"aboveFit":false,"run":null,"source":"Datacurve AI","sourceUrl":"https://deepswe.datacurve.ai/blog","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:emb:overall","benchmarkName":"EMB","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":52.911,"unit":"percent","family":"vals-ai:emb:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/emb","measuredAt":"2026-10-20","stderr":3.111,"costPerTest":1.55492,"latency":1107.838},{"benchmarkId":"arc-prize:v2-semi-private:overall","benchmarkName":"ARC-AGI-2 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":52.64,"unit":"percent","family":"arc-prize:v2-semi-private:overall","index":63.3573660616274,"aboveFit":false,"run":"high effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.78,"latency":null},{"benchmarkId":"omniscienceAccuracy","benchmarkName":"Artificial Analysis Omniscience Accuracy","category":"knowledge","score":51.6,"unit":"percent","family":"omniscienceAccuracy","index":75.71915924409666,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:fabv2:overall","benchmarkName":"Finance Agent (v2)","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":48.349,"unit":"percent","family":"vals-ai:fabv2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/fabv2","measuredAt":"2026-10-29","stderr":0.458,"costPerTest":1.156495,"latency":289.014},{"benchmarkId":"epoch-ai:epoch-simpleqa-verified:overall","benchmarkName":"SimpleQA Verified","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":48.3,"unit":"percent","family":"epoch-ai:epoch-simpleqa-verified:overall","index":64.15721615013558,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.5810153729833274,"costPerTest":null,"latency":null},{"benchmarkId":"tau2-bench:banking-knowledge:overall","benchmarkName":"τ²-bench Banking","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":47.93814432989691,"unit":"percent","family":"tau2-bench","index":34.599903758075655,"aboveFit":false,"run":"high effort · Sierra","source":"Sierra Research","sourceUrl":"https://github.com/sierra-research/tau2-bench","measuredAt":"2026-10-20","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gdpvalAaNormalized","benchmarkName":"GDPval-AA normalized","category":"agentic","score":44.2,"unit":"percent","family":"gdpvalAaNormalized","index":65.63574778397177,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-browser:overall","benchmarkName":"LiveBench Browser Use","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":44,"unit":"percent","family":"livebench-2-agentic","index":50.76803975887031,"aboveFit":false,"run":"xhigh effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Browser","measuredAt":"2026-10-14","stderr":null,"costPerTest":2.9176960000000003,"latency":null},{"benchmarkId":"vals-ai:medcode:overall","benchmarkName":"MedCode","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":43.291,"unit":"percent","family":"vals-ai:medcode:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medcode","measuredAt":"2026-10-29","stderr":2.313,"costPerTest":0.048461,"latency":59.637},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":42.7,"unit":"percent","family":"hle","index":67.2272445260857,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaTau3Banking","benchmarkName":"Artificial Analysis Tau3-Banking","category":"agentic","score":42.1,"unit":"percent","family":"aaTau3Banking","index":66.60429853472773,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/tau3-banking","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaEnterpriseOpsGym","benchmarkName":"Artificial Analysis EnterpriseOps-Gym","category":"agentic","score":40.8,"unit":"percent","family":"aaEnterpriseOpsGym","index":null,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/enterprise-ops-gym-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:ioi:overall","benchmarkName":"IOI","category":"coding","domain":null,"eligible":true,"ranked":true,"score":40.556,"unit":"percent","family":"vals-ai:ioi:overall","index":61.34226506007751,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/ioi","measuredAt":"2026-10-29","stderr":3.951,"costPerTest":5.637814,"latency":5194.818},{"benchmarkId":"vals-ai:legal_research:overall","benchmarkName":"Legal Research Bench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":37.981,"unit":"percent","family":"vals-ai:legal_research:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_research","measuredAt":"2026-11-09","stderr":3.373,"costPerTest":1.086814,"latency":953.767},{"benchmarkId":"vals-ai:code-migration:overall","benchmarkName":"Code Migration","category":"coding","domain":null,"eligible":true,"ranked":true,"score":36.596,"unit":"percent","family":"vals-ai:code-migration:overall","index":64.63039991373161,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/code-migration","measuredAt":"2026-10-29","stderr":4.141,"costPerTest":3.369806,"latency":1408.81},{"benchmarkId":"vals-ai:web_search:overall","benchmarkName":"Vals Web Search Index","category":null,"domain":"Other","eligible":false,"ranked":true,"score":35.5375,"unit":"percent","family":"vals-ai:web_search:overall","index":null,"aboveFit":false,"run":"Native","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/web_search","measuredAt":"2026-11-04","stderr":1.9338961321642898,"costPerTest":0.991213,"latency":137.042},{"benchmarkId":"epoch-ai:epoch-chess-puzzles:overall","benchmarkName":"Chess Puzzles","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":35,"unit":"percent","family":"epoch-ai:epoch-chess-puzzles:overall","index":60.550207555392305,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":4.793724854411022,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-mirrorcode:overall","benchmarkName":"MirrorCode","category":"coding","domain":null,"eligible":true,"ranked":true,"score":32.22222222222222,"unit":"percent","family":"epoch-ai:epoch-mirrorcode:overall","index":63.71671629420966,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":10.723394129558436,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:proof_bench:overall","benchmarkName":"ProofBench v1.1","category":"math","domain":null,"eligible":true,"ranked":true,"score":31,"unit":"percent","family":"vals-ai:proof_bench:overall","index":57.859206321415705,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/proof_bench","measuredAt":"2026-11-09","stderr":4.648,"costPerTest":0.573991,"latency":668.86},{"benchmarkId":"livebench:livebench-long-horizon:overall","benchmarkName":"LiveBench Long Agentic","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":27.9,"unit":"percent","family":"livebench-2-agentic","index":38.99360913065095,"aboveFit":false,"run":"xhigh effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Long%20Agentic","measuredAt":"2026-10-14","stderr":null,"costPerTest":4.304822727272728,"latency":null},{"benchmarkId":"vals-ai:tax_agent_bench:overall","benchmarkName":"Tax Agent Bench","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":24.855,"unit":"percent","family":"vals-ai:tax_agent_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench","measuredAt":"2026-11-09","stderr":3.118,"costPerTest":0.473604,"latency":283.566},{"benchmarkId":"epoch-ai:epoch-frontiermath-tier-4-v2-private:overall","benchmarkName":"FrontierMath-Tier-4-v2-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":24.390243902439025,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tier-4-v2-private:overall","index":58.070012760481895,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":6.789956540036613,"costPerTest":null,"latency":null},{"benchmarkId":"postTrainBench11","benchmarkName":"PostTrainBench v1.1","category":"coding","score":23.45,"unit":"percent","family":"postTrainBench11","index":60.64851877496818,"aboveFit":false,"run":null,"source":"PostTrainBench team","sourceUrl":"https://posttrainbench.com/?version=v1.1","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-furniture-assembly:overall","benchmarkName":"Furniture Assembly","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":22.5,"unit":"percent","family":"epoch-ai:epoch-furniture-assembly:overall","index":57.15250127189026,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":5.371114414765075,"costPerTest":null,"latency":null},{"benchmarkId":"frontierBench","benchmarkName":"Terminal-Bench 3.0","category":"agentic","score":15.68,"unit":"percent","family":"frontierBench","index":64.15721615013558,"aboveFit":false,"run":null,"source":"Ryan Marten, Alex Shaw, Andy Konwinski, Harbor, and the Laude Institute","sourceUrl":"https://www.frontierbench.ai/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":15.4,"unit":"percent","family":"critpt","index":66.66171908993577,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"terminal-bench:terminal-bench-4-0-0:overall","benchmarkName":"Terminal-Bench 4.0.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":12.42,"unit":"percent","family":"terminal-bench-4","index":60.413509268077924,"aboveFit":false,"run":"high effort · Grok Build","source":"Terminal-Bench","sourceUrl":"https://www.tbench.ai/leaderboard","measuredAt":"2026-09-29","stderr":1.336734693877551,"costPerTest":6.345787878787879,"latency":3162.1},{"benchmarkId":"terminal-bench:terminal-bench-4-1-0:overall","benchmarkName":"Terminal-Bench 4.1.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":12.42,"unit":"percent","family":"terminal-bench:terminal-bench-4-1-0:overall","index":58.77368602905727,"aboveFit":false,"run":"high effort · Grok Build","source":"Terminal-Bench","sourceUrl":"https://www.tbench.ai/leaderboard","measuredAt":"2026-09-29","stderr":1.336734693877551,"costPerTest":6.345787878787879,"latency":3162.1},{"benchmarkId":"vals-ai:terminal-bench-4:overall","benchmarkName":"Terminal-Bench 4.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":8.586,"unit":"percent","family":"terminal-bench-4","index":58.59297795521803,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-4","measuredAt":"2026-11-09","stderr":1.336,"costPerTest":5.51917,"latency":2022.621},{"benchmarkId":"vals-ai:time_horizon_index:overall","benchmarkName":"Time Horizon Index: KSP","category":null,"domain":"Other","eligible":false,"ranked":true,"score":6.333,"unit":"percent","family":"vals-ai:time_horizon_index:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/time_horizon_index","measuredAt":"2026-10-07","stderr":0,"costPerTest":344.035038,"latency":0},{"benchmarkId":"lmarena:agent-bash-recovery:overall","benchmarkName":"Agent Arena command recovery","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":5.136066786226423,"unit":"index","family":"agent-arena","index":70.75345339692437,"aboveFit":false,"run":null,"source":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset","measuredAt":"2026-10-08","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:srebench:overall","benchmarkName":"SRE Bench","category":null,"domain":"Other","eligible":false,"ranked":true,"score":0.763,"unit":"percent","family":"vals-ai:srebench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/srebench","measuredAt":"2026-11-05","stderr":0.539,"costPerTest":13.415223,"latency":3939.029},{"benchmarkId":"arc-prize:v3-semi-private:overall","benchmarkName":"ARC-AGI-3 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":0.3,"unit":"percent","family":"arc-prize:v3-semi-private:overall","index":65.43304620237365,"aboveFit":false,"run":"high effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:programbench:overall","benchmarkName":"ProgramBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":0,"unit":"percent","family":"vals-ai:programbench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/programbench","measuredAt":"2026-11-09","stderr":0,"costPerTest":4.124197,"latency":1139.043}],"price":{"inputPerMillion":2,"outputPerMillion":6,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":53,"timeToFirstToken":1.762,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":{"elo":1448.0478043849753,"eloUpper":1452.4076181446253,"eloLower":1443.6879906253253,"votes":39155,"rank":59,"rankUpper":null,"rankLower":null,"arenaName":"Grok 4.5","organization":"xai","updatedAt":"2026-11-09"},"quality":{"coveragePercent":95,"benchmarkCount":47,"categoryCount":7,"confidence":"high","state":"available"},"hosts":[{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":2,"outputPerMillion":6,"cacheReadPerMillion":0.3,"contextTokens":500000,"maxOutputTokens":450000,"deprecated":false},{"host":"xai","name":"xAI","kind":"maker","url":"https://docs.x.ai/docs/models","inputPerMillion":2,"outputPerMillion":6,"cacheReadPerMillion":0.3,"contextTokens":500000,"maxOutputTokens":500000,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:livebench","name":"LiveBench","url":"https://livebench.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:arc-prize-foundation","name":"ARC Prize Foundation","url":"https://arcprize.org","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:sierra-research","name":"Sierra Research","url":"https://github.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:terminal-bench","name":"Terminal-Bench","url":"https://www.tbench.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:lmarena","name":"LMArena","url":"https://huggingface.co","license":"CC BY 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gemini-3-6-flash","providerId":"google","slug":"gemini-3-6-flash","name":"Gemini 3.6 Flash","provider":"Google","description":"Gemini 3.6 Flash is a reasoning model from Google. 38 benchmarks count toward its index, in 8 categories.","logoUrl":null,"releaseDate":"2026-07-21","contextWindowTokens":1048576,"modalities":["text","image","video","file","audio"],"openWeights":false,"details":{"apiId":"google/gemini-3.6-flash","maxOutputTokens":65536,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.075,"weightsUrl":null},"reasoning":true,"overallScore":63.73235301593225,"overallScoreRange":{"low":61.10923263624782,"high":66.35547339561668},"badges":[],"capabilityScores":{"agentic":59.71674513654591,"coding":62.00010409657334,"reasoning":62.09493328753453,"multimodal":68.48058054546051,"knowledge":67.897713212022,"multilingual":70.13068778387978,"instruction":79.0867738937466,"math":60.73348471075296},"capabilityCounts":{"math":4,"knowledge":6,"reasoning":8,"multimodal":2,"coding":9,"agentic":7,"multilingual":1,"instruction":1},"benchmarks":[{"benchmarkId":"arc-prize:v1-public-eval:overall","benchmarkName":"ARC-AGI-1 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":96,"unit":"percent","family":"arc-prize:v1-public-eval:overall","index":null,"aboveFit":false,"run":"high effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.2538783975,"latency":null},{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":94.16666666666667,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":63.70469684976368,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":3.092635435901545,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":94.12878787878788,"unit":"percent","family":"gpqa-diamond","index":63.54145290359445,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.4028967942064967,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:gpqa:overall","benchmarkName":"GPQA Diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":93.434,"unit":"percent","family":"gpqa-diamond","index":62.904212446304726,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","measuredAt":"2026-09-01","stderr":1.329,"costPerTest":0.03189,"latency":18.064},{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":92.8,"unit":"percent","family":"gpqa-diamond","index":62.322724970405574,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"arc-prize:v1-semi-private:overall","benchmarkName":"ARC-AGI-1 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":91.16666666666666,"unit":"percent","family":"arc-prize:v1-semi-private:overall","index":68.89996691468993,"aboveFit":false,"run":"high effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.33563630999999994,"latency":null},{"benchmarkId":"vals-ai:mmlu_pro:overall","benchmarkName":"MMLU Pro","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":89.276,"unit":"percent","family":"mmlu-pro","index":56.957802364438706,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","measuredAt":"2026-09-01","stderr":0.305,"costPerTest":0.013327,"latency":7.298},{"benchmarkId":"vals-ai:mmmu:overall","benchmarkName":"MMMU Pro","category":"multimodal","domain":null,"eligible":true,"ranked":true,"score":88.382,"unit":"percent","family":"mmmu-pro","index":70.17423384359773,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmmu","measuredAt":"2026-09-01","stderr":0.77,"costPerTest":0.0186,"latency":11.313},{"benchmarkId":"vals-ai:lcb:overall","benchmarkName":"LiveCodeBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":88.08,"unit":"percent","family":"livecodebench","index":62.427373954060045,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","measuredAt":"2026-09-01","stderr":0.939,"costPerTest":0.056383,"latency":31.883},{"benchmarkId":"vals-ai:legal_bench:overall","benchmarkName":"LegalBench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":86.705,"unit":"percent","family":"vals-ai:legal_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","measuredAt":"2026-10-13","stderr":0.414,"costPerTest":0.004766,"latency":2.962},{"benchmarkId":"livebench:livebench-math:overall","benchmarkName":"LiveBench Mathematics","category":"math","domain":null,"eligible":true,"ranked":true,"score":86.4025,"unit":"percent","family":"livebench:livebench-math:overall","index":60.1956555087895,"aboveFit":false,"run":"high effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Mathematics","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.09172401055408973,"latency":null},{"benchmarkId":"livebench:livebench-reasoning:overall","benchmarkName":"LiveBench Reasoning","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":85.149,"unit":"percent","family":"livebench","index":71.92463067330254,"aboveFit":false,"run":"high effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Reasoning","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.09729405940594059,"latency":null},{"benchmarkId":"livebench:livebench-language:overall","benchmarkName":"LiveBench Language","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":83.89766666666667,"unit":"percent","family":"livebench:livebench-language:overall","index":69.4310301491354,"aboveFit":false,"run":"high effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Language","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.05135666666666667,"latency":null},{"benchmarkId":"aaMmmuPro","benchmarkName":"Artificial Analysis MMMU-Pro","category":"multimodal","score":83.2,"unit":"percent","family":"mmmu-pro","index":63.43031548575022,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmmu-pro","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"osWorldVerified","benchmarkName":"OSWorld-Verified","category":"agentic","score":83,"unit":"percent","family":"osWorldVerified","index":67.48334666242155,"aboveFit":false,"run":null,"source":"Tianbao Xie, Danyang Zhang, Jixuan Chen, Xiaochuan Li, Siheng Zhao, Ruisheng Cao, Toh Jing Hua, Zhoujun Cheng, Dongchan Shin, Fangyu Lei, Yitao Liu, Yiheng Xu, Shuyan Zhou, Silvio Savarese, Caiming Xiong, Victor Zhong, Tao Yu","sourceUrl":"https://os-world.github.io/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":80,"unit":"percent","family":"lcr","index":62.18449150602352,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:medscribe:overall","benchmarkName":"MedScribe","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":79.661,"unit":"percent","family":"vals-ai:medscribe:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medscribe","measuredAt":"2026-10-29","stderr":1.861,"costPerTest":0.073178,"latency":33.552},{"benchmarkId":"vals-ai:swebench:overall","benchmarkName":"SWE-bench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":79.6,"unit":"percent","family":"swe-bench-verified","index":59.93751072105352,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/swebench","measuredAt":"2026-09-01","stderr":1.804,"costPerTest":1.194601,"latency":788.09},{"benchmarkId":"livebench:livebench-coding:overall","benchmarkName":"LiveBench Coding","category":"coding","domain":null,"eligible":true,"ranked":true,"score":77.863,"unit":"percent","family":"livebench:livebench-coding:overall","index":64.33143489820056,"aboveFit":false,"run":"high effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Coding","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.07573846153846153,"latency":null},{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":75.91874999999999,"unit":"percent","family":"euroeval","index":75.22451080180325,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":0.44747730389372825,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-french:overall","benchmarkName":"EuroEval French","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":75.63,"unit":"percent","family":"euroeval","index":74.83274349128541,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/french","measuredAt":null,"stderr":0.3498861421948574,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-instruction-following:overall","benchmarkName":"LiveBench Instruction Following","category":"instruction","domain":null,"eligible":true,"ranked":true,"score":75.36675,"unit":"percent","family":"livebench:livebench-instruction-following:overall","index":79.0867738937466,"aboveFit":false,"run":"high effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=IF","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.11487399999999999,"latency":null},{"benchmarkId":"euroeval:euroeval-italian:overall","benchmarkName":"EuroEval Italian","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":75.11125000000001,"unit":"percent","family":"euroeval","index":74.12891910225989,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/italian","measuredAt":null,"stderr":0.3193107812147908,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:tax_eval_v2:overall","benchmarkName":"TaxEval v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":74.857,"unit":"percent","family":"vals-ai:tax_eval_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","measuredAt":"2026-09-01","stderr":0.847,"costPerTest":0.020385,"latency":12.335},{"benchmarkId":"vals-ai:terminal-bench-2-1:overall","benchmarkName":"Terminal-Bench 2.1","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":73.783,"unit":"percent","family":"terminal-bench-2-1","index":64.29350456426472,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2-1","measuredAt":"2026-10-01","stderr":1.633,"costPerTest":1.174404,"latency":552.153},{"benchmarkId":"euroeval:euroeval-portuguese:overall","benchmarkName":"EuroEval Portuguese","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":72.56875000000001,"unit":"percent","family":"euroeval","index":70.67933161484306,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/portuguese","measuredAt":null,"stderr":0.3435385713424331,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-dutch:overall","benchmarkName":"EuroEval Dutch","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":71.76,"unit":"percent","family":"euroeval","index":69.58204395291648,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/dutch","measuredAt":null,"stderr":0.3347488463362597,"costPerTest":null,"latency":null},{"benchmarkId":"parseBenchLlamaIndex","benchmarkName":"ParseBench: document parsing for AI agents (LlamaIndex leaderboard)","category":"multimodal","score":69.12,"unit":"percent","family":"parseBenchLlamaIndex","index":70.15888642624707,"aboveFit":false,"run":null,"source":"Boyang Zhang, Sebastián G. Acosta, Preston Carlson, Sacha Bron, Pierre-Loïc Doulcet, Daniel B. Ospina and Simon Suo (LlamaIndex)","sourceUrl":"https://arxiv.org/abs/2604.08538","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:mortgage_tax:overall","benchmarkName":"MortgageTax","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":67.608,"unit":"percent","family":"vals-ai:mortgage_tax:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mortgage_tax","measuredAt":"2026-09-01","stderr":0.915,"costPerTest":0.019658,"latency":9.239},{"benchmarkId":"euroeval:euroeval-spanish:overall","benchmarkName":"EuroEval Spanish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":66.80875,"unit":"percent","family":"euroeval","index":62.86433695308461,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/spanish","measuredAt":null,"stderr":0.43396248397758996,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-simpleqa-verified:overall","benchmarkName":"SimpleQA Verified","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":66.2,"unit":"percent","family":"epoch-ai:epoch-simpleqa-verified:overall","index":79.06949633212267,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.4965960710224384,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":65.95875,"unit":"percent","family":"euroeval","index":61.711082532512606,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.5190782094251308,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":65.86857142857143,"unit":"percent","family":"euroeval","index":61.588730960582176,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.42817600805924544,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:emb:overall","benchmarkName":"EMB","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":65.411,"unit":"percent","family":"vals-ai:emb:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/emb","measuredAt":"2026-10-20","stderr":2.646,"costPerTest":3.893069,"latency":726.48},{"benchmarkId":"arc-prize:v2-public-eval:overall","benchmarkName":"ARC-AGI-2 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":65.27777777777779,"unit":"percent","family":"arc-prize:v2-public-eval:overall","index":null,"aboveFit":false,"run":"high effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.65810195,"latency":null},{"benchmarkId":"vals-ai:vibe-code:overall","benchmarkName":"Vibe Code Bench v1.1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":64.005,"unit":"percent","family":"vals-ai:vibe-code:overall","index":65.34353218493048,"aboveFit":false,"run":"OpenHands","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","measuredAt":"2026-11-09","stderr":4.29,"costPerTest":3.041993,"latency":1523.201},{"benchmarkId":"vals-ai:corp_fin_v2:overall","benchmarkName":"CorpFin v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":63.326,"unit":"percent","family":"vals-ai:corp_fin_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","measuredAt":"2026-08-12","stderr":0.949,"costPerTest":0.136842,"latency":6.329},{"benchmarkId":"livebench:livebench-data-analysis:overall","benchmarkName":"LiveBench Data Analysis","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":62.99933333333333,"unit":"percent","family":"livebench","index":43.23119946998836,"aboveFit":false,"run":"high effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Data%20Analysis","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.19121266666666667,"latency":null},{"benchmarkId":"arc-prize:v2-semi-private:overall","benchmarkName":"ARC-AGI-2 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":60.416666666666664,"unit":"percent","family":"arc-prize:v2-semi-private:overall","index":66.92790358621863,"aboveFit":false,"run":"high effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.6060796625000002,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tiers-1-3-v2-private:overall","benchmarkName":"FrontierMath-Tiers-1-3-v2-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":58.94736842105262,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tiers-1-3-v2-private:overall","index":61.921744550299906,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.9190634943914042,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:biomysterybench:overall","benchmarkName":"BioMysteryBench","category":null,"domain":"Science","eligible":false,"ranked":true,"score":58.519,"unit":"percent","family":"vals-ai:biomysterybench:overall","index":null,"aboveFit":false,"run":"Terminus 2","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/biomysterybench","measuredAt":"2026-11-09","stderr":1.614,"costPerTest":1.252177,"latency":761.958},{"benchmarkId":"vals-ai:public-benefits-bench:overall","benchmarkName":"Public Benefits Bench v1.1","category":null,"domain":"Public benefits","eligible":false,"ranked":true,"score":56.563,"unit":"percent","family":"vals-ai:public-benefits-bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/public-benefits-bench","measuredAt":"2026-10-12","stderr":1.289,"costPerTest":0.385496,"latency":121.522},{"benchmarkId":"vals-ai:fabv2:overall","benchmarkName":"Finance Agent (v2)","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":56.296,"unit":"percent","family":"vals-ai:fabv2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/fabv2","measuredAt":"2026-10-29","stderr":0.183,"costPerTest":1.402901,"latency":224.899},{"benchmarkId":"cursorBench32","benchmarkName":"cursorBench32","category":"coding","score":53.5,"unit":"percent","family":"cursorBench32","index":59.794433915653975,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaSciCode","benchmarkName":"Artificial Analysis SciCode","category":"coding","score":53.4,"unit":"percent","family":"scicode-family","index":65.45690566105802,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/scicode","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:medcode:overall","benchmarkName":"MedCode","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":53.153,"unit":"percent","family":"vals-ai:medcode:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medcode","measuredAt":"2026-10-29","stderr":2.157,"costPerTest":0.044216,"latency":16.685},{"benchmarkId":"omniscienceAccuracy","benchmarkName":"Artificial Analysis Omniscience Accuracy","category":"knowledge","score":50,"unit":"percent","family":"omniscienceAccuracy","index":73.7027496696014,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"deepSwe","benchmarkName":"DeepSWE","category":"agentic","score":49,"unit":"percent","family":"deepSwe","index":56.69835274856993,"aboveFit":false,"run":null,"source":"Datacurve AI","sourceUrl":"https://deepswe.datacurve.ai/blog","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:cyber:overall","benchmarkName":"CyberBench v1.1","category":null,"domain":"Other","eligible":false,"ranked":true,"score":45.357,"unit":"percent","family":"vals-ai:cyber:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/cyber","measuredAt":"2026-11-05","stderr":3.745,"costPerTest":1.306182,"latency":660.036},{"benchmarkId":"livebench:livebench-agentic-coding:overall","benchmarkName":"LiveBench Agentic Coding","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":43.434333333333335,"unit":"percent","family":"livebench:livebench-agentic-coding:overall","index":53.95577856960306,"aboveFit":false,"run":"high effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Agentic%20Coding","measuredAt":"2026-06-25","stderr":null,"costPerTest":1.3513083333333333,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":40.8,"unit":"percent","family":"hle","index":65.32098831052909,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-chess-puzzles:overall","benchmarkName":"Chess Puzzles","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":40,"unit":"percent","family":"epoch-ai:epoch-chess-puzzles:overall","index":64.34528353114278,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":4.92365963917331,"costPerTest":null,"latency":null},{"benchmarkId":"gdpvalAaNormalized","benchmarkName":"GDPval-AA normalized","category":"agentic","score":39.2,"unit":"percent","family":"gdpvalAaNormalized","index":62.0608639755126,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:ioi:overall","benchmarkName":"IOI","category":"coding","domain":null,"eligible":true,"ranked":true,"score":35.056,"unit":"percent","family":"vals-ai:ioi:overall","index":59.27744185568335,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/ioi","measuredAt":"2026-10-29","stderr":5.695,"costPerTest":8.66611,"latency":2903.307},{"benchmarkId":"vals-ai:code-migration:overall","benchmarkName":"Code Migration","category":"coding","domain":null,"eligible":true,"ranked":true,"score":30.928,"unit":"percent","family":"vals-ai:code-migration:overall","index":61.61933106263203,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/code-migration","measuredAt":"2026-10-29","stderr":4.068,"costPerTest":8.930416,"latency":5404.829},{"benchmarkId":"epoch-ai:epoch-mystery-game-puzzles:overall","benchmarkName":"Mystery Game Puzzles","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":30,"unit":"percent","family":"epoch-ai:epoch-mystery-game-puzzles:overall","index":61.91344510322534,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":4.605661864718382,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:legal_research:overall","benchmarkName":"Legal Research Bench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":25,"unit":"percent","family":"vals-ai:legal_research:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_research","measuredAt":"2026-11-09","stderr":3.01,"costPerTest":0.466741,"latency":168.482},{"benchmarkId":"epoch-ai:epoch-furniture-assembly:overall","benchmarkName":"Furniture Assembly","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":23.333333333333332,"unit":"percent","family":"epoch-ai:epoch-furniture-assembly:overall","index":57.54098141754946,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":5.3765901164774155,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-mirrorcode:overall","benchmarkName":"MirrorCode","category":"coding","domain":null,"eligible":true,"ranked":true,"score":23.333333333333332,"unit":"percent","family":"epoch-ai:epoch-mirrorcode:overall","index":59.81297261588808,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":9.315175080076958,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tier-4-v2-private:overall","benchmarkName":"FrontierMath-Tier-4-v2-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":21.951219512195124,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tier-4-v2-private:overall","index":57.11184193415877,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":6.5445892024384085,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:tax_agent_bench:overall","benchmarkName":"Tax Agent Bench","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":18.038,"unit":"percent","family":"vals-ai:tax_agent_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench","measuredAt":"2026-11-09","stderr":2.766,"costPerTest":0.356293,"latency":105.32},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":10.6,"unit":"percent","family":"critpt","index":57.369479169781116,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:terminal-bench-science:overall","benchmarkName":"Terminal-Bench-Science","category":null,"domain":"Science","eligible":false,"ranked":true,"score":4.286,"unit":"percent","family":"vals-ai:terminal-bench-science:overall","index":null,"aboveFit":false,"run":"Mini-SWE-agent","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-science","measuredAt":"2026-11-09","stderr":2.438,"costPerTest":4.390771,"latency":6744.655},{"benchmarkId":"mazeBench","benchmarkName":"MazeBench: visual spatial reasoning in a 3D open world","category":"agentic","score":0,"unit":"percent","family":"mazeBench","index":61.97866734834783,"aboveFit":false,"run":null,"source":"Jonathan Pappas and David Pappas","sourceUrl":"https://mazebench.com/blog?post=maze-bench-results","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:programbench:overall","benchmarkName":"ProgramBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":0,"unit":"percent","family":"vals-ai:programbench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/programbench","measuredAt":"2026-11-09","stderr":0,"costPerTest":5.923527,"latency":2852.749},{"benchmarkId":"lmarena:agent-bash-recovery:overall","benchmarkName":"Agent Arena command recovery","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":-18.225355243790563,"unit":"index","family":"agent-arena","index":51.54670208710171,"aboveFit":false,"run":"high effort","source":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset","measuredAt":"2026-10-08","stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":0.75,"outputPerMillion":3.75,"batch":{"inputPerMillion":0.375,"outputPerMillion":1.875},"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":121,"timeToFirstToken":1.611,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":{"elo":1479.2630091424253,"eloUpper":1483.6828507030714,"eloLower":1474.8431675817797,"votes":35405,"rank":22,"rankUpper":null,"rankLower":null,"arenaName":"Gemini 3.6 Flash-high","organization":"google","updatedAt":"2026-11-09"},"quality":{"coveragePercent":100,"benchmarkCount":38,"categoryCount":8,"confidence":"high","state":"available"},"hosts":[{"host":"google-ai-studio","name":"Google AI Studio","kind":"maker","url":"https://ai.google.dev/gemini-api/docs/models","inputPerMillion":0.75,"outputPerMillion":3.75,"cacheReadPerMillion":0.075,"contextTokens":1048576,"maxOutputTokens":65536,"deprecated":false},{"host":"google-vertex","name":"Google Vertex AI","kind":"maker","url":"https://cloud.google.com/vertex-ai/generative-ai/docs/models","inputPerMillion":0.75,"outputPerMillion":3.75,"cacheReadPerMillion":0.075,"contextTokens":1048576,"maxOutputTokens":65536,"deprecated":true},{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":0.75,"outputPerMillion":3.75,"cacheReadPerMillion":0.075,"contextTokens":1048576,"maxOutputTokens":65536,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:arc-prize-foundation","name":"ARC Prize Foundation","url":"https://arcprize.org","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:livebench","name":"LiveBench","url":"https://livebench.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:lmarena","name":"LMArena","url":"https://huggingface.co","license":"CC BY 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"dots3-note-preview","providerId":"dots-studio","slug":"dots3-note-preview","name":"dots3-note Preview","provider":"Dots Studio","description":"dots3-note Preview is a reasoning model from Dots Studio in the dots3-note family. 23 benchmarks count toward its index, in 7 categories.","logoUrl":null,"releaseDate":"2026-08-14","contextWindowTokens":512000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":true,"overallScore":63.39430292254704,"overallScoreRange":{"low":59.52262484724414,"high":67.26598099784994},"badges":[],"capabilityScores":{"agentic":63.33571828125192,"coding":60.31941024962586,"reasoning":74.4526111855501,"multimodal":59.15220226077046,"knowledge":77.15984270188066,"multilingual":null,"instruction":58.22363339803903,"math":62.29222733915347},"capabilityCounts":{"instruction":2,"agentic":8,"coding":4,"math":1,"multimodal":6,"reasoning":1,"knowledge":1},"benchmarks":[{"benchmarkId":"ifeval","benchmarkName":"Instruction-Following Eval","category":"instruction","score":93.9,"unit":"percent","family":"ifeval","index":52.28898397172111,"aboveFit":false,"run":null,"source":"Jeffrey Zhou, Tianjian Lu, Swaroop Mishra, Siddhartha Brahma, Sujoy Basu, Yi Luan, Denny Zhou, Le Hou","sourceUrl":"https://arxiv.org/abs/2311.07911","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"deepSearchQa","benchmarkName":"DeepSearchQA","category":"agentic","score":92.1,"unit":"percent","family":"deepSearchQa","index":67.41156627515778,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"liveCodeBenchV6","benchmarkName":"LiveCodeBench v6","category":"coding","score":91.5,"unit":"percent","family":"liveCodeBenchV6","index":59.543800065196514,"aboveFit":false,"run":null,"source":"LiveCodeBench maintainers","sourceUrl":"https://github.com/LiveCodeBench/LiveCodeBench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"imoAnswerBench","benchmarkName":"IMOAnswerBench","category":"math","score":90.9,"unit":"percent","family":"imoAnswerBench","index":62.29222733915347,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mathVision","benchmarkName":"MathVision","category":"multimodal","score":87.7,"unit":"percent","family":"mathVision","index":58.69838054135451,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"videoMmmu","benchmarkName":"VideoMMMU","category":"multimodal","score":86.8,"unit":"percent","family":"videoMmmu","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"browseComp","benchmarkName":"BrowseComp","category":"agentic","score":83.3,"unit":"percent","family":"browseComp","index":65.71764389034095,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"charxivNoTools","benchmarkName":"CharXiv Reasoning without tools","category":"multimodal","score":83.1,"unit":"percent","family":"charxivNoTools","index":63.38331888154075,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"ifBench","benchmarkName":"Instruction Following Benchmark","category":"instruction","score":80.4,"unit":"percent","family":"ifbench-family","index":64.15828282435696,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmvu","benchmarkName":"Multimodal Multi-disciplinary Video Understanding","category":"multimodal","score":79.9,"unit":"percent","family":"mmvu","index":null,"aboveFit":false,"run":null,"source":"MMVU benchmark maintainers","sourceUrl":"https://www.kimi.com/blog/kimi-k2-5.html","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmmuPro","benchmarkName":"Massive Multi-discipline Multimodal Understanding Pro","category":"multimodal","score":79.1,"unit":"percent","family":"mmmu-pro","index":58.09452519876163,"aboveFit":false,"run":null,"source":"MMMU-Pro authors","sourceUrl":"https://arxiv.org/abs/2409.02813","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"wideResearch","benchmarkName":"WideResearch","category":"agentic","score":78.9,"unit":"percent","family":"wideResearch","index":61.32685099305167,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"sweVerified","benchmarkName":"Software Engineering Benchmark Verified","category":"coding","score":78.4,"unit":"percent","family":"swe-bench-verified","index":59.052441627713094,"aboveFit":false,"run":null,"source":"Carlos E. Jimenez, John Yang, Alexander Wettig, Shunyu Yao, Kexin Pei, Ofir Press, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2310.06770","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"arc-prize:v2-semi-private:overall","benchmarkName":"ARC-AGI-2 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":76.80555555555556,"unit":"percent","family":"arc-prize:v2-semi-private:overall","index":74.4526111855501,"aboveFit":false,"run":"max effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.07665574466666672,"latency":null},{"benchmarkId":"terminalBench21","benchmarkName":"Terminal-Bench 2.1","category":"agentic","score":75.1,"unit":"percent","family":"terminal-bench-2-1","index":65.06375907927563,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"clawEval","benchmarkName":"Claw-Eval","category":"agentic","score":73.4,"unit":"percent","family":"clawEval","index":71.26954309217268,"aboveFit":false,"run":null,"source":"Bowen Ye, Rang Li, Qibin Yang, Yuanxin Liu, Linli Yao, Hanglong Lv, Zhihui Xie, Chenxin An, Lei Li, Lingpeng Kong, Qi Liu, Zhifang Sui, Tong Yang","sourceUrl":"https://arxiv.org/abs/2604.06132","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"simpleVqa","benchmarkName":"SimpleVQA","category":"multimodal","score":72.5,"unit":"percent","family":"simpleVqa","index":62.71970957571321,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"swePro","benchmarkName":"SWE-bench Pro","category":"coding","score":61,"unit":"percent","family":"swePro","index":60.787592526380486,"aboveFit":false,"run":null,"source":"Xiang Deng, Jeff Da, Edwin Pan, Yannis Yiming He, Charles Ide, Kanak Garg, Niklas Lauffer, Andrew Park, Nitin Pasari, Chetan Rane, Karmini Sampath, Maya Krishnan, Srivatsa Kundurthy, Sean Hendryx, Zifan Wang, Chen Bo Calvin Zhang, Noah Jacobson, Bing Liu, Brad Kenstler","sourceUrl":"https://arxiv.org/abs/2509.16941","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gdpPdf","benchmarkName":"GDP.pdf mean criteria pass rate without tools","category":"multimodal","score":60.7,"unit":"percent","family":"gdpPdf","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"toolathlonVerified","benchmarkName":"Toolathlon-Verified","category":"agentic","score":55.6,"unit":"percent","family":"toolathlonVerified","index":54.433626988136716,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"perceptionBench","benchmarkName":"PerceptionBench (Internal)","category":"multimodal","score":53.4,"unit":"percent","family":"perceptionBench","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hleWithTools","benchmarkName":"Humanity's Last Exam with tools","category":"agentic","score":52.6,"unit":"percent","family":"hleWithTools","index":62.256902299974755,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hle","benchmarkName":"Humanity's Last Exam","category":"knowledge","score":52.6,"unit":"percent","family":"hle","index":77.15984270188066,"aboveFit":false,"run":null,"source":"Center for AI Safety, Scale AI, and thousands of expert contributors","sourceUrl":"https://lastexam.ai/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"babyVision","benchmarkName":"BabyVision","category":"multimodal","score":50,"unit":"percent","family":"babyVision","index":51.43418665710826,"aboveFit":false,"run":null,"source":"Meta AI","sourceUrl":"https://ai.meta.com/static-resource/muse-spark-1-1-evaluation-report","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"nl2Repo","benchmarkName":"NL2Repo","category":"coding","score":49.8,"unit":"percent","family":"nl2Repo","index":61.893806779213364,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"apexAgents","benchmarkName":"APEX-Agents","category":"agentic","score":30.8,"unit":"percent","family":"apexAgents","index":59.20585363190515,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"zeroBench","benchmarkName":"ZeroBench","category":"multimodal","score":19,"unit":"percent","family":"zeroBench","index":60.58309271014437,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":0,"outputPerMillion":0,"batch":null,"freeTier":true,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":95,"benchmarkCount":23,"categoryCount":7,"confidence":"medium","state":"available"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:arc-prize-foundation","name":"ARC Prize Foundation","url":"https://arcprize.org","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"qwen38max0902","providerId":"alibaba","slug":"alibaba-qwen3-8-max-0902","name":"Qwen3.8 Max (0902)","provider":"Alibaba","description":"Qwen3.8 Max (0902) is a model from Alibaba. 10 benchmarks count toward its index, in 5 categories.","logoUrl":null,"releaseDate":"2026-09-03T21:08:24.000Z","contextWindowTokens":1000000,"modalities":["text","image","video"],"openWeights":null,"details":{"apiId":"qwen/qwen3.8-max-0902","maxOutputTokens":131072,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.25,"weightsUrl":null},"reasoning":null,"overallScore":63.37248272087589,"overallScoreRange":{"low":56.25907974438362,"high":70.48588569736816},"badges":[],"capabilityScores":{"agentic":66.43754623979237,"coding":70.54826773127242,"reasoning":59.027649390302585,"multimodal":null,"knowledge":62.593206815333616,"multilingual":null,"instruction":null,"math":64.81064915820194},"capabilityCounts":{"math":3,"knowledge":2,"coding":1,"reasoning":2,"agentic":2},"benchmarks":[{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":100,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":67.09849544987334,"aboveFit":false,"run":"xhigh effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":92.2979797979798,"unit":"percent","family":"gpqa-diamond","index":61.86228575885495,"aboveFit":false,"run":"xhigh effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.7249976996605707,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tiers-1-3-v2-private:overall","benchmarkName":"FrontierMath-Tiers-1-3-v2-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":65.6140350877193,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tiers-1-3-v2-private:overall","index":65.43075595895804,"aboveFit":false,"run":"xhigh effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.818576398907258,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-mirrorcode:overall","benchmarkName":"MirrorCode","category":"coding","domain":null,"eligible":true,"ranked":true,"score":47.77777777777778,"unit":"percent","family":"epoch-ai:epoch-mirrorcode:overall","index":70.54826773127242,"aboveFit":false,"run":"xhigh effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":10.26973086960115,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-simpleqa-verified:overall","benchmarkName":"SimpleQA Verified","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":47.3,"unit":"percent","family":"epoch-ai:epoch-simpleqa-verified:overall","index":63.32412787181229,"aboveFit":false,"run":"xhigh effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.579621855130273,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-chess-puzzles:overall","benchmarkName":"Chess Puzzles","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":37,"unit":"percent","family":"epoch-ai:epoch-chess-puzzles:overall","index":62.068237945692495,"aboveFit":false,"run":"xhigh effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":4.8523658709390975,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tier-4-v2-private:overall","benchmarkName":"FrontierMath-Tier-4-v2-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":34.146341463414636,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tier-4-v2-private:overall","index":61.90269606577442,"aboveFit":false,"run":"xhigh effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":7.497768853141171,"costPerTest":null,"latency":null},{"benchmarkId":"terminal-bench:terminal-bench-4-0-0:overall","benchmarkName":"Terminal-Bench 4.0.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":26.97,"unit":"percent","family":"terminal-bench-4","index":67.32241135522227,"aboveFit":false,"run":"max effort · Claude Code","source":"Terminal-Bench","sourceUrl":"https://www.tbench.ai/leaderboard","measuredAt":"2026-09-29","stderr":1.9285714285714286,"costPerTest":14.04748484848485,"latency":9953},{"benchmarkId":"terminal-bench:terminal-bench-4-1-0:overall","benchmarkName":"Terminal-Bench 4.1.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":26.97,"unit":"percent","family":"terminal-bench:terminal-bench-4-1-0:overall","index":65.55268112436246,"aboveFit":false,"run":"max effort · Claude Code","source":"Terminal-Bench","sourceUrl":"https://www.tbench.ai/leaderboard","measuredAt":"2026-09-29","stderr":1.9744897959183674,"costPerTest":14.04748484848485,"latency":9953},{"benchmarkId":"epoch-ai:epoch-furniture-assembly:overall","benchmarkName":"Furniture Assembly","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":20,"unit":"percent","family":"epoch-ai:epoch-furniture-assembly:overall","index":55.98706083491267,"aboveFit":false,"run":"xhigh effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":5.207556439232955,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":2,"outputPerMillion":6,"batch":{"inputPerMillion":1.15,"outputPerMillion":3.45},"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":33,"timeToFirstToken":3.2495,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":80,"benchmarkCount":10,"categoryCount":5,"confidence":"low","state":"available"},"hosts":[{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":2,"outputPerMillion":6,"cacheReadPerMillion":0.25,"contextTokens":1000000,"maxOutputTokens":131072,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:terminal-bench","name":"Terminal-Bench","url":"https://www.tbench.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"claude-opus-4-7-max","providerId":"anthropic","slug":"claude-opus-4-7-adaptive","name":"Claude Opus 4.7 (Adaptive)","provider":"Anthropic","description":"Claude Opus 4.7 (Adaptive) is a reasoning model from Anthropic in the Claude Opus 4.7 family. 24 benchmarks count toward its index, in 6 categories.","logoUrl":null,"releaseDate":"2026-04-16","contextWindowTokens":1000000,"modalities":["text"],"openWeights":false,"details":null,"reasoning":true,"overallScore":63.24075592009286,"overallScoreRange":{"low":58.44006849169253,"high":68.04144334849319},"badges":[],"capabilityScores":{"agentic":63.51631646583182,"coding":64.71793840381194,"reasoning":60.686151317960864,"multimodal":60.41131498840571,"knowledge":69.12143628588001,"multilingual":null,"instruction":43.785951421073776,"math":null},"capabilityCounts":{"knowledge":3,"multimodal":4,"agentic":12,"coding":2,"reasoning":2,"instruction":1},"benchmarks":[{"benchmarkId":"gpqa","benchmarkName":"Graduate-Level Google-Proof Q&A","category":"knowledge","score":94.2,"unit":"percent","family":"gpqa-diamond","index":63.60676671529328,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gpqaDiamond","benchmarkName":"GPQA Diamond","category":"knowledge","score":94.2,"unit":"percent","family":"gpqa-diamond","index":63.60676671529328,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":91.4,"unit":"percent","family":"gpqa-diamond","index":61.038683225517886,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"charxiv","benchmarkName":"CharXiv Reasoning","category":"multimodal","score":91,"unit":"percent","family":"charxiv","index":67.78755775796732,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"tau2Bench","benchmarkName":"τ²-Bench Tool-Agent-User Evaluation","category":"agentic","score":88.6,"unit":"percent","family":"tau2-bench","index":61.582017861593144,"aboveFit":false,"run":null,"source":"Victor Barres, Honghua Dong, Soham Ray, Xujie Si, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2506.07982","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"sweVerified","benchmarkName":"Software Engineering Benchmark Verified","category":"coding","score":87.6,"unit":"percent","family":"swe-bench-verified","index":65.83797134332306,"aboveFit":false,"run":null,"source":"Carlos E. Jimenez, John Yang, Alexander Wettig, Shunyu Yao, Kexin Pei, Ofir Press, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2310.06770","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaTerminalBench21","benchmarkName":"Artificial Analysis Terminal-Bench v2.1","category":"agentic","score":83.1,"unit":"percent","family":"aaTerminalBench21","index":65.67883539404644,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/terminalbench-v2-1","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"charxivNoTools","benchmarkName":"CharXiv Reasoning without tools","category":"multimodal","score":82.1,"unit":"percent","family":"charxivNoTools","index":62.06316712318629,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"browseComp","benchmarkName":"BrowseComp","category":"agentic","score":79.3,"unit":"percent","family":"browseComp","index":62.557564666343666,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaMmmuPro","benchmarkName":"Artificial Analysis MMMU-Pro","category":"multimodal","score":78.8,"unit":"percent","family":"mmmu-pro","index":57.70410151922589,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmmu-pro","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":78.7,"unit":"percent","family":"lcr","index":61.29258682276216,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"osWorldVerified","benchmarkName":"OSWorld-Verified","category":"agentic","score":78,"unit":"percent","family":"osWorldVerified","index":63.44706136403889,"aboveFit":false,"run":null,"source":"Tianbao Xie, Danyang Zhang, Jixuan Chen, Xiaochuan Li, Siheng Zhao, Ruisheng Cao, Toh Jing Hua, Zhoujun Cheng, Dongchan Shin, Fangyu Lei, Yitao Liu, Yiheng Xu, Shuyan Zhou, Silvio Savarese, Caiming Xiong, Victor Zhong, Tao Yu","sourceUrl":"https://os-world.github.io/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mcpAtlas","benchmarkName":"MCP Atlas","category":"agentic","score":77.3,"unit":"percent","family":"mcpAtlas","index":63.115762887957736,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"cyberGym","benchmarkName":"CyberGym","category":"agentic","score":73.1,"unit":"percent","family":"cyberGym","index":62.4406131542611,"aboveFit":false,"run":null,"source":"Zhun Wang, Tianneng Shi, Jingxuan He, Matthew Cai, Jialin Zhang, Dawn Song","sourceUrl":"https://www.cybergym.io/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"terminalBench2","benchmarkName":"Terminal-Bench 2.0","category":"agentic","score":69.4,"unit":"percent","family":"terminalBench2","index":63.26501600912889,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"swePro","benchmarkName":"SWE-bench Pro","category":"coding","score":64.3,"unit":"percent","family":"swePro","index":63.59790546430082,"aboveFit":false,"run":null,"source":"Xiang Deng, Jeff Da, Edwin Pan, Yannis Yiming He, Charles Ide, Kanak Garg, Niklas Lauffer, Andrew Park, Nitin Pasari, Chetan Rane, Karmini Sampath, Maya Krishnan, Srivatsa Kundurthy, Sean Hendryx, Zifan Wang, Chen Bo Calvin Zhang, Noah Jacobson, Bing Liu, Brad Kenstler","sourceUrl":"https://arxiv.org/abs/2509.16941","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mrcrv2_128_256","benchmarkName":"OpenAI MRCR v2 8-needle 128K-256K","category":"reasoning","score":59.2,"unit":"percent","family":"mrcrv2_128_256","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaIfBench","benchmarkName":"Artificial Analysis IFBench","category":"instruction","score":58.6,"unit":"percent","family":"ifbench-family","index":43.785951421073776,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hle","benchmarkName":"Humanity's Last Exam","category":"knowledge","score":54.7,"unit":"percent","family":"hle","index":79.26675746644324,"aboveFit":false,"run":null,"source":"Center for AI Safety, Scale AI, and thousands of expert contributors","sourceUrl":"https://lastexam.ai/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"omniscienceAccuracy","benchmarkName":"Artificial Analysis Omniscience Accuracy","category":"knowledge","score":48.9,"unit":"percent","family":"omniscienceAccuracy","index":72.31646808713592,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hleNoTools","benchmarkName":"Humanity's Last Exam without tools","category":"knowledge","score":46.9,"unit":"percent","family":"hle","index":71.44107405521083,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaItbench","benchmarkName":"Artificial Analysis ITBench-AA","category":"agentic","score":46.7,"unit":"percent","family":"aaItbench","index":67.30184141748363,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/itbench-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"jobBench","benchmarkName":"JobBench","category":"agentic","score":45.9,"unit":"percent","family":"jobBench","index":61.88084335257225,"aboveFit":false,"run":null,"source":"Yuetai Li, Yichen Feng, Zhangchen Xu, Zixian Ma, Kaiyuan Zheng, Fengqing Jiang, Xinghua Sun, Rulin Shao, Zichen Chen, Yue Huang, Xinyang Han, Brian Lee, Kayla Xu, Shenglai Zeng, Hang Hua, Xiangliang Zhang, Basel Alomair, Ranjay Krishna, Luke Zettlemoyer, Pang Wei Koh, Bhaskar Ramasubramanian, Luyao Niu, Xiang Yue, Radha Poovendran","sourceUrl":"https://arxiv.org/abs/2605.26329","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaAnalystAgent","benchmarkName":"Artificial Analysis AnalystAgent","category":"agentic","score":43.8,"unit":"percent","family":"aaAnalystAgent","index":65.00447886013762,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/aa-analyst-agent","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"officeQaPro","benchmarkName":"OfficeQA Pro","category":"multimodal","score":43.6,"unit":"percent","family":"officeQaPro","index":54.09043355324334,"aboveFit":false,"run":null,"source":"OfficeQA Pro authors","sourceUrl":"https://arxiv.org/abs/2603.08655","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gdpvalAaNormalized","benchmarkName":"GDPval-AA normalized","category":"agentic","score":42.6,"unit":"percent","family":"gdpvalAaNormalized","index":64.49178496526483,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":42.3,"unit":"percent","family":"hle","index":66.82592742807378,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"osWorld2","benchmarkName":"OSWorld 2.0","category":"agentic","score":18.2,"unit":"percent","family":"osWorld2","index":61.42997765715362,"aboveFit":false,"run":null,"source":"Mengqi Yuan, Zilong Zhou, Xinzhuang Xiong, Weiming Wu, Jiayang Sun, Jiamin Song, Kaiqian Cui, Bowen Wang, Haoyuan Wu, Yitong Li, Dunjie Lu, Haikong Lu, Qi Zhen, Xinyuan Wang, Jiaqi Deng, Yuhao Yang, Cheng Chen, Boyuan Zheng, Alex Su, Xiao Yu, Hao Zou, Saaket Agashe, Xing Han Lu, Manpreet Kaur, Zhengyang Qi, Vincent Sunn Chen, Frederic Sala, Dayiheng Liu, Junyang Lin, Zhou Yu, Yu Su, Siva Reddy, Xin Eric Wang, Peng Qi, Tianbao Xie, Tao Yu","sourceUrl":"https://arxiv.org/abs/2606.29537","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":12,"unit":"percent","family":"critpt","index":60.07971581315955,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":5,"outputPerMillion":25,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":85,"benchmarkCount":24,"categoryCount":6,"confidence":"medium","state":"available"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"glm-5-3-flash","providerId":"z-ai","slug":"glm-5-3-flash","name":"GLM-5.3-Flash","provider":"Z.AI","description":"GLM-5.3-Flash is a reasoning model from Z.AI in the GLM-5 family. 56 benchmarks count toward its index, in 8 categories.","logoUrl":null,"releaseDate":"2026-08-26","contextWindowTokens":1048576,"modalities":["text","image","video"],"openWeights":true,"details":{"apiId":"z-ai/glm-5.3-flash","maxOutputTokens":131072,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.03,"weightsUrl":"https://huggingface.co/zai-org/GLM-5.3-Flash"},"reasoning":true,"overallScore":63.07198175230822,"overallScoreRange":{"low":60.91117461144627,"high":65.23278889317017},"badges":["budget"],"capabilityScores":{"agentic":69.66678466758728,"coding":62.26669692446226,"reasoning":58.887197575847324,"multimodal":64.96786525242459,"knowledge":57.68002982220753,"multilingual":64.02654228623454,"instruction":33.46584764353369,"math":57.282355926007895},"capabilityCounts":{"math":5,"coding":14,"reasoning":5,"knowledge":3,"multimodal":6,"agentic":21,"multilingual":1,"instruction":1},"benchmarks":[{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":93.88888888888889,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":63.54308739261559,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.162226738085332,"costPerTest":null,"latency":null},{"benchmarkId":"arc-prize:v1-public-eval:overall","benchmarkName":"ARC-AGI-1 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":92.5,"unit":"percent","family":"arc-prize:v1-public-eval:overall","index":null,"aboveFit":false,"run":"max effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.033078684499999976,"latency":null},{"benchmarkId":"vals-ai:swebench:overall","benchmarkName":"SWE-bench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":92,"unit":"percent","family":"swe-bench-verified","index":69.08322468557131,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/swebench","measuredAt":"2026-09-01","stderr":1.214,"costPerTest":0.017996,"latency":3817.379},{"benchmarkId":"arc-prize:v1-semi-private:overall","benchmarkName":"ARC-AGI-1 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":91,"unit":"percent","family":"arc-prize:v1-semi-private:overall","index":68.82334423086087,"aboveFit":false,"run":"max effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.0430698225,"latency":null},{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":90.15151515151516,"unit":"percent","family":"gpqa-diamond","index":59.89360703743621,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.7183231147411449,"costPerTest":null,"latency":null},{"benchmarkId":"charxiv","benchmarkName":"CharXiv Reasoning","category":"multimodal","score":89.4,"unit":"percent","family":"charxiv","index":65.96596027706042,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:sage:overall","benchmarkName":"SAGE","category":null,"domain":"Education","eligible":false,"ranked":true,"score":89.255,"unit":"percent","family":"vals-ai:sage:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/sage","measuredAt":"2026-11-05","stderr":1.758,"costPerTest":0.006372,"latency":197.476},{"benchmarkId":"vals-ai:medscribe:overall","benchmarkName":"MedScribe","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":88.936,"unit":"percent","family":"vals-ai:medscribe:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medscribe","measuredAt":"2026-10-29","stderr":1.907,"costPerTest":0.002235,"latency":86.98},{"benchmarkId":"vals-ai:gpqa:overall","benchmarkName":"GPQA Diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":86.364,"unit":"percent","family":"gpqa-diamond","index":56.41980163462185,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","measuredAt":"2026-09-01","stderr":2.126,"costPerTest":0.001626,"latency":131.004},{"benchmarkId":"vals-ai:mmlu_pro:overall","benchmarkName":"MMLU Pro","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":86.059,"unit":"percent","family":"mmlu-pro","index":52.90476742944041,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","measuredAt":"2026-09-01","stderr":0.341,"costPerTest":0.001058,"latency":44.762},{"benchmarkId":"vals-ai:mmmu:overall","benchmarkName":"MMMU Pro","category":"multimodal","domain":null,"eligible":true,"ranked":true,"score":86.012,"unit":"percent","family":"mmmu-pro","index":67.08988677526531,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmmu","measuredAt":"2026-09-01","stderr":0.834,"costPerTest":0.000962,"latency":28.218},{"benchmarkId":"terminalBench21","benchmarkName":"Terminal-Bench 2.1","category":"agentic","score":84.3,"unit":"percent","family":"terminal-bench-2-1","index":70.44442843242702,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaTerminalBench21","benchmarkName":"Artificial Analysis Terminal-Bench v2.1","category":"agentic","score":84.3,"unit":"percent","family":"aaTerminalBench21","index":66.89613078917513,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/terminalbench-v2-1","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-browser:overall","benchmarkName":"LiveBench Browser Use","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":84,"unit":"percent","family":"livebench-2-agentic","index":80.02128355568858,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Browser","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.176892,"latency":null},{"benchmarkId":"vals-ai:legal_bench:overall","benchmarkName":"LegalBench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":83.928,"unit":"percent","family":"vals-ai:legal_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","measuredAt":"2026-10-13","stderr":0.427,"costPerTest":0.000278,"latency":12.521},{"benchmarkId":"livebench:livebench-math:overall","benchmarkName":"LiveBench Mathematics","category":"math","domain":null,"eligible":true,"ranked":true,"score":81.2445,"unit":"percent","family":"livebench:livebench-math:overall","index":53.00685139294953,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Mathematics","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.021652242744063323,"latency":null},{"benchmarkId":"extractBenchLlamaIndex","benchmarkName":"ExtractBench: schema-guided enterprise document extraction (LlamaIndex leaderboard)","category":"multimodal","score":80.75,"unit":"percent","family":"extractBenchLlamaIndex","index":59.157834037012805,"aboveFit":false,"run":null,"source":"Boyang Zhang, Adrian Lyjak, Eli Stewart, Zhaoqi Li and Simon Suo (LlamaIndex)","sourceUrl":"https://arxiv.org/abs/2607.29677","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-coding-2:overall","benchmarkName":"LiveBench Coding 2.0","category":"coding","domain":null,"eligible":true,"ranked":true,"score":80.6,"unit":"percent","family":"livebench:livebench-coding-2:overall","index":66.55252000461803,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Coding","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.16958620689655174,"latency":null},{"benchmarkId":"vals-ai:lcb:overall","benchmarkName":"LiveCodeBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":80.51,"unit":"percent","family":"livecodebench","index":56.146232850509314,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","measuredAt":"2026-09-01","stderr":1.051,"costPerTest":0.007053,"latency":301.901},{"benchmarkId":"mmvu","benchmarkName":"Multimodal Multi-disciplinary Video Understanding","category":"multimodal","score":80.5,"unit":"percent","family":"mmvu","index":null,"aboveFit":false,"run":null,"source":"MMVU benchmark maintainers","sourceUrl":"https://www.kimi.com/blog/kimi-k2-5.html","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-coding:overall","benchmarkName":"LiveBench Coding","category":"coding","domain":null,"eligible":true,"ranked":true,"score":78.95,"unit":"percent","family":"livebench:livebench-coding:overall","index":65.86017821660711,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Coding","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.0032820512820512823,"latency":null},{"benchmarkId":"toolathlonVerified","benchmarkName":"Toolathlon-Verified","category":"agentic","score":78.4,"unit":"percent","family":"toolathlonVerified","index":72.07418964543838,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"chartographyWithTools","benchmarkName":"Chartography with image and code tools","category":"multimodal","score":78,"unit":"percent","family":"chartographyWithTools","index":null,"aboveFit":false,"run":null,"source":"Surge AI and Anthropic","sourceUrl":"https://surgehq.ai/blog/chartography","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-reasoning:overall","benchmarkName":"LiveBench Reasoning","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":77.64425,"unit":"percent","family":"livebench","index":62.202722390721675,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Reasoning","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.013781683168316832,"latency":null},{"benchmarkId":"livebench:livebench-language:overall","benchmarkName":"LiveBench Language","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":77.30733333333333,"unit":"percent","family":"livebench:livebench-language:overall","index":61.97861770115314,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Language","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.006467333333333333,"latency":null},{"benchmarkId":"livebench:livebench-design:overall","benchmarkName":"LiveBench Design","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":77.2,"unit":"percent","family":"livebench-2-agentic","index":75.04823211022948,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Design","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.06340799999999999,"latency":null},{"benchmarkId":"livebench:livebench-data-analysis:overall","benchmarkName":"LiveBench Data Analysis","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":76.401,"unit":"percent","family":"livebench","index":60.592174075012466,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Data%20Analysis","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.024275333333333333,"latency":null},{"benchmarkId":"vals-ai:tax_eval_v2:overall","benchmarkName":"TaxEval v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":75.593,"unit":"percent","family":"vals-ai:tax_eval_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","measuredAt":"2026-09-01","stderr":0.836,"costPerTest":0.001505,"latency":53.06},{"benchmarkId":"aaCyberGymE2e","benchmarkName":"Artificial Analysis CyberGym-E2E-AA","category":"agentic","score":74,"unit":"percent","family":"aaCyberGymE2e","index":95,"aboveFit":true,"run":null,"source":"Artificial Analysis; dataset by Berkeley RDI","sourceUrl":"https://artificialanalysis.ai/evaluations/cybergym-e2e-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-italian:overall","benchmarkName":"EuroEval Italian","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":72.2275,"unit":"percent","family":"euroeval","index":70.21633388423108,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/italian","measuredAt":null,"stderr":0.33703671906781907,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-french:overall","benchmarkName":"EuroEval French","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":71.125,"unit":"percent","family":"euroeval","index":68.72049506225386,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/french","measuredAt":null,"stderr":0.3783186487605389,"costPerTest":null,"latency":null},{"benchmarkId":"parseBenchLlamaIndex","benchmarkName":"ParseBench: document parsing for AI agents (LlamaIndex leaderboard)","category":"multimodal","score":70.55,"unit":"percent","family":"parseBenchLlamaIndex","index":72.15423604168127,"aboveFit":false,"run":null,"source":"Boyang Zhang, Sebastián G. Acosta, Preston Carlson, Sacha Bron, Pierre-Loïc Doulcet, Daniel B. Ospina and Simon Suo (LlamaIndex)","sourceUrl":"https://arxiv.org/abs/2604.08538","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-research:overall","benchmarkName":"LiveBench Research","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":70.1,"unit":"percent","family":"livebench-2-agentic","index":69.85578133629423,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Research","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.18694999999999998,"latency":null},{"benchmarkId":"euroeval:euroeval-portuguese:overall","benchmarkName":"EuroEval Portuguese","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":69.89500000000001,"unit":"percent","family":"euroeval","index":67.05166807719087,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/portuguese","measuredAt":null,"stderr":0.39585548183649044,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":69.34625,"unit":"percent","family":"euroeval","index":66.30714059096864,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":0.6581804938616762,"costPerTest":null,"latency":null},{"benchmarkId":"hermesBench","benchmarkName":"Hermes Bench: 150 agent tasks graded on the files and state the agent leaves behind","category":"agentic","score":68.88,"unit":"percent","family":"hermesBench","index":null,"aboveFit":false,"run":null,"source":"Nous Research","sourceUrl":"https://portal.nousresearch.com/bench","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:cyber:overall","benchmarkName":"CyberBench v1.1","category":null,"domain":"Other","eligible":false,"ranked":true,"score":67.857,"unit":"percent","family":"vals-ai:cyber:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/cyber","measuredAt":"2026-11-05","stderr":5.566,"costPerTest":0.284024,"latency":2148.165},{"benchmarkId":"arc-prize:v2-public-eval:overall","benchmarkName":"ARC-AGI-2 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":66.80555555555556,"unit":"percent","family":"arc-prize:v2-public-eval:overall","index":null,"aboveFit":false,"run":"max effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.09910295416666666,"latency":null},{"benchmarkId":"euroeval:euroeval-dutch:overall","benchmarkName":"EuroEval Dutch","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":65.98444444444443,"unit":"percent","family":"euroeval","index":61.745943981500474,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/dutch","measuredAt":null,"stderr":0.3779182745280019,"costPerTest":null,"latency":null},{"benchmarkId":"arc-prize:v2-semi-private:overall","benchmarkName":"ARC-AGI-2 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":65.83333333333333,"unit":"percent","family":"arc-prize:v2-semi-private:overall","index":69.41488321650615,"aboveFit":false,"run":"max effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.09331568458333339,"latency":null},{"benchmarkId":"deepSwe","benchmarkName":"DeepSWE","category":"agentic","score":63.4,"unit":"percent","family":"deepSwe","index":67.16675075808288,"aboveFit":false,"run":null,"source":"Datacurve AI","sourceUrl":"https://deepswe.datacurve.ai/blog","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:terminal-bench-2-1:overall","benchmarkName":"Terminal-Bench 2.1","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":62.921,"unit":"percent","family":"terminal-bench-2-1","index":57.940805595359244,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2-1","measuredAt":"2026-10-01","stderr":2.247,"costPerTest":0.028403,"latency":965.737},{"benchmarkId":"officeQaPro","benchmarkName":"OfficeQA Pro","category":"multimodal","score":62.4,"unit":"percent","family":"officeQaPro","index":72.0277910778195,"aboveFit":false,"run":null,"source":"OfficeQA Pro authors","sourceUrl":"https://arxiv.org/abs/2603.08655","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-spanish:overall","benchmarkName":"EuroEval Spanish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":62.21124999999999,"unit":"percent","family":"euroeval","index":56.62658730769667,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/spanish","measuredAt":null,"stderr":0.4990882311776145,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":61.011428571428574,"unit":"percent","family":"euroeval","index":54.99870570017079,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.44872836199324734,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":59.82875,"unit":"percent","family":"euroeval","index":53.3940830053287,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.5939039274158743,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:fabv2:overall","benchmarkName":"Finance Agent (v2)","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":57.85,"unit":"percent","family":"vals-ai:fabv2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/fabv2","measuredAt":"2026-10-29","stderr":2.061,"costPerTest":0.09525,"latency":890.519},{"benchmarkId":"openHarmonyBench","benchmarkName":"OpenHarmony Bench v1.0","category":"coding","score":57.3,"unit":"percent","family":"openHarmonyBench","index":64.25562796270816,"aboveFit":false,"run":null,"source":"OpenHarmony Bench authors","sourceUrl":"https://arxiv.org/abs/2608.16022","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-agentic-coding:overall","benchmarkName":"LiveBench Agentic Coding","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":56.76766666666666,"unit":"percent","family":"livebench:livebench-agentic-coding:overall","index":67.70621814719551,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Agentic%20Coding","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.1112763888888889,"latency":null},{"benchmarkId":"nl2Repo","benchmarkName":"NL2Repo","category":"coding","score":56.3,"unit":"percent","family":"nl2Repo","index":66.4765252819407,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:emb:overall","benchmarkName":"EMB","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":55.932,"unit":"percent","family":"vals-ai:emb:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/emb","measuredAt":"2026-10-20","stderr":3.561,"costPerTest":0.713033,"latency":4495.202},{"benchmarkId":"aaCweBench","benchmarkName":"Artificial Analysis CWE-Bench-AA","category":"agentic","score":55.8,"unit":"percent","family":"aaCweBench","index":68.317633283132,"aboveFit":false,"run":null,"source":"Artificial Analysis; dataset by Collinear AI","sourceUrl":"https://artificialanalysis.ai/evaluations/cwe-bench-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tiers-1-3-v2-private:overall","benchmarkName":"FrontierMath-Tiers-1-3-v2-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":55.78947368421052,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tiers-1-3-v2-private:overall","index":60.259581251461846,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.946997752822398,"costPerTest":null,"latency":null},{"benchmarkId":"hleWithTools","benchmarkName":"Humanity's Last Exam with tools","category":"agentic","score":55.3,"unit":"percent","family":"hleWithTools","index":64.55753939065194,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"babyVision","benchmarkName":"BabyVision","category":"multimodal","score":53.4,"unit":"percent","family":"babyVision","index":53.411483305708245,"aboveFit":false,"run":null,"source":"Meta AI","sourceUrl":"https://ai.meta.com/static-resource/muse-spark-1-1-evaluation-report","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-instruction-following:overall","benchmarkName":"LiveBench Instruction Following","category":"instruction","domain":null,"eligible":true,"ranked":true,"score":52.820750000000004,"unit":"percent","family":"livebench:livebench-instruction-following:overall","index":33.46584764353369,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=IF","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.0187505,"latency":null},{"benchmarkId":"vals-ai:ioi:overall","benchmarkName":"IOI","category":"coding","domain":null,"eligible":true,"ranked":true,"score":52.5,"unit":"percent","family":"vals-ai:ioi:overall","index":65.82631021522003,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/ioi","measuredAt":"2026-10-29","stderr":2.619,"costPerTest":0.429263,"latency":5548.524},{"benchmarkId":"aaItbench","benchmarkName":"Artificial Analysis ITBench-AA","category":"agentic","score":51.2,"unit":"percent","family":"aaItbench","index":73.46610648907574,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/itbench-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaCyberIndex","benchmarkName":"Artificial Analysis Cyber Index","category":"agentic","score":49.9,"unit":"percent","family":"aaCyberIndex","index":78.45832671803568,"aboveFit":false,"run":null,"source":"Artificial Analysis with the Cyber Index Alliance (Collinear AI, IBM, NVIDIA, Pasta Lab, Vercel)","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-cyber-index","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"automationBench","benchmarkName":"AutomationBench","category":"agentic","score":48.8,"unit":"percent","family":"automationBench","index":74.91252361957493,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaTau3Banking","benchmarkName":"Artificial Analysis Tau3-Banking","category":"agentic","score":47.2,"unit":"percent","family":"aaTau3Banking","index":72.91487670348333,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/tau3-banking","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:legal_research:overall","benchmarkName":"Legal Research Bench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":45.192,"unit":"percent","family":"vals-ai:legal_research:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_research","measuredAt":"2026-11-09","stderr":3.459,"costPerTest":0.240392,"latency":3067.486},{"benchmarkId":"vals-ai:skillsbench:overall","benchmarkName":"SkillsBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":40.159,"unit":"percent","family":"vals-ai:skillsbench:overall","index":55.03432322870727,"aboveFit":false,"run":"OpenHands","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/skillsbench","measuredAt":"2026-10-29","stderr":4.432,"costPerTest":0.078103,"latency":728.997},{"benchmarkId":"vals-ai:terminal-bench-4:overall","benchmarkName":"Terminal-Bench 4.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":37.374,"unit":"percent","family":"terminal-bench-4","index":72.26263247650405,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-4","measuredAt":"2026-11-09","stderr":2.672,"costPerTest":1.219907,"latency":3681.073},{"benchmarkId":"cursorBench40","benchmarkName":"cursorBench40","category":"coding","score":36.8,"unit":"percent","family":"cursorBench40","index":64.39636997299648,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mka","benchmarkName":"Multimodal Knowledge Agent Benchmark","category":"agentic","score":36.2,"unit":"percent","family":"mka","index":66.04301223589258,"aboveFit":false,"run":null,"source":"Google DeepMind, in partnership with Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mka","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"terminal-bench:terminal-bench-4-0-0:overall","benchmarkName":"Terminal-Bench 4.0.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":35.76,"unit":"percent","family":"terminal-bench-4","index":71.49624292539401,"aboveFit":false,"run":"none effort · Claude Code","source":"Terminal-Bench","sourceUrl":"https://www.tbench.ai/leaderboard","measuredAt":"2026-09-29","stderr":1.806122448979592,"costPerTest":0.35981068024242424,"latency":6209.8},{"benchmarkId":"terminal-bench:terminal-bench-4-1-0:overall","benchmarkName":"Terminal-Bench 4.1.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":35.45,"unit":"percent","family":"terminal-bench:terminal-bench-4-1-0:overall","index":69.50360060258845,"aboveFit":false,"run":"none effort · Claude Code","source":"Terminal-Bench","sourceUrl":"https://www.tbench.ai/leaderboard","measuredAt":"2026-09-29","stderr":1.816326530612245,"costPerTest":0.35981068024242424,"latency":6209.8},{"benchmarkId":"hermesIndex","benchmarkName":"Hermes Index: agentic capability and cost per task inside Hermes Agent","category":"agentic","score":34.95,"unit":"percent","family":"hermesIndex","index":64.58640609269396,"aboveFit":false,"run":null,"source":"Nous Research","sourceUrl":"https://portal.nousresearch.com/bench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:tax_agent_bench:overall","benchmarkName":"Tax Agent Bench","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":31.55,"unit":"percent","family":"vals-ai:tax_agent_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench","measuredAt":"2026-11-09","stderr":3.35,"costPerTest":0.138062,"latency":1297.006},{"benchmarkId":"vals-ai:vibe-code:overall","benchmarkName":"Vibe Code Bench v1.1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":30.759,"unit":"percent","family":"vals-ai:vibe-code:overall","index":53.34785305109573,"aboveFit":false,"run":"OpenHands","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","measuredAt":"2026-11-09","stderr":5.277,"costPerTest":4.69761,"latency":5027.281},{"benchmarkId":"livebench:livebench-long-horizon:overall","benchmarkName":"LiveBench Long Agentic","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":28.4,"unit":"percent","family":"livebench-2-agentic","index":39.35927467811118,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Long%20Agentic","measuredAt":"2026-10-14","stderr":null,"costPerTest":1.6287722222222223,"latency":null},{"benchmarkId":"agentsLastExam","benchmarkName":"Agents' Last Exam","category":"agentic","score":26.3,"unit":"percent","family":"agentsLastExam","index":62.891843635136006,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:proof_bench:overall","benchmarkName":"ProofBench v1.1","category":"math","domain":null,"eligible":true,"ranked":true,"score":21,"unit":"percent","family":"vals-ai:proof_bench:overall","index":54.40675931149999,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/proof_bench","measuredAt":"2026-11-09","stderr":4.094,"costPerTest":0.448469,"latency":4274.803},{"benchmarkId":"epoch-ai:epoch-chess-puzzles:overall","benchmarkName":"Chess Puzzles","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":21,"unit":"percent","family":"epoch-ai:epoch-chess-puzzles:overall","index":49.923994823290954,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":4.093601807403323,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:code-migration:overall","benchmarkName":"Code Migration","category":"coding","domain":null,"eligible":true,"ranked":true,"score":20.515,"unit":"percent","family":"vals-ai:code-migration:overall","index":56.08752796691927,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/code-migration","measuredAt":"2026-10-29","stderr":4.207,"costPerTest":6.257643,"latency":12331.871},{"benchmarkId":"aaDeepsecBench","benchmarkName":"Artificial Analysis DeepsecBench-AA","category":"agentic","score":19.9,"unit":"percent","family":"aaDeepsecBench","index":65.59200379951415,"aboveFit":false,"run":null,"source":"Artificial Analysis; dataset by Vercel","sourceUrl":"https://artificialanalysis.ai/evaluations/deepsecbench-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"frontierSweV2","benchmarkName":"FrontierSWE v2","category":"coding","score":18.1,"unit":"percent","family":"frontierSweV2","index":61.94141581438207,"aboveFit":false,"run":null,"source":"Proximal","sourceUrl":"https://www.frontierswe.com/blog/v2","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"bug-hunt-bench","benchmarkName":"Bug Hunt Bench","category":"coding","score":17.7,"unit":"percent","family":"bug-hunt-bench","index":62.11847067554691,"aboveFit":false,"run":null,"source":"Pawel Huryn","sourceUrl":"https://bughunt.productcompass.pm/method","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tier-4-v2-private:overall","benchmarkName":"FrontierMath-Tier-4-v2-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":17.073170731707318,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tier-4-v2-private:overall","index":55.1955002815125,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":5.949419959829496,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:vcb-1-100:overall","benchmarkName":"Vibe Code Bench 1-100","category":"coding","domain":null,"eligible":true,"ranked":true,"score":16.033,"unit":"percent","family":"vals-ai:vcb-1-100:overall","index":64.60717701564928,"aboveFit":false,"run":"OpenHands","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vcb-1-100","measuredAt":"2026-10-29","stderr":3.279,"costPerTest":0.925685,"latency":2343.096},{"benchmarkId":"vals-ai:hlab:overall","benchmarkName":"HLAB-Vals-1.1","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":10.417,"unit":"percent","family":"vals-ai:hlab:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/hlab","measuredAt":"2026-11-09","stderr":2.572,"costPerTest":0.323243,"latency":744.843},{"benchmarkId":"epoch-ai:epoch-mystery-game-puzzles:overall","benchmarkName":"Mystery Game Puzzles","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":8,"unit":"percent","family":"epoch-ai:epoch-mystery-game-puzzles:overall","index":44.876317375711565,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.7265992434429074,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:terminal-bench-science:overall","benchmarkName":"Terminal-Bench-Science","category":null,"domain":"Science","eligible":false,"ranked":true,"score":2.857,"unit":"percent","family":"vals-ai:terminal-bench-science:overall","index":null,"aboveFit":false,"run":"Mini-SWE-agent","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-science","measuredAt":"2026-11-09","stderr":2.006,"costPerTest":0.524048,"latency":12461.546},{"benchmarkId":"lmarena:agent-task-outcome:overall","benchmarkName":"Agent Arena task outcome","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":1.3404427824021847,"unit":"index","family":"agent-arena","index":67.6328554207663,"aboveFit":false,"run":null,"source":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset","measuredAt":"2026-11-11","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:srebench:overall","benchmarkName":"SRE Bench","category":null,"domain":"Other","eligible":false,"ranked":true,"score":0.763,"unit":"percent","family":"vals-ai:srebench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/srebench","measuredAt":"2026-11-05","stderr":0.539,"costPerTest":2.290191,"latency":11693.322},{"benchmarkId":"mazeBench","benchmarkName":"MazeBench: visual spatial reasoning in a 3D open world","category":"agentic","score":0,"unit":"percent","family":"mazeBench","index":61.97866734834783,"aboveFit":false,"run":null,"source":"Jonathan Pappas and David Pappas","sourceUrl":"https://mazebench.com/blog?post=maze-bench-results","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:programbench:overall","benchmarkName":"ProgramBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":0,"unit":"percent","family":"vals-ai:programbench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/programbench","measuredAt":"2026-11-09","stderr":0,"costPerTest":1.597239,"latency":19801.258},{"benchmarkId":"lmarena:agent-steerability:overall","benchmarkName":"Agent Arena steerability","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":-5.0090373649497755,"unit":"index","family":"agent-arena","index":62.41258732321006,"aboveFit":false,"run":null,"source":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset","measuredAt":"2026-11-11","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lmarena:agent-bash-recovery:overall","benchmarkName":"Agent Arena command recovery","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":-5.699069676314816,"unit":"index","family":"agent-arena","index":61.845272585986386,"aboveFit":false,"run":null,"source":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset","measuredAt":"2026-10-08","stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":0.0399,"outputPerMillion":0.0784,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":121.5,"timeToFirstToken":0.897,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":{"elo":1470.6575559766993,"eloUpper":1475.088386083356,"eloLower":1466.226725870043,"votes":50189,"rank":31,"rankUpper":null,"rankLower":null,"arenaName":"GLM-5.3-Flash","organization":"zai","updatedAt":"2026-11-09"},"quality":{"coveragePercent":100,"benchmarkCount":56,"categoryCount":8,"confidence":"high","state":"available"},"hosts":[{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":0.0399,"outputPerMillion":0.0784,"cacheReadPerMillion":0.03,"contextTokens":1048576,"maxOutputTokens":131072,"deprecated":false},{"host":"vultr","name":"Vultr","kind":"cloud","url":"https://api.vultrinference.com/","inputPerMillion":0.1,"outputPerMillion":0.35,"cacheReadPerMillion":null,"contextTokens":1048576,"maxOutputTokens":131072,"deprecated":false},{"host":"coreweave","name":"CoreWeave","kind":"host","url":"https://docs.wandb.ai/inference","inputPerMillion":0.12,"outputPerMillion":0.38,"cacheReadPerMillion":0.02,"contextTokens":1048576,"maxOutputTokens":1048576,"deprecated":false},{"host":"volcengine","name":"Volcengine Ark","kind":"cloud","url":"https://www.volcengine.com/docs/82379/1330310","inputPerMillion":0.11875,"outputPerMillion":0.41563,"cacheReadPerMillion":0.03414,"contextTokens":1000000,"maxOutputTokens":131072,"deprecated":false},{"host":"zhipu-ai","name":"Zhipu AI","kind":"maker","url":"https://docs.bigmodel.cn/cn/guide/start/pricing","inputPerMillion":0.11952,"outputPerMillion":0.41831,"cacheReadPerMillion":0.03436,"contextTokens":1000000,"maxOutputTokens":131072,"deprecated":false},{"host":"arcee-ai","name":"Arcee AI","kind":"host","url":"https://docs.arcee.ai","inputPerMillion":0.15,"outputPerMillion":0.5,"cacheReadPerMillion":0.03,"contextTokens":1000000,"maxOutputTokens":131072,"deprecated":false},{"host":"baseten","name":"Baseten","kind":"host","url":"https://docs.baseten.co/inference/model-apis/overview","inputPerMillion":0.15,"outputPerMillion":0.5,"cacheReadPerMillion":0.03,"contextTokens":1048576,"maxOutputTokens":131072,"deprecated":false},{"host":"cloudflare","name":"Cloudflare Workers AI","kind":"cloud","url":"https://developers.cloudflare.com/workers-ai/models/","inputPerMillion":0.15,"outputPerMillion":0.5,"cacheReadPerMillion":0.03,"contextTokens":1048576,"maxOutputTokens":1048576,"deprecated":true},{"host":"deepinfra","name":"Deep Infra","kind":"host","url":"https://deepinfra.com/models","inputPerMillion":0.15,"outputPerMillion":0.5,"cacheReadPerMillion":0.03,"contextTokens":1048576,"maxOutputTokens":131072,"deprecated":false},{"host":"digitalocean","name":"DigitalOcean","kind":"cloud","url":"https://docs.digitalocean.com/products/gradient-ai-platform/details/models/","inputPerMillion":0.15,"outputPerMillion":0.5,"cacheReadPerMillion":0.03,"contextTokens":1048576,"maxOutputTokens":131072,"deprecated":false},{"host":"fireworks-ai","name":"Fireworks AI","kind":"host","url":"https://fireworks.ai/docs/","inputPerMillion":0.15,"outputPerMillion":0.5,"cacheReadPerMillion":0.03,"contextTokens":1048573,"maxOutputTokens":131072,"deprecated":false},{"host":"friendli","name":"Friendli","kind":"host","url":"https://friendli.ai/docs/guides/serverless_endpoints/introduction","inputPerMillion":0.15,"outputPerMillion":0.5,"cacheReadPerMillion":0.03,"contextTokens":1048576,"maxOutputTokens":131072,"deprecated":false},{"host":"mistral","name":"Mistral","kind":"host","url":"https://docs.mistral.ai/getting-started/models/","inputPerMillion":0.15,"outputPerMillion":0.5,"cacheReadPerMillion":0.03,"contextTokens":1048576,"maxOutputTokens":131072,"deprecated":false},{"host":"modal","name":"Modal","kind":"host","url":"https://modal.com/docs/guide/endpoints","inputPerMillion":0.15,"outputPerMillion":0.5,"cacheReadPerMillion":0.03,"contextTokens":1000000,"maxOutputTokens":131072,"deprecated":false},{"host":"nebius","name":"Nebius","kind":"host","url":"https://docs.tokenfactory.nebius.com/","inputPerMillion":0.15,"outputPerMillion":0.5,"cacheReadPerMillion":0.03,"contextTokens":1024000,"maxOutputTokens":1024000,"deprecated":false},{"host":"novita-ai","name":"Novita AI","kind":"host","url":"https://novita.ai/docs/guides/introduction","inputPerMillion":0.15,"outputPerMillion":0.5,"cacheReadPerMillion":0.03,"contextTokens":1048576,"maxOutputTokens":131072,"deprecated":false},{"host":"siliconflow","name":"SiliconFlow","kind":"host","url":"https://cloud.siliconflow.com/models","inputPerMillion":0.15,"outputPerMillion":0.5,"cacheReadPerMillion":0.03,"contextTokens":1049000,"maxOutputTokens":262000,"deprecated":false},{"host":"together-ai","name":"Together AI","kind":"host","url":"https://docs.together.ai/docs/serverless-models","inputPerMillion":0.15,"outputPerMillion":0.5,"cacheReadPerMillion":0.03,"contextTokens":1048575,"maxOutputTokens":400000,"deprecated":false},{"host":"z-ai","name":"Z.AI","kind":"maker","url":"https://docs.z.ai/guides/overview/pricing","inputPerMillion":0.15,"outputPerMillion":0.5,"cacheReadPerMillion":0.03,"contextTokens":1000000,"maxOutputTokens":131072,"deprecated":false},{"host":"berget-ai","name":"Berget AI","kind":"host","url":"https://api.berget.ai","inputPerMillion":0.29,"outputPerMillion":0.58,"cacheReadPerMillion":null,"contextTokens":1048576,"maxOutputTokens":16384,"deprecated":false},{"host":"privatemode","name":"Privatemode","kind":"host","url":"https://docs.privatemode.ai/api/overview","inputPerMillion":0.2311,"outputPerMillion":0.7511,"cacheReadPerMillion":0.0578,"contextTokens":1000000,"maxOutputTokens":131072,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:arc-prize-foundation","name":"ARC Prize Foundation","url":"https://arcprize.org","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:livebench","name":"LiveBench","url":"https://livebench.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:terminal-bench","name":"Terminal-Bench","url":"https://www.tbench.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:lmarena","name":"LMArena","url":"https://huggingface.co","license":"CC BY 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"qwen3-8-omni-flash","providerId":"alibaba","slug":"qwen3-8-omni-flash","name":"Qwen3.8-Omni-Flash","provider":"Alibaba","description":"Qwen3.8-Omni-Flash is a reasoning model from Alibaba. 13 benchmarks count toward its index, in 5 categories.","logoUrl":null,"releaseDate":"2026-09-18","contextWindowTokens":1000000,"modalities":["text","image","audio","video"],"openWeights":false,"details":{"apiId":"qwen/qwen3.8-omni-flash","maxOutputTokens":131072,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.016,"weightsUrl":null},"reasoning":true,"overallScore":62.66919470097193,"overallScoreRange":{"low":55.18444361449095,"high":70.15394578745291},"badges":[],"capabilityScores":{"agentic":64.5041905813019,"coding":61.484178311686016,"reasoning":null,"multimodal":64.49958150999184,"knowledge":60.83932183122547,"multilingual":null,"instruction":65.1862445006694,"math":null},"capabilityCounts":{"coding":3,"multimodal":5,"knowledge":2,"agentic":2,"instruction":1},"benchmarks":[{"benchmarkId":"mathVisionPython","benchmarkName":"MathVision with Python","category":"multimodal","score":96.2,"unit":"percent","family":"mathVisionPython","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"liveCodeBenchV6","benchmarkName":"LiveCodeBench v6","category":"coding","score":92.6,"unit":"percent","family":"liveCodeBenchV6","index":60.44696292695562,"aboveFit":false,"run":null,"source":"LiveCodeBench maintainers","sourceUrl":"https://github.com/LiveCodeBench/LiveCodeBench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mathVision","benchmarkName":"MathVision","category":"multimodal","score":91.8,"unit":"percent","family":"mathVision","index":62.60433020438575,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"charxiv","benchmarkName":"CharXiv Reasoning","category":"multimodal","score":91.4,"unit":"percent","family":"charxiv","index":68.24295712819405,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gpqa","benchmarkName":"Graduate-Level Google-Proof Q&A","category":"knowledge","score":91,"unit":"percent","family":"gpqa-diamond","index":60.67181415554997,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gpqaDiamond","benchmarkName":"GPQA Diamond","category":"knowledge","score":91,"unit":"percent","family":"gpqa-diamond","index":60.67181415554997,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"realWorldQa","benchmarkName":"RealWorldQA","category":"multimodal","score":87.7,"unit":"percent","family":"realWorldQa","index":63.82766783896797,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"androidWorld","benchmarkName":"AndroidWorld","category":"agentic","score":87.1,"unit":"percent","family":"androidWorld","index":65.91267407488706,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"charxivNoTools","benchmarkName":"CharXiv Reasoning without tools","category":"multimodal","score":83.5,"unit":"percent","family":"charxivNoTools","index":63.91137958488254,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"ifBench","benchmarkName":"Instruction Following Benchmark","category":"instruction","score":81.5,"unit":"percent","family":"ifbench-family","index":65.1862445006694,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lvBench","benchmarkName":"LVBench","category":"multimodal","score":76.9,"unit":"percent","family":"lvBench","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"coworkBench","benchmarkName":"CoWorkBench","category":"agentic","score":75.3,"unit":"percent","family":"coworkBench","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"erqa","benchmarkName":"ERQA","category":"multimodal","score":71,"unit":"percent","family":"erqa","index":63.91157279352889,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"swePro","benchmarkName":"SWE-bench Pro","category":"coding","score":63.3,"unit":"percent","family":"swePro","index":62.74629548311284,"aboveFit":false,"run":null,"source":"Xiang Deng, Jeff Da, Edwin Pan, Yannis Yiming He, Charles Ide, Kanak Garg, Niklas Lauffer, Andrew Park, Nitin Pasari, Chetan Rane, Karmini Sampath, Maya Krishnan, Srivatsa Kundurthy, Sean Hendryx, Zifan Wang, Chen Bo Calvin Zhang, Noah Jacobson, Bing Liu, Brad Kenstler","sourceUrl":"https://arxiv.org/abs/2509.16941","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vision2Web","benchmarkName":"Vision2Web","category":"multimodal","score":62.9,"unit":"percent","family":"vision2Web","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"deepSwe","benchmarkName":"DeepSWE","category":"agentic","score":57.8,"unit":"percent","family":"deepSwe","index":63.09570708771673,"aboveFit":false,"run":null,"source":"Datacurve AI","sourceUrl":"https://deepswe.datacurve.ai/blog","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"nl2Repo","benchmarkName":"NL2Repo","category":"coding","score":48.9,"unit":"percent","family":"nl2Repo","index":61.25927652498958,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hle","benchmarkName":"Humanity's Last Exam","category":"knowledge","score":36.5,"unit":"percent","family":"hle","index":61.00682950690097,"aboveFit":false,"run":null,"source":"Center for AI Safety, Scale AI, and thousands of expert contributors","sourceUrl":"https://lastexam.ai/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":0.15,"outputPerMillion":0.47,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":41,"timeToFirstToken":1.115,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":70,"benchmarkCount":13,"categoryCount":5,"confidence":"low","state":"available"},"hosts":[{"host":"alibaba-cloud","name":"Alibaba Cloud","kind":"maker","url":"https://www.alibabacloud.com/help/en/model-studio/models","inputPerMillion":0.15,"outputPerMillion":0.47,"cacheReadPerMillion":0.016,"contextTokens":1000000,"maxOutputTokens":131072,"deprecated":false},{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":0.15,"outputPerMillion":0.47,"cacheReadPerMillion":0.016,"contextTokens":1000000,"maxOutputTokens":131072,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gemini-3-1-pro","providerId":"google","slug":"gemini-3-1-pro","name":"Gemini 3.1 Pro","provider":"Google","description":"Gemini 3.1 Pro is a reasoning model from Google. 38 benchmarks count toward its index, in 7 categories.","logoUrl":null,"releaseDate":"2026-02-19","contextWindowTokens":1000000,"modalities":["text"],"openWeights":false,"details":null,"reasoning":true,"overallScore":62.283947049675234,"overallScoreRange":{"low":59.160826669990804,"high":65.40706742935967},"badges":[],"capabilityScores":{"agentic":58.44026589090937,"coding":57.99898889318562,"reasoning":65.07570745729129,"multimodal":63.52350033477777,"knowledge":69.38855726263641,"multilingual":null,"instruction":61.074397795419586,"math":61.69699363031686},"capabilityCounts":{"agentic":9,"math":5,"knowledge":4,"multimodal":7,"reasoning":6,"coding":6,"instruction":1},"benchmarks":[{"benchmarkId":"tau2Bench","benchmarkName":"τ²-Bench Tool-Agent-User Evaluation","category":"agentic","score":95.6,"unit":"percent","family":"tau2-bench","index":66.22702964504438,"aboveFit":false,"run":null,"source":"Victor Barres, Honghua Dong, Soham Ray, Xujie Si, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2506.07982","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":95.55555555555556,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":64.51274413550408,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":3.1067790907534736,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":94.44444444444444,"unit":"percent","family":"gpqa-diamond","index":63.83096448027367,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.6319950700767385,"costPerTest":null,"latency":null},{"benchmarkId":"gpqaDiamond","benchmarkName":"GPQA Diamond","category":"knowledge","score":94.3,"unit":"percent","family":"gpqa-diamond","index":63.698483982785255,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":94.1,"unit":"percent","family":"gpqa-diamond","index":63.515049447801296,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaGlobalMmluLite","benchmarkName":"Artificial Analysis Global-MMLU-Lite","category":"multilingual","score":93.2,"unit":"percent","family":"aaGlobalMmluLite","index":null,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/global-mmlu-lite","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-2025-02-28-public:overall","benchmarkName":"FrontierMath-2025-02-28-Public","category":"math","domain":null,"eligible":false,"ranked":true,"score":88.88888888888889,"unit":"percent","family":"epoch-ai:epoch-frontiermath-2025-02-28-public:overall","index":null,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":11.111111111111112,"costPerTest":null,"latency":null},{"benchmarkId":"screenSpotPro","benchmarkName":"ScreenSpot Pro","category":"multimodal","score":84.4,"unit":"percent","family":"screenSpotPro","index":64.33731809876006,"aboveFit":false,"run":null,"source":"Kaixin Li, Ziyang Meng, Hongzhan Lin, Ziyang Luo, Yuchen Tian, Jing Ma, Zhiyong Huang, Tat-Seng Chua","sourceUrl":"https://arxiv.org/abs/2504.07981","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmmuPro","benchmarkName":"Massive Multi-discipline Multimodal Understanding Pro","category":"multimodal","score":83.9,"unit":"percent","family":"mmmu-pro","index":64.34130407133364,"aboveFit":false,"run":null,"source":"MMMU-Pro authors","sourceUrl":"https://arxiv.org/abs/2409.02813","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"liveCodeBenchPro","benchmarkName":"LiveCodeBench Pro","category":"coding","score":82.9,"unit":"percent","family":"liveCodeBenchPro","index":null,"aboveFit":false,"run":null,"source":"LiveCodeBench Pro authors","sourceUrl":"https://arxiv.org/abs/2506.11928","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaMmmuPro","benchmarkName":"Artificial Analysis MMMU-Pro","category":"multimodal","score":82.4,"unit":"percent","family":"mmmu-pro","index":62.38918567365489,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmmu-pro","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":82,"unit":"percent","family":"lcr","index":63.55665255719484,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"medXpertQaMm","benchmarkName":"MedXpertQA Multimodal","category":"multimodal","score":81.3,"unit":"percent","family":"medXpertQaMm","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"charxiv","benchmarkName":"CharXiv Reasoning","category":"multimodal","score":80.2,"unit":"percent","family":"charxiv","index":55.49177476184574,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"reactNativeEvals","benchmarkName":"React Native Evals","category":"coding","score":78.9,"unit":"percent","family":"reactNativeEvals","index":55.58921076772292,"aboveFit":false,"run":null,"source":"Callstack","sourceUrl":"https://rn-evals.vercel.app/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaIfBench","benchmarkName":"Artificial Analysis IFBench","category":"instruction","score":77.1,"unit":"percent","family":"ifbench-family","index":61.074397795419586,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-swe-bench-verified:overall","benchmarkName":"SWE-Bench verified","category":"coding","domain":null,"eligible":true,"ranked":true,"score":75.6198347107438,"unit":"percent","family":"swe-bench-verified","index":57.001909651130994,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.9537213360116366,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-simpleqa-verified:overall","benchmarkName":"SimpleQA Verified","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":73.5,"unit":"percent","family":"epoch-ai:epoch-simpleqa-verified:overall","index":85.15104076388278,"aboveFit":true,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.3963164754810065,"costPerTest":null,"latency":null},{"benchmarkId":"simpleVqa","benchmarkName":"SimpleVQA","category":"multimodal","score":72.4,"unit":"percent","family":"simpleVqa","index":62.625943230763326,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"medXpertQaText","benchmarkName":"MedXpertQA Text","category":"knowledge","score":71.5,"unit":"percent","family":"medXpertQaText","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"deepSearchQa","benchmarkName":"DeepSearchQA","category":"agentic","score":69.7,"unit":"percent","family":"deepSearchQa","index":51.68264171479413,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"erqa","benchmarkName":"ERQA","category":"multimodal","score":69.4,"unit":"percent","family":"erqa","index":62.43036540134379,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"parseBenchLlamaIndex","benchmarkName":"ParseBench: document parsing for AI agents (LlamaIndex leaderboard)","category":"multimodal","score":67.32,"unit":"percent","family":"parseBenchLlamaIndex","index":67.64725753968649,"aboveFit":false,"run":null,"source":"Boyang Zhang, Sebastián G. Acosta, Preston Carlson, Sacha Bron, Pierre-Loïc Doulcet, Daniel B. Ospina and Simon Suo (LlamaIndex)","sourceUrl":"https://arxiv.org/abs/2604.08538","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tiers-1-3-v2-private:overall","benchmarkName":"FrontierMath-Tiers-1-3-v2-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":59.64912280701754,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tiers-1-3-v2-private:overall","index":62.29111417226392,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.911181956525014,"costPerTest":null,"latency":null},{"benchmarkId":"aaSciCode","benchmarkName":"Artificial Analysis SciCode","category":"coding","score":58.7,"unit":"percent","family":"scicode-family","index":72.3664687370561,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/scicode","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"clawEval","benchmarkName":"Claw-Eval","category":"agentic","score":57.8,"unit":"percent","family":"clawEval","index":48.69451344655581,"aboveFit":false,"run":null,"source":"Bowen Ye, Rang Li, Qibin Yang, Yuanxin Liu, Linli Yao, Hanglong Lv, Zhihui Xie, Chenxin An, Lei Li, Lingpeng Kong, Qi Liu, Zhifang Sui, Tong Yang","sourceUrl":"https://arxiv.org/abs/2604.06132","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gertLabs","benchmarkName":"Gert Labs Composite Game Benchmark","category":"agentic","score":56.87,"unit":"percent","family":"gertLabs","index":61.59535053776323,"aboveFit":false,"run":null,"source":"Gert Labs","sourceUrl":"https://gertlabs.com/rankings","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"terminalBenchHard","benchmarkName":"Terminal-Bench Hard","category":"agentic","score":53.8,"unit":"percent","family":"terminalBenchHard","index":63.24599400663665,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/terminalbench-hard","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-chess-puzzles:overall","benchmarkName":"Chess Puzzles","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":51,"unit":"percent","family":"epoch-ai:epoch-chess-puzzles:overall","index":72.69445067779384,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":5.024183937956913,"costPerTest":null,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":47,"unit":"percent","family":"hle","index":71.54140332971382,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hleNoTools","benchmarkName":"Humanity's Last Exam without tools","category":"knowledge","score":45.4,"unit":"percent","family":"hle","index":69.93613493766614,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaEnterpriseOpsGym","benchmarkName":"Artificial Analysis EnterpriseOps-Gym","category":"agentic","score":42.2,"unit":"percent","family":"aaEnterpriseOpsGym","index":null,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/enterprise-ops-gym-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaAnalystAgent","benchmarkName":"Artificial Analysis AnalystAgent","category":"agentic","score":41.3,"unit":"percent","family":"aaAnalystAgent","index":62.17923610169161,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/aa-analyst-agent","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"scarfBench","benchmarkName":"ScarfBench: Cross-Framework Application Migration in Enterprise Java","category":"coding","score":37.6,"unit":"percent","family":"scarfBench","index":50.1042487385192,"aboveFit":false,"run":null,"source":"Advait Pavuluri, Bridget McGinn, Ashita Saxena, George Safta, Srikanth Tamilselvam, Raju Pavuluri, Michele Merler, Baishakhi Ray, Rahul Krishna (IBM Research)","sourceUrl":"https://arxiv.org/abs/2605.06754","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-2025-02-28-private:overall","benchmarkName":"FrontierMath-2025-02-28-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":36.9,"unit":"percent","family":"epoch-ai:epoch-frontiermath-2025-02-28-private:overall","index":62.48866777031461,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.8000000000000003,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-mystery-game-puzzles:overall","benchmarkName":"Mystery Game Puzzles","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":34,"unit":"percent","family":"epoch-ai:epoch-mystery-game-puzzles:overall","index":65.01110469004604,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":4.760952285695233,"costPerTest":null,"latency":null},{"benchmarkId":"apexAgentsAa","benchmarkName":"APEX-Agents-AA","category":"agentic","score":32,"unit":"percent","family":"apexAgentsAa","index":64.06654062537862,"aboveFit":false,"run":null,"source":"Artificial Analysis / Mercor","sourceUrl":"https://artificialanalysis.ai/evaluations/apex-agents-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"zeroBench","benchmarkName":"ZeroBench","category":"multimodal","score":29,"unit":"percent","family":"zeroBench","index":68.76659843855073,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tier-4-v2-private:overall","benchmarkName":"FrontierMath-Tier-4-v2-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":26.82926829268293,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tier-4-v2-private:overall","index":59.02818358680503,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":7.005564203095157,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-furniture-assembly:overall","benchmarkName":"Furniture Assembly","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":26.666666666666668,"unit":"percent","family":"epoch-ai:epoch-furniture-assembly:overall","index":59.09490200018625,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":5.6331678510110805,"costPerTest":null,"latency":null},{"benchmarkId":"postTrainBench11","benchmarkName":"PostTrainBench v1.1","category":"coding","score":21.99,"unit":"percent","family":"postTrainBench11","index":59.46270632606898,"aboveFit":false,"run":null,"source":"PostTrainBench team","sourceUrl":"https://posttrainbench.com/?version=v1.1","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"healthBenchHard","benchmarkName":"HealthBench Hard","category":"knowledge","score":20.6,"unit":"percent","family":"healthBenchHard","index":57.96593517018764,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":17.7,"unit":"percent","family":"critpt","index":71.11425071834321,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-private:overall","benchmarkName":"FrontierMath-Tier-4-2025-07-01-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":16.7,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-private:overall","index":60.16425848669668,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":5.4,"costPerTest":null,"latency":null},{"benchmarkId":"gdpvalAaNormalized","benchmarkName":"GDPval-AA normalized","category":"agentic","score":14.6,"unit":"percent","family":"gdpvalAaNormalized","index":44.47243563789353,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-ebr-bench:overall","benchmarkName":"EBR-bench","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":14.285714285714285,"unit":"percent","family":"epoch-ai:epoch-ebr-bench:overall","index":58.98288410018353,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"researchClawBench","benchmarkName":"ResearchClawBench","category":"agentic","score":13.3,"unit":"percent","family":"researchClawBench","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-mirrorcode:overall","benchmarkName":"MirrorCode","category":"coding","domain":null,"eligible":true,"ranked":true,"score":8.888888888888888,"unit":"percent","family":"epoch-ai:epoch-mirrorcode:overall","index":53.46938913861552,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":4.561939004190449,"costPerTest":null,"latency":null},{"benchmarkId":"mazeBench","benchmarkName":"MazeBench: visual spatial reasoning in a 3D open world","category":"agentic","score":1,"unit":"percent","family":"mazeBench","index":63.79865130242642,"aboveFit":false,"run":null,"source":"Jonathan Pappas and David Pappas","sourceUrl":"https://mazebench.com/blog?post=maze-bench-results","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-public:overall","benchmarkName":"FrontierMath-Tier-4-2025-07-01-Public","category":"math","domain":null,"eligible":false,"ranked":true,"score":0,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-public:overall","index":null,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":2,"outputPerMillion":12,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":151,"timeToFirstToken":19.95,"source":"BenchLM","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":95,"benchmarkCount":38,"categoryCount":7,"confidence":"high","state":"available"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"claude-opus-4-7","providerId":"anthropic","slug":"claude-opus-4-7","name":"Claude Opus 4.7","provider":"Anthropic","description":"Claude Opus 4.7 is a standard model from Anthropic. 43 benchmarks count toward its index, in 7 categories.","logoUrl":null,"releaseDate":"2026-04-16","contextWindowTokens":1000000,"modalities":["text","image","file"],"openWeights":false,"details":{"apiId":"anthropic/claude-opus-4.7","maxOutputTokens":128000,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.5,"weightsUrl":null},"reasoning":false,"overallScore":62.00844022142496,"overallScoreRange":{"low":59.042538339146745,"high":64.97434210370318},"badges":[],"capabilityScores":{"agentic":60.632103848487844,"coding":64.45218401928437,"reasoning":61.120846205303145,"multimodal":61.31389474567618,"knowledge":61.76005963349079,"multilingual":null,"instruction":45.69706250053448,"math":64.2749567429087},"capabilityCounts":{"math":7,"knowledge":6,"reasoning":8,"multimodal":2,"coding":10,"agentic":8,"instruction":2},"benchmarks":[{"benchmarkId":"vals-ai:aime:overall","benchmarkName":"AIME","category":"math","domain":null,"eligible":true,"ranked":true,"score":96.25,"unit":"percent","family":"vals-ai:aime:overall","index":57.56822043203955,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/aime","measuredAt":"2026-04-16","stderr":0.522,"costPerTest":0.450513,"latency":182.065},{"benchmarkId":"livebench:livebench-math:overall","benchmarkName":"LiveBench Mathematics","category":"math","domain":null,"eligible":true,"ranked":true,"score":92.85374999999999,"unit":"percent","family":"livebench:livebench-math:overall","index":69.18688710094004,"aboveFit":false,"run":"xhigh effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Mathematics","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.2990437994722955,"latency":null},{"benchmarkId":"vals-ai:gpqa:overall","benchmarkName":"GPQA Diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":90.152,"unit":"percent","family":"gpqa-diamond","index":59.89405172721799,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","measuredAt":"2026-09-01","stderr":1.545,"costPerTest":0.365129,"latency":202.52},{"benchmarkId":"epoch-ai:epoch-frontiermath-2025-02-28-public:overall","benchmarkName":"FrontierMath-2025-02-28-Public","category":"math","domain":null,"eligible":false,"ranked":true,"score":90,"unit":"percent","family":"epoch-ai:epoch-frontiermath-2025-02-28-public:overall","index":null,"aboveFit":false,"run":"xhigh effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":10,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:mmlu_pro:overall","benchmarkName":"MMLU Pro","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":89.871,"unit":"percent","family":"mmlu-pro","index":57.70743114476945,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","measuredAt":"2026-09-01","stderr":0.305,"costPerTest":0.061711,"latency":187.83},{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":88.5,"unit":"percent","family":"gpqa-diamond","index":58.37888246825051,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-reasoning:overall","benchmarkName":"LiveBench Reasoning","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":87.19225,"unit":"percent","family":"livebench","index":74.571526185929,"aboveFit":false,"run":"xhigh effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Reasoning","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.30255,"latency":null},{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":86.66666666666667,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":59.341241506765535,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":5.124707431905383,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":86.36363636363636,"unit":"percent","family":"gpqa-diamond","index":56.419468117285504,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.445015597318985,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:mmmu:overall","benchmarkName":"MMMU Pro","category":"multimodal","domain":null,"eligible":true,"ranked":true,"score":85.549,"unit":"percent","family":"mmmu-pro","index":66.48733289651514,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmmu","measuredAt":"2026-09-01","stderr":0.845,"costPerTest":0.170461,"latency":70.801},{"benchmarkId":"vals-ai:legal_bench:overall","benchmarkName":"LegalBench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":85.251,"unit":"percent","family":"vals-ai:legal_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","measuredAt":"2026-10-13","stderr":0.451,"costPerTest":0.008332,"latency":26.622},{"benchmarkId":"vals-ai:lcb:overall","benchmarkName":"LiveCodeBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":85.073,"unit":"percent","family":"livecodebench","index":59.9323420784488,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","measuredAt":"2026-09-01","stderr":1.006,"costPerTest":0.350587,"latency":461.91},{"benchmarkId":"epoch-ai:epoch-swe-bench-verified:overall","benchmarkName":"SWE-Bench verified","category":"coding","domain":null,"eligible":true,"ranked":true,"score":83.47107438016529,"unit":"percent","family":"swe-bench-verified","index":62.79265096430874,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.6901169179529603,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:medscribe:overall","benchmarkName":"MedScribe","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":82.953,"unit":"percent","family":"vals-ai:medscribe:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medscribe","measuredAt":"2026-10-29","stderr":1.977,"costPerTest":0.177841,"latency":67.496},{"benchmarkId":"reactNativeEvals","benchmarkName":"React Native Evals","category":"coding","score":82.8,"unit":"percent","family":"reactNativeEvals","index":59.80553207352714,"aboveFit":false,"run":null,"source":"Callstack","sourceUrl":"https://rn-evals.vercel.app/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-coding:overall","benchmarkName":"LiveBench Coding","category":"coding","domain":null,"eligible":true,"ranked":true,"score":82.088,"unit":"percent","family":"livebench:livebench-coding:overall","index":70.27342249734284,"aboveFit":false,"run":"xhigh effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Coding","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.028659829059829056,"latency":null},{"benchmarkId":"vals-ai:swebench:overall","benchmarkName":"SWE-bench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":82,"unit":"percent","family":"swe-bench-verified","index":61.70764890773438,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/swebench","measuredAt":"2026-09-01","stderr":1.72,"costPerTest":2.423427,"latency":441.988},{"benchmarkId":"livebench:livebench-data-analysis:overall","benchmarkName":"LiveBench Data Analysis","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":78.26366666666667,"unit":"percent","family":"livebench","index":63.005135798501456,"aboveFit":false,"run":"xhigh effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Data%20Analysis","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.4982393333333333,"latency":null},{"benchmarkId":"livebench:livebench-language:overall","benchmarkName":"LiveBench Language","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":77.914,"unit":"percent","family":"livebench:livebench-language:overall","index":62.66464221459847,"aboveFit":false,"run":"xhigh effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Language","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.09734333333333334,"latency":null},{"benchmarkId":"aaMmmuPro","benchmarkName":"Artificial Analysis MMMU-Pro","category":"multimodal","score":76.4,"unit":"percent","family":"mmmu-pro","index":54.580712082939904,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmmu-pro","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":75.7,"unit":"percent","family":"lcr","index":59.234345246005184,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:tax_eval_v2:overall","benchmarkName":"TaxEval v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":75.266,"unit":"percent","family":"vals-ai:tax_eval_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","measuredAt":"2026-09-01","stderr":0.841,"costPerTest":0.124592,"latency":62.071},{"benchmarkId":"tau2Bench","benchmarkName":"τ²-Bench Tool-Agent-User Evaluation","category":"agentic","score":74,"unit":"percent","family":"tau2-bench","index":51.89385042753771,"aboveFit":false,"run":null,"source":"Victor Barres, Honghua Dong, Soham Ray, Xujie Si, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2506.07982","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:vibe-code:overall","benchmarkName":"Vibe Code Bench v1.1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":71.003,"unit":"percent","family":"vals-ai:vibe-code:overall","index":67.86852053175645,"aboveFit":false,"run":"OpenHands","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","measuredAt":"2026-11-09","stderr":4.506,"costPerTest":21.407199,"latency":2141.902},{"benchmarkId":"vals-ai:mortgage_tax:overall","benchmarkName":"MortgageTax","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":70.27,"unit":"percent","family":"vals-ai:mortgage_tax:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mortgage_tax","measuredAt":"2026-09-01","stderr":0.896,"costPerTest":0.084417,"latency":16.101},{"benchmarkId":"epoch-ai:epoch-frontiermath-tiers-1-3-v2-private:overall","benchmarkName":"FrontierMath-Tiers-1-3-v2-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":70.17543859649122,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tiers-1-3-v2-private:overall","index":67.83165850172412,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.7146911721220195,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:terminal-bench-2:overall","benchmarkName":"Terminal-Bench 2.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":68.539,"unit":"percent","family":"vals-ai:terminal-bench-2:overall","index":67.81936786401259,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2","measuredAt":"2026-06-04","stderr":4.95,"costPerTest":0.975907,"latency":660.89},{"benchmarkId":"vals-ai:terminal-bench-2-1:overall","benchmarkName":"Terminal-Bench 2.1","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":68.539,"unit":"percent","family":"terminal-bench-2-1","index":61.22652303296843,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2-1","measuredAt":"2026-10-01","stderr":1.297,"costPerTest":1.977087,"latency":816.587},{"benchmarkId":"vals-ai:case_law_v2:overall","benchmarkName":"CaseLaw v2","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":68.381,"unit":"percent","family":"vals-ai:case_law_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/case_law_v2","measuredAt":"2026-05-04","stderr":0.429,"costPerTest":0.161535,"latency":17.677},{"benchmarkId":"livebench:livebench-instruction-following:overall","benchmarkName":"LiveBench Instruction Following","category":"instruction","domain":null,"eligible":true,"ranked":true,"score":66.7375,"unit":"percent","family":"livebench:livebench-instruction-following:overall","index":61.62583280243773,"aboveFit":false,"run":"xhigh effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=IF","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.281429,"latency":null},{"benchmarkId":"vals-ai:corp_fin_v2:overall","benchmarkName":"CorpFin v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":66.084,"unit":"percent","family":"vals-ai:corp_fin_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","measuredAt":"2026-08-12","stderr":0.933,"costPerTest":0.816017,"latency":17.75},{"benchmarkId":"gertLabs","benchmarkName":"Gert Labs Composite Game Benchmark","category":"agentic","score":65.59,"unit":"percent","family":"gertLabs","index":68.65608091562527,"aboveFit":false,"run":null,"source":"Gert Labs","sourceUrl":"https://gertlabs.com/rankings","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:finance_agent:overall","benchmarkName":"Finance Agent v1.1","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":64.373,"unit":"percent","family":"vals-ai:finance_agent:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/finance_agent","measuredAt":"2026-06-04","stderr":2.793,"costPerTest":0.802032,"latency":271.055},{"benchmarkId":"vals-ai:emb:overall","benchmarkName":"EMB","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":63.728,"unit":"percent","family":"vals-ai:emb:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/emb","measuredAt":"2026-10-20","stderr":3.106,"costPerTest":12.756352,"latency":2315.595},{"benchmarkId":"parseBenchLlamaIndex","benchmarkName":"ParseBench: document parsing for AI agents (LlamaIndex leaderboard)","category":"multimodal","score":63.34,"unit":"percent","family":"parseBenchLlamaIndex","index":62.09376700162484,"aboveFit":false,"run":null,"source":"Boyang Zhang, Sebastián G. Acosta, Preston Carlson, Sacha Bron, Pierre-Loïc Doulcet, Daniel B. Ospina and Simon Suo (LlamaIndex)","sourceUrl":"https://arxiv.org/abs/2604.08538","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:medcode:overall","benchmarkName":"MedCode","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":54.858,"unit":"percent","family":"vals-ai:medcode:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medcode","measuredAt":"2026-10-29","stderr":2.205,"costPerTest":0.226314,"latency":54.251},{"benchmarkId":"terminalBenchHard","benchmarkName":"Terminal-Bench Hard","category":"agentic","score":54.5,"unit":"percent","family":"terminalBenchHard","index":63.82106440473345,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/terminalbench-hard","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-simpleqa-verified:overall","benchmarkName":"SimpleQA Verified","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":51.7,"unit":"percent","family":"epoch-ai:epoch-simpleqa-verified:overall","index":66.9897162964348,"aboveFit":false,"run":"xhigh effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.5810153729833274,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:fabv2:overall","benchmarkName":"Finance Agent (v2)","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":51.509,"unit":"percent","family":"vals-ai:fabv2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/fabv2","measuredAt":"2026-10-29","stderr":0.49,"costPerTest":4.032269,"latency":360.437},{"benchmarkId":"livebench:livebench-agentic-coding:overall","benchmarkName":"LiveBench Agentic Coding","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":50.65633333333333,"unit":"percent","family":"livebench:livebench-agentic-coding:overall","index":61.403704166806016,"aboveFit":false,"run":"xhigh effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Agentic%20Coding","measuredAt":"2026-06-25","stderr":null,"costPerTest":2.638165277777778,"latency":null},{"benchmarkId":"vals-ai:ioi-v1:overall","benchmarkName":"IOI v1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":47.084,"unit":"percent","family":"vals-ai:ioi-v1:overall","index":63.56736576991991,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/ioi-v1","measuredAt":"2026-08-09","stderr":8.704,"costPerTest":10.486954,"latency":3081.905},{"benchmarkId":"omniscienceAccuracy","benchmarkName":"Artificial Analysis Omniscience Accuracy","category":"knowledge","score":44.7,"unit":"percent","family":"omniscienceAccuracy","index":67.02339295408586,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:code-migration:overall","benchmarkName":"Code Migration","category":"coding","domain":null,"eligible":true,"ranked":true,"score":43.878,"unit":"percent","family":"vals-ai:code-migration:overall","index":68.49889027606525,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/code-migration","measuredAt":"2026-10-29","stderr":4.22,"costPerTest":36.387681,"latency":4327.01},{"benchmarkId":"epoch-ai:epoch-frontiermath-2025-02-28-private:overall","benchmarkName":"FrontierMath-2025-02-28-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":43.79310344827586,"unit":"percent","family":"epoch-ai:epoch-frontiermath-2025-02-28-private:overall","index":68.57374527076256,"aboveFit":false,"run":"xhigh effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.918426387376174,"costPerTest":null,"latency":null},{"benchmarkId":"aaIfBench","benchmarkName":"Artificial Analysis IFBench","category":"instruction","score":43.6,"unit":"percent","family":"ifbench-family","index":29.76829219863123,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-chess-puzzles:overall","benchmarkName":"Chess Puzzles","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":41,"unit":"percent","family":"epoch-ai:epoch-chess-puzzles:overall","index":65.10429872629288,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":4.943110704237103,"costPerTest":null,"latency":null},{"benchmarkId":"tau2-bench:banking-knowledge:overall","benchmarkName":"τ²-bench Banking","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":40.20618556701031,"unit":"percent","family":"tau2-bench","index":29.46918382053746,"aboveFit":false,"run":"max effort · Sierra","source":"Sierra Research","sourceUrl":"https://github.com/sierra-research/tau2-bench","measuredAt":"2026-10-20","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"frontierCode","benchmarkName":"FrontierCode 1.1 Main","category":"coding","score":38.5,"unit":"percent","family":"frontierCode","index":64.29800634922704,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:legal_research:overall","benchmarkName":"Legal Research Bench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":38.462,"unit":"percent","family":"vals-ai:legal_research:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_research","measuredAt":"2026-11-09","stderr":3.381,"costPerTest":3.05609,"latency":511.955},{"benchmarkId":"epoch-ai:epoch-furniture-assembly:overall","benchmarkName":"Furniture Assembly","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":33.33333333333333,"unit":"percent","family":"epoch-ai:epoch-furniture-assembly:overall","index":62.20274316545983,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":6.137164119322156,"costPerTest":null,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":33.3,"unit":"percent","family":"hle","index":57.796292722805624,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tier-4-v2-private:overall","benchmarkName":"FrontierMath-Tier-4-v2-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":31.70731707317073,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tier-4-v2-private:overall","index":60.94452523945129,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":7.3576112824382225,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-mirrorcode:overall","benchmarkName":"MirrorCode","category":"coding","domain":null,"eligible":true,"ranked":true,"score":31.11111111111111,"unit":"percent","family":"epoch-ai:epoch-mirrorcode:overall","index":63.228748334419464,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":9.036473223803354,"costPerTest":null,"latency":null},{"benchmarkId":"postTrainBench11","benchmarkName":"PostTrainBench v1.1","category":"coding","score":28.56,"unit":"percent","family":"postTrainBench11","index":64.79886234611538,"aboveFit":false,"run":null,"source":"PostTrainBench team","sourceUrl":"https://posttrainbench.com/?version=v1.1","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-mystery-game-puzzles:overall","benchmarkName":"Mystery Game Puzzles","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":28.000000000000004,"unit":"percent","family":"epoch-ai:epoch-mystery-game-puzzles:overall","index":60.36461530981501,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":4.512608598542129,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:tax_agent_bench:overall","benchmarkName":"Tax Agent Bench","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":23.738,"unit":"percent","family":"vals-ai:tax_agent_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench","measuredAt":"2026-11-09","stderr":3.063,"costPerTest":1.274806,"latency":279.035},{"benchmarkId":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-private:overall","benchmarkName":"FrontierMath-Tier-4-2025-07-01-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":22.916666666666664,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-private:overall","index":66.4784191486778,"aboveFit":false,"run":"xhigh effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":6.130657275434798,"costPerTest":null,"latency":null},{"benchmarkId":"researchClawBench","benchmarkName":"ResearchClawBench","category":"agentic","score":20.7,"unit":"percent","family":"researchClawBench","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-ebr-bench:overall","benchmarkName":"EBR-bench","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":19.047619047619047,"unit":"percent","family":"epoch-ai:epoch-ebr-bench:overall","index":61.16825260153245,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"osWorld2","benchmarkName":"OSWorld 2.0","category":"agentic","score":13.9,"unit":"percent","family":"osWorld2","index":59.69135160197817,"aboveFit":false,"run":null,"source":"Mengqi Yuan, Zilong Zhou, Xinzhuang Xiong, Weiming Wu, Jiayang Sun, Jiamin Song, Kaiqian Cui, Bowen Wang, Haoyuan Wu, Yitong Li, Dunjie Lu, Haikong Lu, Qi Zhen, Xinyuan Wang, Jiaqi Deng, Yuhao Yang, Cheng Chen, Boyuan Zheng, Alex Su, Xiao Yu, Hao Zou, Saaket Agashe, Xing Han Lu, Manpreet Kaur, Zhengyang Qi, Vincent Sunn Chen, Frederic Sala, Dayiheng Liu, Junyang Lin, Zhou Yu, Yu Su, Siva Reddy, Xin Eric Wang, Peng Qi, Tianbao Xie, Tao Yu","sourceUrl":"https://arxiv.org/abs/2606.29537","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"apprenticeBench","benchmarkName":"ApprenticeBench: end-to-end computer use, continual learning, and long-horizon agency on a real accounts-payable job","category":"agentic","score":7,"unit":"percent","family":"apprenticeBench","index":61.757221677741256,"aboveFit":false,"run":null,"source":"NeoCognition","sourceUrl":"https://neocognition.io/blog/apprentice-bench/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":5.1,"unit":"percent","family":"critpt","index":46.72212092793723,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"arc-prize:v3-semi-private:overall","benchmarkName":"ARC-AGI-3 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":0.18,"unit":"percent","family":"arc-prize:v3-semi-private:overall","index":65.38206267316735,"aboveFit":false,"run":"high effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:programbench:overall","benchmarkName":"ProgramBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":0,"unit":"percent","family":"vals-ai:programbench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/programbench","measuredAt":"2026-11-09","stderr":0,"costPerTest":null,"latency":3246.698},{"benchmarkId":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-public:overall","benchmarkName":"FrontierMath-Tier-4-2025-07-01-Public","category":"math","domain":null,"eligible":false,"ranked":true,"score":0,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-public:overall","index":null,"aboveFit":false,"run":"xhigh effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":5,"outputPerMillion":25,"batch":{"inputPerMillion":2.5,"outputPerMillion":12.5},"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":47,"timeToFirstToken":1.047,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":{"elo":1483.1994040999593,"eloUpper":1486.9633224867503,"eloLower":1479.4354857131684,"votes":65369,"rank":18,"rankUpper":null,"rankLower":null,"arenaName":"Claude Opus 4.7","organization":"anthropic","updatedAt":"2026-11-09"},"quality":{"coveragePercent":95,"benchmarkCount":43,"categoryCount":7,"confidence":"high","state":"available"},"hosts":[{"host":"gmi-cloud","name":"GMI Cloud","kind":"host","url":"https://docs.gmicloud.ai/inference-engine/api-reference/llm-api-reference","inputPerMillion":4.5,"outputPerMillion":22.5,"cacheReadPerMillion":0.45,"contextTokens":409600,"maxOutputTokens":128000,"deprecated":false},{"host":"amazon-bedrock","name":"Amazon Bedrock","kind":"cloud","url":"https://docs.aws.amazon.com/bedrock/latest/userguide/models-supported.html","inputPerMillion":5,"outputPerMillion":25,"cacheReadPerMillion":0.5,"contextTokens":1000000,"maxOutputTokens":128000,"deprecated":false},{"host":"databricks","name":"Databricks","kind":"cloud","url":"https://docs.databricks.com/aws/en/machine-learning/foundation-models/","inputPerMillion":5,"outputPerMillion":25,"cacheReadPerMillion":0.5,"contextTokens":1000000,"maxOutputTokens":128000,"deprecated":false},{"host":"digitalocean","name":"DigitalOcean","kind":"cloud","url":"https://docs.digitalocean.com/products/gradient-ai-platform/details/models/","inputPerMillion":5,"outputPerMillion":25,"cacheReadPerMillion":0.5,"contextTokens":200000,"maxOutputTokens":8192,"deprecated":false},{"host":"google-vertex","name":"Google Vertex AI","kind":"cloud","url":"https://cloud.google.com/vertex-ai/generative-ai/docs/models","inputPerMillion":5,"outputPerMillion":25,"cacheReadPerMillion":0.5,"contextTokens":1000000,"maxOutputTokens":128000,"deprecated":false},{"host":"azure","name":"Microsoft Azure","kind":"cloud","url":"https://learn.microsoft.com/en-us/azure/ai-services/openai/concepts/models","inputPerMillion":5,"outputPerMillion":25,"cacheReadPerMillion":0.5,"contextTokens":1000000,"maxOutputTokens":128000,"deprecated":false},{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":5,"outputPerMillion":25,"cacheReadPerMillion":0.5,"contextTokens":1000000,"maxOutputTokens":128000,"deprecated":false},{"host":"sap-ai-core","name":"SAP AI Core","kind":"cloud","url":"https://help.sap.com/docs/sap-ai-core","inputPerMillion":5,"outputPerMillion":25,"cacheReadPerMillion":0.5,"contextTokens":1000000,"maxOutputTokens":128000,"deprecated":false},{"host":"snowflake","name":"Snowflake Cortex","kind":"cloud","url":"https://docs.snowflake.com/en/user-guide/snowflake-cortex/cortex-rest-api","inputPerMillion":5,"outputPerMillion":25,"cacheReadPerMillion":0.5,"contextTokens":1000000,"maxOutputTokens":128000,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:livebench","name":"LiveBench","url":"https://livebench.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:sierra-research","name":"Sierra Research","url":"https://github.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:arc-prize-foundation","name":"ARC Prize Foundation","url":"https://arcprize.org","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gpt-5-3-codex","providerId":"openai","slug":"gpt-5-3-codex","name":"GPT-5.3 Codex","provider":"OpenAI","description":"GPT-5.3 Codex is a reasoning model from OpenAI. 18 benchmarks count toward its index, in 6 categories.","logoUrl":null,"releaseDate":"2026-02-05","contextWindowTokens":400000,"modalities":["text","image","file"],"openWeights":false,"details":{"apiId":"openai/gpt-5.3-codex","maxOutputTokens":128000,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.175,"weightsUrl":null},"reasoning":true,"overallScore":61.98799624243705,"overallScoreRange":{"low":56.67669069184156,"high":67.29930179303254},"badges":[],"capabilityScores":{"agentic":60.62993474123783,"coding":61.76204861321024,"reasoning":67.00705065272015,"multimodal":57.31367783969014,"knowledge":64.0784932350448,"multilingual":null,"instruction":59.48572975020944,"math":null},"capabilityCounts":{"knowledge":2,"coding":6,"agentic":6,"reasoning":2,"multimodal":1,"instruction":1},"benchmarks":[{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":91.5,"unit":"percent","family":"gpqa-diamond","index":61.13040049300986,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:lcb:overall","benchmarkName":"LiveCodeBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":87.313,"unit":"percent","family":"livecodebench","index":61.79096243141494,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","measuredAt":"2026-09-01","stderr":0.968,"costPerTest":0.091585,"latency":132.631},{"benchmarkId":"tau2Bench","benchmarkName":"τ²-Bench Tool-Agent-User Evaluation","category":"agentic","score":86,"unit":"percent","family":"tau2-bench","index":59.856727770596976,"aboveFit":false,"run":null,"source":"Victor Barres, Honghua Dong, Soham Ray, Xujie Si, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2506.07982","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"sweVerified","benchmarkName":"Software Engineering Benchmark Verified","category":"coding","score":85,"unit":"percent","family":"swe-bench-verified","index":63.92032164108546,"aboveFit":false,"run":null,"source":"Carlos E. Jimenez, John Yang, Alexander Wettig, Shunyu Yao, Kexin Pei, Ofir Press, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2310.06770","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":83.3,"unit":"percent","family":"lcr","index":64.4485572404562,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaMmmuPro","benchmarkName":"Artificial Analysis MMMU-Pro","category":"multimodal","score":78.5,"unit":"percent","family":"mmmu-pro","index":57.31367783969014,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmmu-pro","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:swebench:overall","benchmarkName":"SWE-bench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":78,"unit":"percent","family":"swe-bench-verified","index":58.75741859659962,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/swebench","measuredAt":"2026-09-01","stderr":1.854,"costPerTest":0.455384,"latency":246.527},{"benchmarkId":"terminalBench2","benchmarkName":"Terminal-Bench 2.0","category":"agentic","score":77.3,"unit":"percent","family":"terminalBench2","index":68.8846454620584,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaIfBench","benchmarkName":"Artificial Analysis IFBench","category":"instruction","score":75.4,"unit":"percent","family":"ifbench-family","index":59.48572975020944,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-swe-bench-verified:overall","benchmarkName":"SWE-Bench verified","category":"coding","domain":null,"eligible":true,"ranked":true,"score":74.79338842975206,"unit":"percent","family":"swe-bench-verified","index":56.392357933954386,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.9756739501473306,"costPerTest":null,"latency":null},{"benchmarkId":"osWorldVerified","benchmarkName":"OSWorld-Verified","category":"agentic","score":64.7,"unit":"percent","family":"osWorldVerified","index":52.71054247034103,"aboveFit":false,"run":null,"source":"Tianbao Xie, Danyang Zhang, Jixuan Chen, Xiaochuan Li, Siheng Zhao, Ruisheng Cao, Toh Jing Hua, Zhoujun Cheng, Dongchan Shin, Fangyu Lei, Yitao Liu, Yiheng Xu, Shuyan Zhou, Silvio Savarese, Caiming Xiong, Victor Zhong, Tao Yu","sourceUrl":"https://os-world.github.io/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:terminal-bench-2:overall","benchmarkName":"Terminal-Bench 2.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":64.045,"unit":"percent","family":"vals-ai:terminal-bench-2:overall","index":65.11347988130507,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2","measuredAt":"2026-06-04","stderr":5.115,"costPerTest":0.345045,"latency":506.283},{"benchmarkId":"vals-ai:vibe-code:overall","benchmarkName":"Vibe Code Bench v1.1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":61.767,"unit":"percent","family":"vals-ai:vibe-code:overall","index":64.53602662331339,"aboveFit":false,"run":"OpenHands","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","measuredAt":"2026-11-09","stderr":4.714,"costPerTest":11.914536,"latency":4548.115},{"benchmarkId":"sweRebench","benchmarkName":"SWE-Rebench","category":"coding","score":58.2,"unit":"percent","family":"sweRebench","index":null,"aboveFit":false,"run":null,"source":"Nebius","sourceUrl":"https://swe-rebench.com","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gertLabs","benchmarkName":"Gert Labs Composite Game Benchmark","category":"agentic","score":57.47,"unit":"percent","family":"gertLabs","index":62.08118060963447,"aboveFit":false,"run":null,"source":"Gert Labs","sourceUrl":"https://gertlabs.com/rankings","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"swePro","benchmarkName":"SWE-bench Pro","category":"coding","score":56.8,"unit":"percent","family":"swePro","index":57.21083060539095,"aboveFit":false,"run":null,"source":"Xiang Deng, Jeff Da, Edwin Pan, Yannis Yiming He, Charles Ide, Kanak Garg, Niklas Lauffer, Andrew Park, Nitin Pasari, Chetan Rane, Karmini Sampath, Maya Krishnan, Srivatsa Kundurthy, Sean Hendryx, Zifan Wang, Chen Bo Calvin Zhang, Noah Jacobson, Bing Liu, Brad Kenstler","sourceUrl":"https://arxiv.org/abs/2509.16941","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:ioi:overall","benchmarkName":"IOI","category":"coding","domain":null,"eligible":true,"ranked":true,"score":53.833,"unit":"percent","family":"vals-ai:ioi:overall","index":66.32674827548502,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/ioi","measuredAt":"2026-10-29","stderr":6.312,"costPerTest":2.68083,"latency":1791.685},{"benchmarkId":"vals-ai:ioi-v1:overall","benchmarkName":"IOI v1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":43.834,"unit":"percent","family":"vals-ai:ioi-v1:overall","index":61.950305147057506,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/ioi-v1","measuredAt":"2026-08-09","stderr":7.81,"costPerTest":2.020025,"latency":3574.257},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":42.5,"unit":"percent","family":"hle","index":67.02658597707973,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"jobBench","benchmarkName":"JobBench","category":"agentic","score":33.7,"unit":"percent","family":"jobBench","index":55.133032253491066,"aboveFit":false,"run":null,"source":"Yuetai Li, Yichen Feng, Zhangchen Xu, Zixian Ma, Kaiyuan Zheng, Fengqing Jiang, Xinghua Sun, Rulin Shao, Zichen Chen, Yue Huang, Xinyang Han, Brian Lee, Kayla Xu, Shenglai Zeng, Hang Hua, Xiangliang Zhang, Basel Alomair, Ranjay Krishna, Luke Zettlemoyer, Pang Wei Koh, Bhaskar Ramasubramanian, Luyao Niu, Xiang Yue, Radha Poovendran","sourceUrl":"https://arxiv.org/abs/2605.26329","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":16.9,"unit":"percent","family":"critpt","index":69.5655440649841,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":1.75,"outputPerMillion":14,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":52,"timeToFirstToken":2.7815,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":85,"benchmarkCount":18,"categoryCount":6,"confidence":"medium","state":"available"},"hosts":[{"host":"digitalocean","name":"DigitalOcean","kind":"cloud","url":"https://docs.digitalocean.com/products/gradient-ai-platform/details/models/","inputPerMillion":1.75,"outputPerMillion":14,"cacheReadPerMillion":0.175,"contextTokens":400000,"maxOutputTokens":128000,"deprecated":false},{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":1.75,"outputPerMillion":14,"cacheReadPerMillion":0.175,"contextTokens":400000,"maxOutputTokens":128000,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"deepseek-v4-flash-max","providerId":"deepseek","slug":"deepseek-v4-flash-0731","name":"DeepSeek V4 Flash 0731","provider":"DeepSeek","description":"DeepSeek V4 Flash 0731 is a reasoning model from DeepSeek in the DeepSeek V4 Flash family. 51 benchmarks count toward its index, in 6 categories.","logoUrl":null,"releaseDate":"2026-07-31","contextWindowTokens":1048576,"modalities":["text"],"openWeights":true,"details":{"apiId":"deepseek/deepseek-v4-flash-0731","maxOutputTokens":943718,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.035,"weightsUrl":"https://huggingface.co/deepseek-ai/DeepSeek-V4-Flash-0731"},"reasoning":true,"overallScore":61.919848840356714,"overallScoreRange":{"low":58.15559594448342,"high":65.68410173623},"badges":[],"capabilityScores":{"agentic":59.924490560142374,"coding":62.82663295886641,"reasoning":65.52027020936389,"multimodal":null,"knowledge":58.59823482072497,"multilingual":null,"instruction":59.15569416498775,"math":61.494573511322045},"capabilityCounts":{"math":7,"knowledge":6,"reasoning":7,"coding":13,"agentic":17,"instruction":1},"benchmarks":[{"benchmarkId":"hmmtFeb2026","benchmarkName":"Harvard-MIT Mathematics Tournament February 2026","category":"math","score":94.8,"unit":"percent","family":"hmmtFeb2026","index":59.74353560074094,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"arc-prize:v1-public-eval:overall","benchmarkName":"ARC-AGI-1 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":94.75,"unit":"percent","family":"arc-prize:v1-public-eval:overall","index":null,"aboveFit":false,"run":"max effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.015125951425000006,"latency":null},{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":94.44444444444444,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":63.86630630691175,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.0854366486931895,"costPerTest":null,"latency":null},{"benchmarkId":"liveCodeBenchPass1Cot","benchmarkName":"LiveCodeBench Pass@1 with Chain-of-Thought","category":"coding","score":91.6,"unit":"percent","family":"liveCodeBenchPass1Cot","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":91.03535353535354,"unit":"percent","family":"gpqa-diamond","index":60.70423945213804,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.7655466909172535,"costPerTest":null,"latency":null},{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":90.8,"unit":"percent","family":"gpqa-diamond","index":60.48837962056601,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:gpqa:overall","benchmarkName":"GPQA Diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":89.899,"unit":"percent","family":"gpqa-diamond","index":59.66200704046329,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","measuredAt":"2026-09-01","stderr":1.694,"costPerTest":0.012875,"latency":79.956},{"benchmarkId":"arc-prize:v1-semi-private:overall","benchmarkName":"ARC-AGI-1 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":89,"unit":"percent","family":"arc-prize:v1-semi-private:overall","index":67.90387202491206,"aboveFit":false,"run":"max effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.020511353200000002,"latency":null},{"benchmarkId":"vals-ai:swebench:overall","benchmarkName":"SWE-bench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":88.8,"unit":"percent","family":"swe-bench-verified","index":66.7230404366635,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/swebench","measuredAt":"2026-09-01","stderr":1.412,"costPerTest":0.009884,"latency":202.391},{"benchmarkId":"vulcanBench","benchmarkName":"VulcanBench v3","category":"coding","score":88.4,"unit":"percent","family":"vulcanBench","index":70.2915891578601,"aboveFit":false,"run":null,"source":"VulcanBench contributors","sourceUrl":"https://github.com/morganlinton/VulcanBench/tree/main","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"imoAnswerBench","benchmarkName":"IMOAnswerBench","category":"math","score":88.4,"unit":"percent","family":"imoAnswerBench","index":60.373457453347946,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gpqa","benchmarkName":"Graduate-Level Google-Proof Q&A","category":"knowledge","score":88.1,"unit":"percent","family":"gpqa-diamond","index":58.01201339828258,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gpqaDiamond","benchmarkName":"GPQA Diamond","category":"knowledge","score":88.1,"unit":"percent","family":"gpqa-diamond","index":58.01201339828258,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:lcb:overall","benchmarkName":"LiveCodeBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":87.264,"unit":"percent","family":"livecodebench","index":61.7503051111938,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","measuredAt":"2026-09-01","stderr":0.968,"costPerTest":0.020251,"latency":126.61},{"benchmarkId":"livebench:livebench-math:overall","benchmarkName":"LiveBench Mathematics","category":"math","domain":null,"eligible":true,"ranked":true,"score":86.78975,"unit":"percent","family":"livebench:livebench-math:overall","index":60.735373305194884,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Mathematics","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.004694986807387862,"latency":null},{"benchmarkId":"livebench:livebench-reasoning:overall","benchmarkName":"LiveBench Reasoning","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":86.6345,"unit":"percent","family":"livebench","index":73.84899787457826,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Reasoning","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.006322772277227723,"latency":null},{"benchmarkId":"vals-ai:mmlu_pro:overall","benchmarkName":"MMLU Pro","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":86.206,"unit":"percent","family":"mmlu-pro","index":53.08996983399272,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","measuredAt":"2026-09-01","stderr":0.339,"costPerTest":0.00265,"latency":17.188},{"benchmarkId":"mmluPro","benchmarkName":"Massive Multitask Language Understanding Professional","category":"knowledge","score":86.2,"unit":"percent","family":"mmlu-pro","index":53.08241055217426,"aboveFit":false,"run":null,"source":"Yubo Wang, Xueguang Ma, Ge Zhang, Yuansheng Ni, Abhranil Chandra, Shiguang Guo, Weiming Ren, Aaran Arulraj, Xuan He, Ziyan Jiang, Tianle Li, Max Ku, Kai Wang, Alex Zhuang, Rongqi Fan, Xiang Yue, Wenhu Chen","sourceUrl":"https://arxiv.org/abs/2406.01574","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"apexShortlist","benchmarkName":"Apex Shortlist","category":"math","score":85.7,"unit":"percent","family":"apexShortlist","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"terminalBench21","benchmarkName":"Terminal-Bench 2.1","category":"agentic","score":82.7,"unit":"percent","family":"terminal-bench-2-1","index":69.50865984927026,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:medscribe:overall","benchmarkName":"MedScribe","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":80.363,"unit":"percent","family":"vals-ai:medscribe:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medscribe","measuredAt":"2026-10-29","stderr":1.973,"costPerTest":0.014247,"latency":77.552},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":79.7,"unit":"percent","family":"lcr","index":61.97866734834783,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-data-analysis:overall","benchmarkName":"LiveBench Data Analysis","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":79.32733333333333,"unit":"percent","family":"livebench","index":64.38304575906933,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Data%20Analysis","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.0077153333333333345,"latency":null},{"benchmarkId":"livebench:livebench-research:overall","benchmarkName":"LiveBench Research","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":79.2,"unit":"percent","family":"livebench-2-agentic","index":76.51089430007039,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Research","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.21117727272727274,"latency":null},{"benchmarkId":"livebench:livebench-language:overall","benchmarkName":"LiveBench Language","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":79.175,"unit":"percent","family":"livebench:livebench-language:overall","index":64.09059316754556,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Language","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.0036213333333333336,"latency":null},{"benchmarkId":"sweVerified","benchmarkName":"Software Engineering Benchmark Verified","category":"coding","score":79,"unit":"percent","family":"swe-bench-verified","index":59.49497617438331,"aboveFit":false,"run":null,"source":"Carlos E. Jimenez, John Yang, Alexander Wettig, Shunyu Yao, Kexin Pei, Ofir Press, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2310.06770","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"chineseSimpleQa","benchmarkName":"Chinese-SimpleQA","category":"knowledge","score":78.9,"unit":"percent","family":"chineseSimpleQa","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mrcr1m","benchmarkName":"MRCR 1M","category":"reasoning","score":78.7,"unit":"percent","family":"mrcr1m","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:legal_bench:overall","benchmarkName":"LegalBench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":77.714,"unit":"percent","family":"vals-ai:legal_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","measuredAt":"2026-10-13","stderr":0.502,"costPerTest":0.000523,"latency":3.947},{"benchmarkId":"livebench:livebench-coding-2:overall","benchmarkName":"LiveBench Coding 2.0","category":"coding","domain":null,"eligible":true,"ranked":true,"score":77,"unit":"percent","family":"livebench:livebench-coding-2:overall","index":63.9449497309675,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Coding","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.12239655172413794,"latency":null},{"benchmarkId":"cyberGym","benchmarkName":"CyberGym","category":"agentic","score":76.7,"unit":"percent","family":"cyberGym","index":64.66619726276546,"aboveFit":false,"run":null,"source":"Zhun Wang, Tianneng Shi, Jingxuan He, Matthew Cai, Jialin Zhang, Dawn Song","sourceUrl":"https://www.cybergym.io/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-coding:overall","benchmarkName":"LiveBench Coding","category":"coding","domain":null,"eligible":true,"ranked":true,"score":74.9845,"unit":"percent","family":"livebench:livebench-coding:overall","index":60.28314819899794,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Coding","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.004796581196581197,"latency":null},{"benchmarkId":"vals-ai:vibe-code:overall","benchmarkName":"Vibe Code Bench v1.1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":74.736,"unit":"percent","family":"vals-ai:vibe-code:overall","index":69.21544558158514,"aboveFit":false,"run":"OpenHands","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","measuredAt":"2026-11-09","stderr":3.442,"costPerTest":0.905592,"latency":2241.572},{"benchmarkId":"browseComp","benchmarkName":"BrowseComp","category":"agentic","score":73.2,"unit":"percent","family":"browseComp","index":57.7384438497478,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:cyber:overall","benchmarkName":"CyberBench v1.1","category":null,"domain":"Other","eligible":false,"ranked":true,"score":72.084,"unit":"percent","family":"vals-ai:cyber:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/cyber","measuredAt":"2026-11-05","stderr":5.408,"costPerTest":0.19286,"latency":927.815},{"benchmarkId":"livebench:livebench-design:overall","benchmarkName":"LiveBench Design","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":71.7,"unit":"percent","family":"livebench-2-agentic","index":71.02591108816696,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Design","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.10078799999999999,"latency":null},{"benchmarkId":"vals-ai:tax_eval_v2:overall","benchmarkName":"TaxEval v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":70.687,"unit":"percent","family":"vals-ai:tax_eval_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","measuredAt":"2026-09-01","stderr":0.897,"costPerTest":0.009755,"latency":72.203},{"benchmarkId":"toolathlonVerified","benchmarkName":"Toolathlon-Verified","category":"agentic","score":70.3,"unit":"percent","family":"toolathlonVerified","index":65.80714764876542,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mcpAtlas","benchmarkName":"MCP Atlas","category":"agentic","score":69,"unit":"percent","family":"mcpAtlas","index":57.526579440561946,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"dsBenchFullStack","benchmarkName":"DeepSeek DSBench FullStack","category":"coding","score":68.7,"unit":"percent","family":"dsBenchFullStack","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:terminal-bench-2-1:overall","benchmarkName":"Terminal-Bench 2.1","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":67.041,"unit":"percent","family":"terminal-bench-2-1","index":60.35040969698791,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2-1","measuredAt":"2026-10-01","stderr":1.35,"costPerTest":0.021068,"latency":683.03},{"benchmarkId":"livebench:livebench-instruction-following:overall","benchmarkName":"LiveBench Instruction Following","category":"instruction","domain":null,"eligible":true,"ranked":true,"score":65.51675,"unit":"percent","family":"livebench:livebench-instruction-following:overall","index":59.15569416498775,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=IF","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.004507,"latency":null},{"benchmarkId":"vals-ai:biomysterybench:overall","benchmarkName":"BioMysteryBench","category":null,"domain":"Science","eligible":false,"ranked":true,"score":64.444,"unit":"percent","family":"vals-ai:biomysterybench:overall","index":null,"aboveFit":false,"run":"Terminus 2","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/biomysterybench","measuredAt":"2026-11-09","stderr":1.283,"costPerTest":0.038768,"latency":1622.715},{"benchmarkId":"arc-prize:v2-public-eval:overall","benchmarkName":"ARC-AGI-2 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":63.88888888888889,"unit":"percent","family":"arc-prize:v2-public-eval:overall","index":null,"aboveFit":false,"run":"max effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.04386672874999998,"latency":null},{"benchmarkId":"vals-ai:corp_fin_v2:overall","benchmarkName":"CorpFin v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":61.849,"unit":"percent","family":"vals-ai:corp_fin_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","measuredAt":"2026-08-12","stderr":0.957,"costPerTest":0.011801,"latency":16.477},{"benchmarkId":"arc-prize:v2-semi-private:overall","benchmarkName":"ARC-AGI-2 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":61.38888888888888,"unit":"percent","family":"arc-prize:v2-semi-private:overall","index":67.37428454550101,"aboveFit":false,"run":"max effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.04163034416666667,"latency":null},{"benchmarkId":"corpusQa1m","benchmarkName":"CorpusQA 1M","category":"reasoning","score":60.5,"unit":"percent","family":"corpusQa1m","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"dsBenchHard","benchmarkName":"DeepSeek DSBench Hard","category":"coding","score":59.6,"unit":"percent","family":"dsBenchHard","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tiers-1-3-v2-private:overall","benchmarkName":"FrontierMath-Tiers-1-3-v2-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":57.54385964912281,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tiers-1-3-v2-private:overall","index":61.18300530637188,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.9329899789943816,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:emb:overall","benchmarkName":"EMB","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":56.977,"unit":"percent","family":"vals-ai:emb:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/emb","measuredAt":"2026-10-20","stderr":3.391,"costPerTest":0.383709,"latency":1574.097},{"benchmarkId":"terminalBench2","benchmarkName":"Terminal-Bench 2.0","category":"agentic","score":56.9,"unit":"percent","family":"terminalBench2","index":54.37319725449357,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:proof_bench:overall","benchmarkName":"ProofBench v1.1","category":"math","domain":null,"eligible":true,"ranked":true,"score":56,"unit":"percent","family":"vals-ai:proof_bench:overall","index":66.49032384620499,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/proof_bench","measuredAt":"2026-11-09","stderr":4.989,"costPerTest":0.156045,"latency":933.981},{"benchmarkId":"deepSwe","benchmarkName":"DeepSWE","category":"agentic","score":54.4,"unit":"percent","family":"deepSwe","index":60.62400200213729,"aboveFit":false,"run":null,"source":"Datacurve AI","sourceUrl":"https://deepswe.datacurve.ai/blog","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"nl2Repo","benchmarkName":"NL2Repo","category":"coding","score":54.2,"unit":"percent","family":"nl2Repo","index":64.99595468875188,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"openHarmonyBench","benchmarkName":"OpenHarmony Bench v1.0","category":"coding","score":53.8,"unit":"percent","family":"openHarmonyBench","index":61.13773698381015,"aboveFit":false,"run":null,"source":"OpenHarmony Bench authors","sourceUrl":"https://arxiv.org/abs/2608.16022","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"swePro","benchmarkName":"SWE-bench Pro","category":"coding","score":52.6,"unit":"percent","family":"swePro","index":53.63406868440143,"aboveFit":false,"run":null,"source":"Xiang Deng, Jeff Da, Edwin Pan, Yannis Yiming He, Charles Ide, Kanak Garg, Niklas Lauffer, Andrew Park, Nitin Pasari, Chetan Rane, Karmini Sampath, Maya Krishnan, Srivatsa Kundurthy, Sean Hendryx, Zifan Wang, Chen Bo Calvin Zhang, Noah Jacobson, Bing Liu, Brad Kenstler","sourceUrl":"https://arxiv.org/abs/2509.16941","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:skillsbench:overall","benchmarkName":"SkillsBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":50.668,"unit":"percent","family":"vals-ai:skillsbench:overall","index":62.85587943836708,"aboveFit":false,"run":"OpenHands","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/skillsbench","measuredAt":"2026-10-29","stderr":4.429,"costPerTest":0.12607,"latency":411.963},{"benchmarkId":"aaSciCode","benchmarkName":"Artificial Analysis SciCode","category":"coding","score":50.3,"unit":"percent","family":"scicode-family","index":61.415463107172336,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/scicode","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:fabv2:overall","benchmarkName":"Finance Agent (v2)","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":49.517,"unit":"percent","family":"vals-ai:fabv2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/fabv2","measuredAt":"2026-10-29","stderr":0.465,"costPerTest":0.276154,"latency":490.245},{"benchmarkId":"toolathlon","benchmarkName":"Toolathlon","category":"agentic","score":47.8,"unit":"percent","family":"toolathlon","index":60.89930903199564,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-agentic-coding:overall","benchmarkName":"LiveBench Agentic Coding","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":46.76766666666666,"unit":"percent","family":"livebench:livebench-agentic-coding:overall","index":57.39338846400117,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Agentic%20Coding","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.693175,"latency":null},{"benchmarkId":"gdpvalAaNormalized","benchmarkName":"GDPval-AA normalized","category":"agentic","score":46.7,"unit":"percent","family":"gdpvalAaNormalized","index":67.42318968820135,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hleWithTools","benchmarkName":"Humanity's Last Exam with tools","category":"agentic","score":45.1,"unit":"percent","family":"hleWithTools","index":55.86624371476035,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:medcode:overall","benchmarkName":"MedCode","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":41.415,"unit":"percent","family":"vals-ai:medcode:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medcode","measuredAt":"2026-10-29","stderr":2.15,"costPerTest":0.019659,"latency":127},{"benchmarkId":"omniscienceAccuracy","benchmarkName":"Artificial Analysis Omniscience Accuracy","category":"knowledge","score":40.4,"unit":"percent","family":"omniscienceAccuracy","index":61.60429222262985,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaTau3Banking","benchmarkName":"Artificial Analysis Tau3-Banking","category":"agentic","score":39.4,"unit":"percent","family":"aaTau3Banking","index":63.26340421009242,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/tau3-banking","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:code-migration:overall","benchmarkName":"Code Migration","category":"coding","domain":null,"eligible":true,"ranked":true,"score":38.631,"unit":"percent","family":"vals-ai:code-migration:overall","index":65.71147350441397,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/code-migration","measuredAt":"2026-10-29","stderr":4.382,"costPerTest":3.605899,"latency":7700.455},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":38.6,"unit":"percent","family":"hle","index":63.113744271463545,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-long-horizon:overall","benchmarkName":"LiveBench Long Agentic","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":36.6,"unit":"percent","family":"livebench-2-agentic","index":45.35618965645892,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Long%20Agentic","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.9027000000000001,"latency":null},{"benchmarkId":"hle","benchmarkName":"Humanity's Last Exam","category":"knowledge","score":34.8,"unit":"percent","family":"hle","index":59.30123184035032,"aboveFit":false,"run":null,"source":"Center for AI Safety, Scale AI, and thousands of expert contributors","sourceUrl":"https://lastexam.ai/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-mystery-game-puzzles:overall","benchmarkName":"Mystery Game Puzzles","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":34,"unit":"percent","family":"epoch-ai:epoch-mystery-game-puzzles:overall","index":65.01110469004604,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":4.760952285695233,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-simpleqa-verified:overall","benchmarkName":"SimpleQA Verified","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":33.633633633633636,"unit":"percent","family":"epoch-ai:epoch-simpleqa-verified:overall","index":51.93883824472067,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.495531772489948,"costPerTest":null,"latency":null},{"benchmarkId":"apex","benchmarkName":"Apex","category":"math","score":33,"unit":"percent","family":"apex","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:ioi:overall","benchmarkName":"IOI","category":"coding","domain":null,"eligible":true,"ranked":true,"score":32.722,"unit":"percent","family":"vals-ai:ioi:overall","index":58.40120597221862,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/ioi","measuredAt":"2026-10-29","stderr":2.35,"costPerTest":0.373112,"latency":1663.091},{"benchmarkId":"epoch-ai:epoch-chess-puzzles:overall","benchmarkName":"Chess Puzzles","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":32,"unit":"percent","family":"epoch-ai:epoch-chess-puzzles:overall","index":58.27316196994201,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":4.688261722621506,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:legal_research:overall","benchmarkName":"Legal Research Bench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":30.288,"unit":"percent","family":"vals-ai:legal_research:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_research","measuredAt":"2026-11-09","stderr":3.194,"costPerTest":0.275172,"latency":1266.744},{"benchmarkId":"vals-ai:tax_agent_bench:overall","benchmarkName":"Tax Agent Bench","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":28.069,"unit":"percent","family":"vals-ai:tax_agent_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench","measuredAt":"2026-11-09","stderr":3.158,"costPerTest":0.156388,"latency":337.343},{"benchmarkId":"agentsLastExam","benchmarkName":"Agents' Last Exam","category":"agentic","score":25.2,"unit":"percent","family":"agentsLastExam","index":61.97866734834783,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"automationBench","benchmarkName":"AutomationBench","category":"agentic","score":25.1,"unit":"percent","family":"automationBench","index":29.833133425726295,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tier-4-v2-private:overall","benchmarkName":"FrontierMath-Tier-4-v2-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":24.390243902439025,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tier-4-v2-private:overall","index":58.070012760481895,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":6.789956540036613,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:terminal-bench-4:overall","benchmarkName":"Terminal-Bench 4.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":18.687,"unit":"percent","family":"terminal-bench-4","index":63.389322971179894,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-4","measuredAt":"2026-11-09","stderr":3.535,"costPerTest":0.766472,"latency":6368.24},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":16.6,"unit":"percent","family":"critpt","index":68.98477906997445,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:terminal-bench-science:overall","benchmarkName":"Terminal-Bench-Science","category":null,"domain":"Science","eligible":false,"ranked":true,"score":4.286,"unit":"percent","family":"vals-ai:terminal-bench-science:overall","index":null,"aboveFit":false,"run":"Mini-SWE-agent","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-science","measuredAt":"2026-11-09","stderr":2.438,"costPerTest":1.095543,"latency":5653.642},{"benchmarkId":"mazeBench","benchmarkName":"MazeBench: visual spatial reasoning in a 3D open world","category":"agentic","score":0,"unit":"percent","family":"mazeBench","index":61.97866734834783,"aboveFit":false,"run":null,"source":"Jonathan Pappas and David Pappas","sourceUrl":"https://mazebench.com/blog?post=maze-bench-results","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:programbench:overall","benchmarkName":"ProgramBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":0,"unit":"percent","family":"vals-ai:programbench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/programbench","measuredAt":"2026-11-09","stderr":0,"costPerTest":1.285061,"latency":20327.596}],"price":{"inputPerMillion":0.0434,"outputPerMillion":0.3,"batch":{"inputPerMillion":0.048,"outputPerMillion":0.144},"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":61,"timeToFirstToken":0.5255,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":85,"benchmarkCount":51,"categoryCount":6,"confidence":"medium","state":"available"},"hosts":[{"host":"deepinfra","name":"Deep Infra","kind":"host","url":"https://deepinfra.com/models","inputPerMillion":0.06,"outputPerMillion":0.18,"cacheReadPerMillion":0.015,"contextTokens":1048576,"maxOutputTokens":384000,"deprecated":false},{"host":"vultr","name":"Vultr","kind":"cloud","url":"https://api.vultrinference.com/","inputPerMillion":0.1,"outputPerMillion":0.25,"cacheReadPerMillion":null,"contextTokens":1048576,"maxOutputTokens":384000,"deprecated":true},{"host":"baseten","name":"Baseten","kind":"host","url":"https://docs.baseten.co/inference/model-apis/overview","inputPerMillion":0.13,"outputPerMillion":0.26,"cacheReadPerMillion":0.028,"contextTokens":1048576,"maxOutputTokens":384000,"deprecated":false},{"host":"coreweave","name":"CoreWeave","kind":"host","url":"https://docs.wandb.ai/inference","inputPerMillion":0.13,"outputPerMillion":0.27,"cacheReadPerMillion":0.05,"contextTokens":262144,"maxOutputTokens":262144,"deprecated":false},{"host":"arcee-ai","name":"Arcee AI","kind":"host","url":"https://docs.arcee.ai","inputPerMillion":0.14,"outputPerMillion":0.28,"cacheReadPerMillion":0.028,"contextTokens":1048576,"maxOutputTokens":384000,"deprecated":false},{"host":"digitalocean","name":"DigitalOcean","kind":"cloud","url":"https://docs.digitalocean.com/products/gradient-ai-platform/details/models/","inputPerMillion":0.14,"outputPerMillion":0.28,"cacheReadPerMillion":0.028,"contextTokens":1048576,"maxOutputTokens":1048576,"deprecated":false},{"host":"together-ai","name":"Together AI","kind":"host","url":"https://docs.together.ai/docs/serverless-models","inputPerMillion":0.14,"outputPerMillion":0.28,"cacheReadPerMillion":0.03,"contextTokens":1048576,"maxOutputTokens":384000,"deprecated":true},{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":0.0434,"outputPerMillion":0.3,"cacheReadPerMillion":0.035,"contextTokens":1048576,"maxOutputTokens":943718,"deprecated":false},{"host":"siliconflow","name":"SiliconFlow","kind":"host","url":"https://cloud.siliconflow.com/models","inputPerMillion":0.22,"outputPerMillion":0.66,"cacheReadPerMillion":0.014,"contextTokens":1000000,"maxOutputTokens":384000,"deprecated":false},{"host":"scaleway","name":"Scaleway","kind":"cloud","url":"https://www.scaleway.com/en/docs/generative-apis/","inputPerMillion":0.468,"outputPerMillion":0.936,"cacheReadPerMillion":0.0936,"contextTokens":256000,"maxOutputTokens":32768,"deprecated":false},{"host":"alibaba-cloud","name":"Alibaba Cloud","kind":"host","url":"https://www.alibabacloud.com/help/en/model-studio/models","inputPerMillion":0.44,"outputPerMillion":1.32,"cacheReadPerMillion":0.088,"contextTokens":1000000,"maxOutputTokens":384000,"deprecated":false},{"host":"chutes","name":"Chutes","kind":"host","url":"https://llm.chutes.ai/v1/models","inputPerMillion":0.44,"outputPerMillion":1.32,"cacheReadPerMillion":0.044,"contextTokens":1048576,"maxOutputTokens":131072,"deprecated":false},{"host":"cloudflare","name":"Cloudflare Workers AI","kind":"cloud","url":"https://developers.cloudflare.com/workers-ai/models/","inputPerMillion":0.44,"outputPerMillion":1.32,"cacheReadPerMillion":0.014,"contextTokens":1048576,"maxOutputTokens":1048576,"deprecated":true},{"host":"novita-ai","name":"Novita AI","kind":"host","url":"https://novita.ai/docs/guides/introduction","inputPerMillion":0.44,"outputPerMillion":1.32,"cacheReadPerMillion":0.028,"contextTokens":1048576,"maxOutputTokens":393216,"deprecated":false},{"host":"volcengine","name":"Volcengine Ark","kind":"cloud","url":"https://www.volcengine.com/docs/82379/1330310","inputPerMillion":0.4453,"outputPerMillion":1.3359,"cacheReadPerMillion":0.01484,"contextTokens":1000000,"maxOutputTokens":384000,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:arc-prize-foundation","name":"ARC Prize Foundation","url":"https://arcprize.org","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:livebench","name":"LiveBench","url":"https://livebench.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gemini31propreview","providerId":"google","slug":"google-gemini-3-1-pro-preview","name":"Gemini 3.1 Pro Preview","provider":"Google","description":"Gemini 3.1 Pro Preview is a model from Google. 27 benchmarks count toward its index, in 7 categories.","logoUrl":null,"releaseDate":"2026-02-19T14:00:27.000Z","contextWindowTokens":1048576,"modalities":["audio","file","image","text","video"],"openWeights":null,"details":{"apiId":"google/gemini-3.1-pro-preview","maxOutputTokens":65536,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.19999999999999998,"weightsUrl":null},"reasoning":null,"overallScore":61.481200874718255,"overallScoreRange":{"low":57.86928292378617,"high":65.09311882565034},"badges":[],"capabilityScores":{"agentic":52.46336090540702,"coding":58.83162387201387,"reasoning":69.7519318293503,"multimodal":69.947788109467,"knowledge":64.991034180874,"multilingual":null,"instruction":86.64085545693331,"math":60.39555379271535},"capabilityCounts":{"math":3,"reasoning":4,"knowledge":3,"coding":8,"multimodal":1,"instruction":1,"agentic":7},"benchmarks":[{"benchmarkId":"vals-ai:aime:overall","benchmarkName":"AIME","category":"math","domain":null,"eligible":true,"ranked":true,"score":98.125,"unit":"percent","family":"vals-ai:aime:overall","index":58.38768123285469,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/aime","measuredAt":"2026-04-16","stderr":0.343,"costPerTest":0.126991,"latency":104.525},{"benchmarkId":"arc-prize:v1-semi-private:overall","benchmarkName":"ARC-AGI-1 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":98,"unit":"percent","family":"arc-prize:v1-semi-private:overall","index":72.04149695168164,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.5223,"latency":null},{"benchmarkId":"arc-prize:v1-public-eval:overall","benchmarkName":"ARC-AGI-1 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":97.24000000000001,"unit":"percent","family":"arc-prize:v1-public-eval:overall","index":null,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.3727,"latency":null},{"benchmarkId":"vals-ai:medqa:overall","benchmarkName":"MedQA","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":96.367,"unit":"percent","family":"vals-ai:medqa:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medqa","measuredAt":"2026-04-16","stderr":0.171,"costPerTest":0.017689,"latency":38.67},{"benchmarkId":"vals-ai:gpqa:overall","benchmarkName":"GPQA Diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":95.454,"unit":"percent","family":"gpqa-diamond","index":64.75690124964268,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","measuredAt":"2026-09-01","stderr":1.046,"costPerTest":0.081442,"latency":65.76},{"benchmarkId":"livebench:livebench-math:overall","benchmarkName":"LiveBench Mathematics","category":"math","domain":null,"eligible":true,"ranked":true,"score":91.04499999999999,"unit":"percent","family":"livebench:livebench-math:overall","index":66.6659973288335,"aboveFit":false,"run":"high effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Mathematics","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.17111846965699212,"latency":null},{"benchmarkId":"vals-ai:mmlu_pro:overall","benchmarkName":"MMLU Pro","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":90.987,"unit":"percent","family":"mmlu-pro","index":59.11345756300324,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","measuredAt":"2026-09-01","stderr":0.284,"costPerTest":0.027545,"latency":23.912},{"benchmarkId":"vals-ai:lcb:overall","benchmarkName":"LiveCodeBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":88.485,"unit":"percent","family":"livecodebench","index":62.763419151806154,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","measuredAt":"2026-09-01","stderr":0.931,"costPerTest":0.101147,"latency":88.891},{"benchmarkId":"vals-ai:mmmu:overall","benchmarkName":"MMMU Pro","category":"multimodal","domain":null,"eligible":true,"ranked":true,"score":88.208,"unit":"percent","family":"mmmu-pro","index":69.947788109467,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmmu","measuredAt":"2026-09-01","stderr":0.775,"costPerTest":0.064084,"latency":76.99},{"benchmarkId":"arc-prize:v2-public-eval:overall","benchmarkName":"ARC-AGI-2 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":88.07000000000001,"unit":"percent","family":"arc-prize:v2-public-eval:overall","index":null,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.9789,"latency":null},{"benchmarkId":"vals-ai:legal_bench:overall","benchmarkName":"LegalBench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":87.398,"unit":"percent","family":"vals-ai:legal_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","measuredAt":"2026-10-13","stderr":0.329,"costPerTest":0.006771,"latency":10.062},{"benchmarkId":"livebench:livebench-language:overall","benchmarkName":"LiveBench Language","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":85.37599999999999,"unit":"percent","family":"livebench:livebench-language:overall","index":71.10274372997608,"aboveFit":false,"run":"high effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Language","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.09741866666666667,"latency":null},{"benchmarkId":"livebench:livebench-reasoning:overall","benchmarkName":"LiveBench Reasoning","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":84.00475,"unit":"percent","family":"livebench","index":70.44233032321185,"aboveFit":false,"run":"high effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Reasoning","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.14840742574257426,"latency":null},{"benchmarkId":"vals-ai:sage:overall","benchmarkName":"SAGE","category":null,"domain":"Education","eligible":false,"ranked":true,"score":83.176,"unit":"percent","family":"vals-ai:sage:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/sage","measuredAt":"2026-11-05","stderr":1.954,"costPerTest":0.053936,"latency":45.639},{"benchmarkId":"livebench:livebench-instruction-following:overall","benchmarkName":"LiveBench Instruction Following","category":"instruction","domain":null,"eligible":true,"ranked":true,"score":79.1,"unit":"percent","family":"livebench:livebench-instruction-following:overall","index":86.64085545693331,"aboveFit":true,"run":"high effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=IF","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.13931000000000002,"latency":null},{"benchmarkId":"vals-ai:swebench:overall","benchmarkName":"SWE-bench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":78.8,"unit":"percent","family":"swe-bench-verified","index":59.34746465882657,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/swebench","measuredAt":"2026-09-01","stderr":1.83,"costPerTest":0.783381,"latency":312.261},{"benchmarkId":"livebench:livebench-data-analysis:overall","benchmarkName":"LiveBench Data Analysis","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":78.54133333333334,"unit":"percent","family":"livebench","index":63.36483463809816,"aboveFit":false,"run":"high effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Data%20Analysis","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.25950133333333336,"latency":null},{"benchmarkId":"arc-prize:v2-semi-private:overall","benchmarkName":"ARC-AGI-2 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":77.08,"unit":"percent","family":"arc-prize:v2-semi-private:overall","index":74.57861815348465,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.9622,"latency":null},{"benchmarkId":"livebench:livebench-coding:overall","benchmarkName":"LiveBench Coding","category":"coding","domain":null,"eligible":true,"ranked":true,"score":76.4545,"unit":"percent","family":"livebench:livebench-coding:overall","index":62.35053796722022,"aboveFit":false,"run":"high effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Coding","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.07733931623931624,"latency":null},{"benchmarkId":"vals-ai:medscribe:overall","benchmarkName":"MedScribe","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":76.114,"unit":"percent","family":"vals-ai:medscribe:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medscribe","measuredAt":"2026-10-29","stderr":1.915,"costPerTest":0.097954,"latency":69.139},{"benchmarkId":"vals-ai:tax_eval_v2:overall","benchmarkName":"TaxEval v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":72.882,"unit":"percent","family":"vals-ai:tax_eval_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","measuredAt":"2026-09-01","stderr":0.863,"costPerTest":0.03149,"latency":41.931},{"benchmarkId":"vals-ai:terminal-bench-2-1:overall","benchmarkName":"Terminal-Bench 2.1","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":70.787,"unit":"percent","family":"terminal-bench-2-1","index":62.54127789230368,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2-1","measuredAt":"2026-10-01","stderr":1.124,"costPerTest":0.582677,"latency":410.249},{"benchmarkId":"vals-ai:mortgage_tax:overall","benchmarkName":"MortgageTax","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":69.396,"unit":"percent","family":"vals-ai:mortgage_tax:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mortgage_tax","measuredAt":"2026-09-01","stderr":0.908,"costPerTest":0.023542,"latency":23.616},{"benchmarkId":"vals-ai:terminal-bench-2:overall","benchmarkName":"Terminal-Bench 2.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":67.416,"unit":"percent","family":"vals-ai:terminal-bench-2:overall","index":67.1431969239635,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2","measuredAt":"2026-06-04","stderr":4.996,"costPerTest":0.516084,"latency":542.887},{"benchmarkId":"livebench:livebench-coding-2:overall","benchmarkName":"LiveBench Coding 2.0","category":"coding","domain":null,"eligible":true,"ranked":true,"score":66.7,"unit":"percent","family":"livebench:livebench-coding-2:overall","index":56.484401448022936,"aboveFit":false,"run":"high effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Coding","measuredAt":"2026-10-14","stderr":null,"costPerTest":1.2907172413793102,"latency":null},{"benchmarkId":"livebench:livebench-research:overall","benchmarkName":"LiveBench Research","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":65.7,"unit":"percent","family":"livebench-2-agentic","index":66.63792451864423,"aboveFit":false,"run":"high effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Research","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.7698954545454545,"latency":null},{"benchmarkId":"vals-ai:case_law_v2:overall","benchmarkName":"CaseLaw v2","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":64.845,"unit":"percent","family":"vals-ai:case_law_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/case_law_v2","measuredAt":"2026-05-04","stderr":1.236,"costPerTest":0.136185,"latency":56.785},{"benchmarkId":"vals-ai:corp_fin_v2:overall","benchmarkName":"CorpFin v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":64.491,"unit":"percent","family":"vals-ai:corp_fin_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","measuredAt":"2026-08-12","stderr":0.942,"costPerTest":0.181957,"latency":25.388},{"benchmarkId":"livebench:livebench-design:overall","benchmarkName":"LiveBench Design","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":61.6,"unit":"percent","family":"livebench-2-agentic","index":63.63946702947035,"aboveFit":false,"run":"high effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Design","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.595808,"latency":null},{"benchmarkId":"vals-ai:biomysterybench:overall","benchmarkName":"BioMysteryBench","category":null,"domain":"Science","eligible":false,"ranked":true,"score":60,"unit":"percent","family":"vals-ai:biomysterybench:overall","index":null,"aboveFit":false,"run":"Terminus 2","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/biomysterybench","measuredAt":"2026-11-09","stderr":2.94,"costPerTest":0.713378,"latency":251.867},{"benchmarkId":"vals-ai:finance_agent:overall","benchmarkName":"Finance Agent v1.1","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":59.717,"unit":"percent","family":"vals-ai:finance_agent:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/finance_agent","measuredAt":"2026-06-04","stderr":2.804,"costPerTest":0.872622,"latency":265.722},{"benchmarkId":"vals-ai:medcode:overall","benchmarkName":"MedCode","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":59.062,"unit":"percent","family":"vals-ai:medcode:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medcode","measuredAt":"2026-10-29","stderr":1.996,"costPerTest":0.024714,"latency":38.523},{"benchmarkId":"vals-ai:public-benefits-bench:overall","benchmarkName":"Public Benefits Bench v1.1","category":null,"domain":"Public benefits","eligible":false,"ranked":true,"score":53.789,"unit":"percent","family":"vals-ai:public-benefits-bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/public-benefits-bench","measuredAt":"2026-10-12","stderr":1.297,"costPerTest":0.376768,"latency":185.553},{"benchmarkId":"vals-ai:emb:overall","benchmarkName":"EMB","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":52.616,"unit":"percent","family":"vals-ai:emb:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/emb","measuredAt":"2026-10-20","stderr":2.974,"costPerTest":3.769246,"latency":708.582},{"benchmarkId":"vals-ai:ioi:overall","benchmarkName":"IOI","category":"coding","domain":null,"eligible":true,"ranked":true,"score":51.833,"unit":"percent","family":"vals-ai:ioi:overall","index":65.57590347388714,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/ioi","measuredAt":"2026-10-29","stderr":2.589,"costPerTest":1.25149,"latency":722.637},{"benchmarkId":"vals-ai:public-benefits-bench-v1:overall","benchmarkName":"Public Benefits Bench v1","category":null,"domain":"Public benefits","eligible":false,"ranked":true,"score":50.88,"unit":"percent","family":"vals-ai:public-benefits-bench-v1:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/public-benefits-bench-v1","measuredAt":"2026-06-09","stderr":1.3,"costPerTest":0.300864,"latency":177.913},{"benchmarkId":"livebench:livebench-agentic-coding:overall","benchmarkName":"LiveBench Agentic Coding","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":44.141333333333336,"unit":"percent","family":"livebench:livebench-agentic-coding:overall","index":54.6848956282049,"aboveFit":false,"run":"high effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Agentic%20Coding","measuredAt":"2026-06-25","stderr":null,"costPerTest":1.3071222222222223,"latency":null},{"benchmarkId":"vals-ai:fabv2:overall","benchmarkName":"Finance Agent (v2)","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":42.982,"unit":"percent","family":"vals-ai:fabv2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/fabv2","measuredAt":"2026-10-29","stderr":1.206,"costPerTest":1.591906,"latency":208.258},{"benchmarkId":"vals-ai:vibe-code:overall","benchmarkName":"Vibe Code Bench v1.1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":32.034,"unit":"percent","family":"vals-ai:vibe-code:overall","index":53.807893082848096,"aboveFit":false,"run":"OpenHands","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","measuredAt":"2026-11-09","stderr":4.342,"costPerTest":3.825202,"latency":1211.906},{"benchmarkId":"tau2-bench:banking-knowledge:overall","benchmarkName":"τ²-bench Banking","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":26.03,"unit":"percent","family":"tau2-bench","index":20.06224824865828,"aboveFit":false,"run":"high effort · Sierra","source":"Sierra Research","sourceUrl":"https://github.com/sierra-research/tau2-bench","measuredAt":"2026-10-20","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:proof_bench:overall","benchmarkName":"ProofBench v1.1","category":"math","domain":null,"eligible":true,"ranked":true,"score":26,"unit":"percent","family":"vals-ai:proof_bench:overall","index":56.13298281645785,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/proof_bench","measuredAt":"2026-11-09","stderr":4.408,"costPerTest":0.784253,"latency":889.059},{"benchmarkId":"livebench:livebench-long-horizon:overall","benchmarkName":"LiveBench Long Agentic","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":23.9,"unit":"percent","family":"livebench-2-agentic","index":36.06828475096912,"aboveFit":false,"run":"high effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Long%20Agentic","measuredAt":"2026-10-14","stderr":null,"costPerTest":8.125694444444443,"latency":null},{"benchmarkId":"vals-ai:mysterymechanism:overall","benchmarkName":"MysteryMechanism","category":null,"domain":"Science","eligible":false,"ranked":true,"score":20.721,"unit":"percent","family":"vals-ai:mysterymechanism:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mysterymechanism","measuredAt":"2026-11-11","stderr":2.726,"costPerTest":1.065147,"latency":360.456},{"benchmarkId":"vals-ai:legal_research:overall","benchmarkName":"Legal Research Bench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":20.673,"unit":"percent","family":"vals-ai:legal_research:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_research","measuredAt":"2026-11-09","stderr":2.815,"costPerTest":1.100145,"latency":242.05},{"benchmarkId":"vals-ai:code-migration:overall","benchmarkName":"Code Migration","category":"coding","domain":null,"eligible":true,"ranked":true,"score":17.313,"unit":"percent","family":"vals-ai:code-migration:overall","index":54.38649718688736,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/code-migration","measuredAt":"2026-10-29","stderr":3.933,"costPerTest":1.614622,"latency":720.574},{"benchmarkId":"vals-ai:tax_agent_bench:overall","benchmarkName":"Tax Agent Bench","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":9.295,"unit":"percent","family":"vals-ai:tax_agent_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench","measuredAt":"2026-11-09","stderr":2.092,"costPerTest":0.301555,"latency":100.956},{"benchmarkId":"vals-ai:vcb-1-100:overall","benchmarkName":"Vibe Code Bench 1-100","category":"coding","domain":null,"eligible":true,"ranked":true,"score":6.694,"unit":"percent","family":"vals-ai:vcb-1-100:overall","index":55.936874006612484,"aboveFit":false,"run":"OpenHands","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vcb-1-100","measuredAt":"2026-10-29","stderr":2.398,"costPerTest":3.358183,"latency":688.056},{"benchmarkId":"livebench:livebench-browser:overall","benchmarkName":"LiveBench Browser Use","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":4,"unit":"percent","family":"livebench-2-agentic","index":21.514795962052027,"aboveFit":false,"run":"high effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Browser","measuredAt":"2026-10-14","stderr":null,"costPerTest":1.777176,"latency":null},{"benchmarkId":"vals-ai:terminal-bench-4:overall","benchmarkName":"Terminal-Bench 4.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":2.525,"unit":"percent","family":"terminal-bench-4","index":55.714981010188346,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-4","measuredAt":"2026-11-09","stderr":0.505,"costPerTest":4.111957,"latency":1124.176},{"benchmarkId":"vals-ai:terminal-bench-science:overall","benchmarkName":"Terminal-Bench-Science","category":null,"domain":"Science","eligible":false,"ranked":true,"score":1.429,"unit":"percent","family":"vals-ai:terminal-bench-science:overall","index":null,"aboveFit":false,"run":"Mini-SWE-agent","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-science","measuredAt":"2026-11-09","stderr":1.429,"costPerTest":5.907095,"latency":2638.065},{"benchmarkId":"arc-prize:v3-semi-private:overall","benchmarkName":"ARC-AGI-3 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":0.42,"unit":"percent","family":"arc-prize:v3-semi-private:overall","index":65.48402973157992,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:programbench:overall","benchmarkName":"ProgramBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":0,"unit":"percent","family":"vals-ai:programbench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/programbench","measuredAt":"2026-11-09","stderr":0,"costPerTest":2.629095,"latency":755.928},{"benchmarkId":"vals-ai:srebench:overall","benchmarkName":"SRE Bench","category":null,"domain":"Other","eligible":false,"ranked":true,"score":0,"unit":"percent","family":"vals-ai:srebench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/srebench","measuredAt":"2026-11-05","stderr":0,"costPerTest":11.336961,"latency":3803.735},{"benchmarkId":"lmarena:agent-steerability:overall","benchmarkName":"Agent Arena steerability","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":-8.22961826507701,"unit":"index","family":"agent-arena","index":59.76476492856178,"aboveFit":false,"run":null,"source":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset","measuredAt":"2026-11-11","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lmarena:agent-task-outcome:overall","benchmarkName":"Agent Arena task outcome","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":-11.296812104736976,"unit":"index","family":"agent-arena","index":57.24305074431072,"aboveFit":false,"run":null,"source":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset","measuredAt":"2026-11-11","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lmarena:agent-bash-recovery:overall","benchmarkName":"Agent Arena command recovery","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":-23.265218021387014,"unit":"index","family":"agent-arena","index":47.40314477793833,"aboveFit":false,"run":null,"source":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset","measuredAt":"2026-10-08","stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":2,"outputPerMillion":12,"batch":{"inputPerMillion":1,"outputPerMillion":6},"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":77,"timeToFirstToken":2.916,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":{"elo":1479.9928559278965,"eloUpper":1482.9082049867043,"eloLower":1477.0775068690887,"votes":132296,"rank":20,"rankUpper":null,"rankLower":null,"arenaName":"Gemini 3.1 Pro Preview","organization":"google","updatedAt":"2026-11-09"},"quality":{"coveragePercent":95,"benchmarkCount":27,"categoryCount":7,"confidence":"medium","state":"available"},"hosts":[{"host":"databricks","name":"Databricks","kind":"cloud","url":"https://docs.databricks.com/aws/en/machine-learning/foundation-models/","inputPerMillion":2,"outputPerMillion":12,"cacheReadPerMillion":0.2,"contextTokens":1048576,"maxOutputTokens":65536,"deprecated":false},{"host":"google-ai-studio","name":"Google AI Studio","kind":"maker","url":"https://ai.google.dev/gemini-api/docs/models","inputPerMillion":2,"outputPerMillion":12,"cacheReadPerMillion":0.2,"contextTokens":1048576,"maxOutputTokens":65536,"deprecated":false},{"host":"google-vertex","name":"Google Vertex AI","kind":"maker","url":"https://cloud.google.com/vertex-ai/generative-ai/docs/models","inputPerMillion":2,"outputPerMillion":12,"cacheReadPerMillion":0.2,"contextTokens":1048576,"maxOutputTokens":65536,"deprecated":false},{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":2,"outputPerMillion":12,"cacheReadPerMillion":0.2,"contextTokens":1048576,"maxOutputTokens":65536,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:arc-prize-foundation","name":"ARC Prize Foundation","url":"https://arcprize.org","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:livebench","name":"LiveBench","url":"https://livebench.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:sierra-research","name":"Sierra Research","url":"https://github.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:lmarena","name":"LMArena","url":"https://huggingface.co","license":"CC BY 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"mistral-large-4","providerId":"mistral","slug":"mistral-large-4","name":"Mistral Large 4","provider":"Mistral","description":"Mistral Large 4 is a reasoning model from Mistral. 40 benchmarks count toward its index, in 6 categories.","logoUrl":null,"releaseDate":"2026-10-06","contextWindowTokens":1000000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":true,"overallScore":60.68193312186661,"overallScoreRange":{"low":56.62523163362047,"high":64.73863461011274},"badges":[],"capabilityScores":{"agentic":66.2796995004991,"coding":62.676160014233005,"reasoning":59.23421504830369,"multimodal":null,"knowledge":43.66752944313745,"multilingual":null,"instruction":57.74736609481352,"math":61.6064128904961},"capabilityCounts":{"math":5,"knowledge":4,"agentic":15,"reasoning":7,"coding":8,"instruction":1},"benchmarks":[{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":98.88888888888889,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":66.45205762128103,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0.3883473863441843,"costPerTest":null,"latency":null},{"benchmarkId":"b3AgentSecurity","benchmarkName":"b3: Backbone Breaker Benchmark","category":"agentic","score":97.1,"unit":"percent","family":"b3AgentSecurity","index":null,"aboveFit":false,"run":null,"source":"Lakera with the UK AI Security Institute","sourceUrl":"https://www.lakera.ai/blog/the-backbone-breaker-benchmark","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"cyberHarmfulRefusal","benchmarkName":"Refusal of harmful cyber requests (JailbreakBench, StrongREJECT, AgentHarm)","category":"agentic","score":96.6,"unit":"percent","family":"cyberHarmfulRefusal","index":null,"aboveFit":false,"run":null,"source":"Mistral AI (composite of JailbreakBench, StrongREJECT and AgentHarm)","sourceUrl":"https://mistral.ai/news/mistral-large-4-oss/","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-math:overall","benchmarkName":"LiveBench Mathematics","category":"math","domain":null,"eligible":true,"ranked":true,"score":93.58474999999999,"unit":"percent","family":"livebench:livebench-math:overall","index":70.20569590448385,"aboveFit":false,"run":"high effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Mathematics","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.05028601583113456,"latency":null},{"benchmarkId":"cybench","benchmarkName":"Cybench","category":"agentic","score":93,"unit":"percent","family":"cybench","index":null,"aboveFit":false,"run":null,"source":"Andy K. Zhang, Neil Perry, Riya Dulepet, Joey Ji, Celeste Menders, Justin W. Lin, Eliot Jones, Gashon Hussein, Samantha Liu, Donovan Jasper, Pura Peetathawatchai, Ari Glenn, Vikram Sivashankar, Daniel Zamoshchin, Leo Glikbarg, Derek Askaryar, Mike Yang, Teddy Zhang, Rishi Alluri, Nathan Tran, Rinnara Sangpisit, Polycarpos Yiorkadjis, Kenny Osele, Gautham Raghupathi, Dan Boneh, Daniel E. Ho, Percy Liang","sourceUrl":"https://arxiv.org/abs/2408.08926","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":89.1098484848485,"unit":"percent","family":"gpqa-diamond","index":58.93821883439478,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.8113832926930726,"costPerTest":null,"latency":null},{"benchmarkId":"browseComp","benchmarkName":"BrowseComp","category":"agentic","score":84.5,"unit":"percent","family":"browseComp","index":66.66566765754015,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-reasoning:overall","benchmarkName":"LiveBench Reasoning","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":83.94225,"unit":"percent","family":"livebench","index":70.36136569845269,"aboveFit":false,"run":"high effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Reasoning","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.06066732673267326,"latency":null},{"benchmarkId":"vals-ai:medscribe:overall","benchmarkName":"MedScribe","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":80.434,"unit":"percent","family":"vals-ai:medscribe:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medscribe","measuredAt":"2026-10-29","stderr":1.996,"costPerTest":0.023874,"latency":125.352},{"benchmarkId":"aaCyberGymE2e","benchmarkName":"Artificial Analysis CyberGym-E2E-AA","category":"agentic","score":79.4,"unit":"percent","family":"aaCyberGymE2e","index":95,"aboveFit":true,"run":null,"source":"Artificial Analysis; dataset by Berkeley RDI","sourceUrl":"https://artificialanalysis.ai/evaluations/cybergym-e2e-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:vibe-code:overall","benchmarkName":"Vibe Code Bench v1.1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":78.403,"unit":"percent","family":"vals-ai:vibe-code:overall","index":70.53855679447608,"aboveFit":false,"run":"OpenHands","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","measuredAt":"2026-11-09","stderr":3.55,"costPerTest":20.281954,"latency":10395.145},{"benchmarkId":"livebench:livebench-coding:overall","benchmarkName":"LiveBench Coding","category":"coding","domain":null,"eligible":true,"ranked":true,"score":77.1585,"unit":"percent","family":"livebench:livebench-coding:overall","index":63.340634835811045,"aboveFit":false,"run":"high effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Coding","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.03128376068376069,"latency":null},{"benchmarkId":"livebench:livebench-data-analysis:overall","benchmarkName":"LiveBench Data Analysis","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":76.54166666666667,"unit":"percent","family":"livebench","index":60.77439845713709,"aboveFit":false,"run":"high effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Data%20Analysis","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.06850333333333335,"latency":null},{"benchmarkId":"finch","benchmarkName":"Finch (FinWorkBench): Finance & Accounting across Spreadsheet-Centric Enterprise Workflows","category":"agentic","score":73.3,"unit":"percent","family":"finch","index":null,"aboveFit":false,"run":null,"source":"Finch authors (ACL 2026 Findings)","sourceUrl":"https://arxiv.org/abs/2512.13168","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:biomysterybench:overall","benchmarkName":"BioMysteryBench","category":null,"domain":"Science","eligible":false,"ranked":true,"score":67.037,"unit":"percent","family":"vals-ai:biomysterybench:overall","index":null,"aboveFit":false,"run":"Terminus 2","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/biomysterybench","measuredAt":"2026-11-09","stderr":2.671,"costPerTest":1.04645,"latency":1532.221},{"benchmarkId":"livebench:livebench-design:overall","benchmarkName":"LiveBench Design","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":65.6,"unit":"percent","family":"livebench-2-agentic","index":66.56479140915218,"aboveFit":false,"run":"high effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Design","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.388956,"latency":null},{"benchmarkId":"livebench:livebench-instruction-following:overall","benchmarkName":"LiveBench Instruction Following","category":"instruction","domain":null,"eligible":true,"ranked":true,"score":64.82075,"unit":"percent","family":"livebench:livebench-instruction-following:overall","index":57.74736609481352,"aboveFit":false,"run":"high effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=IF","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.038413,"latency":null},{"benchmarkId":"livebench:livebench-coding-2:overall","benchmarkName":"LiveBench Coding 2.0","category":"coding","domain":null,"eligible":true,"ranked":true,"score":60,"unit":"percent","family":"livebench:livebench-coding-2:overall","index":51.6314234387289,"aboveFit":false,"run":"high effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Coding","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.9825862068965517,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tiers-1-3-v2-private:overall","benchmarkName":"FrontierMath-Tiers-1-3-v2-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":58.59649122807018,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tiers-1-3-v2-private:overall","index":61.7370597393179,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.922773823944577,"costPerTest":null,"latency":null},{"benchmarkId":"aaCweBench","benchmarkName":"Artificial Analysis CWE-Bench-AA","category":"agentic","score":58.3,"unit":"percent","family":"aaCweBench","index":70.43833818983995,"aboveFit":false,"run":null,"source":"Artificial Analysis; dataset by Collinear AI","sourceUrl":"https://artificialanalysis.ai/evaluations/cwe-bench-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaAutomationBench","benchmarkName":"Artificial Analysis AutomationBench","category":"agentic","score":57.8,"unit":"percent","family":"aaAutomationBench","index":68.36440284935125,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/automationbench-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-agentic-coding:overall","benchmarkName":"LiveBench Agentic Coding","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":57.17166666666666,"unit":"percent","family":"livebench:livebench-agentic-coding:overall","index":68.12285646639657,"aboveFit":false,"run":"high effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Agentic%20Coding","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.9099555555555556,"latency":null},{"benchmarkId":"vals-ai:emb:overall","benchmarkName":"EMB","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":56.213,"unit":"percent","family":"vals-ai:emb:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/emb","measuredAt":"2026-10-20","stderr":3.22,"costPerTest":4.839324,"latency":4904.981},{"benchmarkId":"vals-ai:fabv2:overall","benchmarkName":"Finance Agent (v2)","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":54.678,"unit":"percent","family":"vals-ai:fabv2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/fabv2","measuredAt":"2026-10-29","stderr":0.58,"costPerTest":0.614903,"latency":1561.879},{"benchmarkId":"aaCyberIndex","benchmarkName":"Artificial Analysis Cyber Index","category":"agentic","score":50.9,"unit":"percent","family":"aaCyberIndex","index":79.48426229872344,"aboveFit":true,"run":null,"source":"Artificial Analysis with the Cyber Index Alliance (Collinear AI, IBM, NVIDIA, Pasta Lab, Vercel)","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-cyber-index","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-language:overall","benchmarkName":"LiveBench Language","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":49.64233333333333,"unit":"percent","family":"livebench:livebench-language:overall","index":30.69476907840834,"aboveFit":false,"run":"high effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Language","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.052888000000000004,"latency":null},{"benchmarkId":"gdpvalAaNormalized","benchmarkName":"GDPval-AA normalized","category":"agentic","score":46.1,"unit":"percent","family":"gdpvalAaNormalized","index":66.99420363118624,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"cweBenchV2","benchmarkName":"CWE-bench v2","category":"agentic","score":45.6,"unit":"percent","family":"cweBenchV2","index":62.43506763753219,"aboveFit":false,"run":null,"source":"Collinear AI","sourceUrl":"https://cwe-bench.com/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:ioi:overall","benchmarkName":"IOI","category":"coding","domain":null,"eligible":true,"ranked":true,"score":45.278,"unit":"percent","family":"vals-ai:ioi:overall","index":63.1150096366501,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/ioi","measuredAt":"2026-10-29","stderr":3.783,"costPerTest":2.728813,"latency":5023.763},{"benchmarkId":"vals-ai:medcode:overall","benchmarkName":"MedCode","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":40.685,"unit":"percent","family":"vals-ai:medcode:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medcode","measuredAt":"2026-10-29","stderr":2.171,"costPerTest":0.021241,"latency":68.682},{"benchmarkId":"mka","benchmarkName":"Multimodal Knowledge Agent Benchmark","category":"agentic","score":35.1,"unit":"percent","family":"mka","index":65.09544468861279,"aboveFit":false,"run":null,"source":"Google DeepMind, in partnership with Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mka","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:legal_research:overall","benchmarkName":"Legal Research Bench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":31.731,"unit":"percent","family":"vals-ai:legal_research:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_research","measuredAt":"2026-11-09","stderr":3.235,"costPerTest":2.723732,"latency":5571.099},{"benchmarkId":"epoch-ai:epoch-chess-puzzles:overall","benchmarkName":"Chess Puzzles","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":31,"unit":"percent","family":"epoch-ai:epoch-chess-puzzles:overall","index":57.51414677479192,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":4.725815626252608,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:code-migration:overall","benchmarkName":"Code Migration","category":"coding","domain":null,"eligible":true,"ranked":true,"score":30.561,"unit":"percent","family":"vals-ai:code-migration:overall","index":61.42436594824361,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/code-migration","measuredAt":"2026-10-29","stderr":4.221,"costPerTest":24.522375,"latency":17441.256},{"benchmarkId":"vals-ai:tax_agent_bench:overall","benchmarkName":"Tax Agent Bench","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":30.391,"unit":"percent","family":"vals-ai:tax_agent_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench","measuredAt":"2026-11-09","stderr":3.309,"costPerTest":1.390372,"latency":3438.427},{"benchmarkId":"livebench:livebench-research:overall","benchmarkName":"LiveBench Research","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":28.1,"unit":"percent","family":"livebench-2-agentic","index":39.13987534963504,"aboveFit":false,"run":"high effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Research","measuredAt":"2026-10-14","stderr":null,"costPerTest":2.327159090909091,"latency":null},{"benchmarkId":"vals-ai:terminal-bench-4:overall","benchmarkName":"Terminal-Bench 4.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":27.778,"unit":"percent","family":"terminal-bench-4","index":67.7060809694087,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-4","measuredAt":"2026-11-09","stderr":1.821,"costPerTest":5.180898,"latency":6011.951},{"benchmarkId":"epoch-ai:epoch-mirrorcode:overall","benchmarkName":"MirrorCode","category":"coding","domain":null,"eligible":true,"ranked":true,"score":27.77777777777778,"unit":"percent","family":"epoch-ai:epoch-mirrorcode:overall","index":61.76484445504887,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":8.85907699198465,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tier-4-v2-private:overall","benchmarkName":"FrontierMath-Tier-4-v2-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":26.82926829268293,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tier-4-v2-private:overall","index":59.02818358680503,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":7.005564203095157,"costPerTest":null,"latency":null},{"benchmarkId":"omniscienceAccuracy","benchmarkName":"Artificial Analysis Omniscience Accuracy","category":"knowledge","score":26,"unit":"percent","family":"omniscienceAccuracy","index":43.456606052172525,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-furniture-assembly:overall","benchmarkName":"Furniture Assembly","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":25.833333333333336,"unit":"percent","family":"epoch-ai:epoch-furniture-assembly:overall","index":58.70642185452705,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":5.9531678253102225,"costPerTest":null,"latency":null},{"benchmarkId":"aaTerminalBench4","benchmarkName":"Artificial Analysis Terminal-Bench v4.0","category":"agentic","score":24.7,"unit":"percent","family":"aaTerminalBench4","index":62.67254107197094,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/terminalbench-v4-0","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-simpleqa-verified:overall","benchmarkName":"SimpleQA Verified","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":21.200000000000003,"unit":"percent","family":"epoch-ai:epoch-simpleqa-verified:overall","index":41.58052380757414,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.319083007236459,"costPerTest":null,"latency":null},{"benchmarkId":"frontierCode2","benchmarkName":"FrontierCode 2.0","category":"coding","score":19.7,"unit":"percent","family":"frontierCode2","index":66.632388123319,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-mystery-game-puzzles:overall","benchmarkName":"Mystery Game Puzzles","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":18.5,"unit":"percent","family":"epoch-ai:epoch-mystery-game-puzzles:overall","index":53.00767379111587,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":4.020151261036848,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-ebr-bench:overall","benchmarkName":"EBR-bench","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":18.095238095238095,"unit":"percent","family":"epoch-ai:epoch-ebr-bench:overall","index":60.731178901262666,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":3.1586902765289517,"costPerTest":null,"latency":null},{"benchmarkId":"aaMlcr","benchmarkName":"Medical Long Context Reasoning (MLCR-AA)","category":"reasoning","score":17.8,"unit":"percent","family":"aaMlcr","index":62.90425275887164,"aboveFit":false,"run":null,"source":"Wisedocs and Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mlcr-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaGdpPdf","benchmarkName":"Artificial Analysis GDP.pdf","category":"agentic","score":17.6,"unit":"percent","family":"aaGdpPdf","index":63.55278077571523,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdp-pdf","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:hlab:overall","benchmarkName":"HLAB-Vals-1.1","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":17.5,"unit":"percent","family":"vals-ai:hlab:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/hlab","measuredAt":"2026-11-09","stderr":3.222,"costPerTest":2.554025,"latency":3483.562},{"benchmarkId":"aaDeepsecBench","benchmarkName":"Artificial Analysis DeepsecBench-AA","category":"agentic","score":15,"unit":"percent","family":"aaDeepsecBench","index":61.822671112568,"aboveFit":false,"run":null,"source":"Artificial Analysis; dataset by Vercel","sourceUrl":"https://artificialanalysis.ai/evaluations/deepsecbench-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:vcb-1-100:overall","benchmarkName":"Vibe Code Bench 1-100","category":"coding","domain":null,"eligible":true,"ranked":true,"score":14.261,"unit":"percent","family":"vals-ai:vcb-1-100:overall","index":62.9620568815864,"aboveFit":false,"run":"OpenHands","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vcb-1-100","measuredAt":"2026-10-29","stderr":3.205,"costPerTest":3.435784,"latency":3593.189},{"benchmarkId":"vals-ai:mysterymechanism:overall","benchmarkName":"MysteryMechanism","category":null,"domain":"Science","eligible":false,"ranked":true,"score":12.162,"unit":"percent","family":"vals-ai:mysterymechanism:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mysterymechanism","measuredAt":"2026-11-11","stderr":2.199,"costPerTest":1.06223,"latency":2136.737},{"benchmarkId":"livebench:livebench-long-horizon:overall","benchmarkName":"LiveBench Long Agentic","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":11.8,"unit":"percent","family":"livebench-2-agentic","index":27.219178502431593,"aboveFit":false,"run":"high effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Long%20Agentic","measuredAt":"2026-10-14","stderr":null,"costPerTest":9.407668965517242,"latency":null},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":10,"unit":"percent","family":"critpt","index":56.20794917976178,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:proof_bench:overall","benchmarkName":"ProofBench v1.1","category":"math","domain":null,"eligible":true,"ranked":true,"score":10,"unit":"percent","family":"vals-ai:proof_bench:overall","index":50.60906760059271,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/proof_bench","measuredAt":"2026-11-09","stderr":3.015,"costPerTest":0.329279,"latency":1258.806},{"benchmarkId":"livebench:livebench-browser:overall","benchmarkName":"LiveBench Browser Use","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":8,"unit":"percent","family":"livebench-2-agentic","index":24.440120341733856,"aboveFit":false,"run":"high effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Browser","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.450644,"latency":null},{"benchmarkId":"stemBench120","benchmarkName":"STEMBench120: six-domain scientific research progress","category":"reasoning","score":2.501,"unit":"percent","family":"stemBench120","index":null,"aboveFit":false,"run":null,"source":"Ulam","sourceUrl":"https://www.ulam.ai/stembench","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lmarena:agent-bash-recovery:overall","benchmarkName":"Agent Arena command recovery","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":1.7771044807505465,"unit":"index","family":"agent-arena","index":67.99185979284286,"aboveFit":false,"run":null,"source":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset","measuredAt":"2026-10-08","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lmarena:agent-task-outcome:overall","benchmarkName":"Agent Arena task outcome","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":-4.706101333667567,"unit":"index","family":"agent-arena","index":62.661648232607874,"aboveFit":false,"run":null,"source":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset","measuredAt":"2026-11-11","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lmarena:agent-steerability:overall","benchmarkName":"Agent Arena steerability","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":-6.3874838014162165,"unit":"index","family":"agent-arena","index":61.279288251556636,"aboveFit":false,"run":null,"source":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset","measuredAt":"2026-11-11","stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":0.7,"outputPerMillion":2.1,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":113,"timeToFirstToken":19.37,"source":"BenchLM","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":{"elo":1414.2620056087549,"eloUpper":1423.111760398755,"eloLower":1405.4122508187543,"votes":5742,"rank":136,"rankUpper":null,"rankLower":null,"arenaName":"Mistral Large 4","organization":"mistral","updatedAt":"2026-11-09"},"quality":{"coveragePercent":85,"benchmarkCount":40,"categoryCount":6,"confidence":"medium","state":"available"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:livebench","name":"LiveBench","url":"https://livebench.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:lmarena","name":"LMArena","url":"https://huggingface.co","license":"CC BY 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"glm-5-2","providerId":"z-ai","slug":"glm-5-2","name":"GLM-5.2","provider":"Z.AI","description":"GLM-5.2 is a reasoning model from Z.AI in the GLM-5 family. 50 benchmarks count toward its index, in 6 categories.","logoUrl":null,"releaseDate":"2026-06-16","contextWindowTokens":1048576,"modalities":["text"],"openWeights":true,"details":{"apiId":"z-ai/glm-5.2","maxOutputTokens":943718,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.006,"weightsUrl":"https://huggingface.co/zai-org/GLM-5.2"},"reasoning":true,"overallScore":60.5513661061873,"overallScoreRange":{"low":56.76458277583001,"high":64.3381494365446},"badges":[],"capabilityScores":{"agentic":61.68498178253605,"coding":63.5349881651333,"reasoning":58.55577900770165,"multimodal":null,"knowledge":55.6839386791053,"multilingual":null,"instruction":55.07664677013689,"math":59.59132278655558},"capabilityCounts":{"math":7,"agentic":13,"knowledge":6,"coding":14,"reasoning":8,"instruction":2},"benchmarks":[{"benchmarkId":"aime2026","benchmarkName":"AIME 2026","category":"math","score":99.2,"unit":"percent","family":"aime2026","index":56.709076045437115,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"tau2Bench","benchmarkName":"τ²-Bench Tool-Agent-User Evaluation","category":"agentic","score":99.1,"unit":"percent","family":"tau2-bench","index":68.54953553677001,"aboveFit":false,"run":null,"source":"Victor Barres, Honghua Dong, Soham Ray, Xujie Si, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2506.07982","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hmmtNov2025","benchmarkName":"Harvard-MIT Mathematics Tournament November 2025","category":"math","score":94.4,"unit":"percent","family":"hmmtNov2025","index":56.167804023845115,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hmmtFeb2026","benchmarkName":"Harvard-MIT Mathematics Tournament February 2026","category":"math","score":92.5,"unit":"percent","family":"hmmtFeb2026","index":58.1339504892799,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":91.85606060606061,"unit":"percent","family":"gpqa-diamond","index":61.45696955150403,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.6123207533687733,"costPerTest":null,"latency":null},{"benchmarkId":"gpqa","benchmarkName":"Graduate-Level Google-Proof Q&A","category":"knowledge","score":91.2,"unit":"percent","family":"gpqa-diamond","index":60.85524869053393,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gpqaDiamond","benchmarkName":"GPQA Diamond","category":"knowledge","score":91.2,"unit":"percent","family":"gpqa-diamond","index":60.85524869053393,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmAnswerBench","benchmarkName":"MMAnswerBench","category":"math","score":91,"unit":"percent","family":"mmAnswerBench","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-math:overall","benchmarkName":"LiveBench Mathematics","category":"math","domain":null,"eligible":true,"ranked":true,"score":89.78125,"unit":"percent","family":"livebench:livebench-math:overall","index":64.90468457168097,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Mathematics","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.09593799472295515,"latency":null},{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":89.5,"unit":"percent","family":"gpqa-diamond","index":59.29605514317029,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:mmlu_pro:overall","benchmarkName":"MMLU Pro","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":86.714,"unit":"percent","family":"mmlu-pro","index":53.72998902795577,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","measuredAt":"2026-09-01","stderr":0.334,"costPerTest":0.013838,"latency":51.64},{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":86.38888888888889,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":59.17963204961745,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":4.2197793305105735,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:gpqa:overall","benchmarkName":"GPQA Diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":85.606,"unit":"percent","family":"gpqa-diamond","index":55.724584747032644,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","measuredAt":"2026-09-01","stderr":1.774,"costPerTest":0.05711,"latency":224.664},{"benchmarkId":"vals-ai:legal_bench:overall","benchmarkName":"LegalBench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":84.073,"unit":"percent","family":"vals-ai:legal_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","measuredAt":"2026-10-13","stderr":0.452,"costPerTest":0.001596,"latency":5.766},{"benchmarkId":"vals-ai:medscribe:overall","benchmarkName":"MedScribe","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":83.534,"unit":"percent","family":"vals-ai:medscribe:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medscribe","measuredAt":"2026-10-29","stderr":2.002,"costPerTest":0.044912,"latency":138.372},{"benchmarkId":"vals-ai:swebench:overall","benchmarkName":"SWE-bench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":82.8,"unit":"percent","family":"swe-bench-verified","index":62.29769496996134,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/swebench","measuredAt":"2026-09-01","stderr":1.689,"costPerTest":0.710002,"latency":669.667},{"benchmarkId":"terminalBench21","benchmarkName":"Terminal-Bench 2.1","category":"agentic","score":81,"unit":"percent","family":"terminal-bench-2-1","index":68.51440572966621,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"arc-prize:v1-public-eval:overall","benchmarkName":"ARC-AGI-1 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":80.375,"unit":"percent","family":"arc-prize:v1-public-eval:overall","index":null,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.1747,"latency":null},{"benchmarkId":"livebench:livebench-coding:overall","benchmarkName":"LiveBench Coding","category":"coding","domain":null,"eligible":true,"ranked":true,"score":79.654,"unit":"percent","family":"livebench:livebench-coding:overall","index":66.85027508519791,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Coding","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.050861538461538465,"latency":null},{"benchmarkId":"epoch-ai:epoch-swe-bench-verified:overall","benchmarkName":"SWE-Bench verified","category":"coding","domain":null,"eligible":true,"ranked":true,"score":78.7,"unit":"percent","family":"swe-bench-verified","index":59.2737089010482,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.8656911901656907,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-reasoning:overall","benchmarkName":"LiveBench Reasoning","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":78.625,"unit":"percent","family":"livebench","index":63.473219282442415,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Reasoning","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.12060198019801982,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":78.3,"unit":"percent","family":"lcr","index":61.018154612527894,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"arc-prize:v1-semi-private:overall","benchmarkName":"ARC-AGI-1 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":77,"unit":"percent","family":"arc-prize:v1-semi-private:overall","index":62.38703878921932,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.1916,"latency":null},{"benchmarkId":"mcpAtlas","benchmarkName":"MCP Atlas","category":"agentic","score":76.8,"unit":"percent","family":"mcpAtlas","index":62.77906508992184,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-language:overall","benchmarkName":"LiveBench Language","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":76.24166666666666,"unit":"percent","family":"livebench:livebench-language:overall","index":60.77355156407361,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Language","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.052132000000000005,"latency":null},{"benchmarkId":"livebench:livebench-data-analysis:overall","benchmarkName":"LiveBench Data Analysis","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":73.73966666666666,"unit":"percent","family":"livebench","index":57.14459239993443,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Data%20Analysis","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.11343666666666666,"latency":null},{"benchmarkId":"vals-ai:tax_eval_v2:overall","benchmarkName":"TaxEval v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":73.344,"unit":"percent","family":"vals-ai:tax_eval_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","measuredAt":"2026-09-01","stderr":0.865,"costPerTest":0.015431,"latency":64.942},{"benchmarkId":"aaIfBench","benchmarkName":"Artificial Analysis IFBench","category":"instruction","score":73.3,"unit":"percent","family":"ifbench-family","index":57.52325745906747,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:lcb:overall","benchmarkName":"LiveCodeBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":69.5,"unit":"percent","family":"livecodebench","index":47.01078191918915,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","measuredAt":"2026-09-01","stderr":1.179,"costPerTest":0.129308,"latency":532.222},{"benchmarkId":"vals-ai:terminal-bench-2-1:overall","benchmarkName":"Terminal-Bench 2.1","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":67.79,"unit":"percent","family":"terminal-bench-2-1","index":60.78846636497817,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2-1","measuredAt":"2026-10-01","stderr":0.991,"costPerTest":0.425184,"latency":820.506},{"benchmarkId":"vals-ai:corp_fin_v2:overall","benchmarkName":"CorpFin v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":66.123,"unit":"percent","family":"vals-ai:corp_fin_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","measuredAt":"2026-08-12","stderr":0.932,"costPerTest":0.121857,"latency":24.586},{"benchmarkId":"vals-ai:vibe-code:overall","benchmarkName":"Vibe Code Bench v1.1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":63.961,"unit":"percent","family":"vals-ai:vibe-code:overall","index":65.32765629363864,"aboveFit":false,"run":"OpenHands","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","measuredAt":"2026-11-09","stderr":4.795,"costPerTest":8.488514,"latency":3748.167},{"benchmarkId":"programBench","benchmarkName":"ProgramBench: Can Language Models Rebuild Programs From Scratch?","category":"coding","score":63.7,"unit":"percent","family":"programBench","index":80.66069591014663,"aboveFit":false,"run":null,"source":"John Yang, Kilian Lieret, Jeffrey Ma, Parth Thakkar, Dmitrii Pedchenko, Sten Sootla, Emily McMilin, Pengcheng Yin, Rui Hou, Gabriel Synnaeve, Diyi Yang, Ofir Press","sourceUrl":"https://programbench.com/static/paper.pdf","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-instruction-following:overall","benchmarkName":"LiveBench Instruction Following","category":"instruction","domain":null,"eligible":true,"ranked":true,"score":62.29175000000001,"unit":"percent","family":"livebench:livebench-instruction-following:overall","index":52.630036081206306,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=IF","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.061971,"latency":null},{"benchmarkId":"swePro","benchmarkName":"SWE-bench Pro","category":"coding","score":62.1,"unit":"percent","family":"swePro","index":61.724363505687265,"aboveFit":false,"run":null,"source":"Xiang Deng, Jeff Da, Edwin Pan, Yannis Yiming He, Charles Ide, Kanak Garg, Niklas Lauffer, Andrew Park, Nitin Pasari, Chetan Rane, Karmini Sampath, Maya Krishnan, Srivatsa Kundurthy, Sean Hendryx, Zifan Wang, Chen Bo Calvin Zhang, Noah Jacobson, Bing Liu, Brad Kenstler","sourceUrl":"https://arxiv.org/abs/2509.16941","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:emb:overall","benchmarkName":"EMB","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":61.532,"unit":"percent","family":"vals-ai:emb:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/emb","measuredAt":"2026-10-20","stderr":2.879,"costPerTest":4.465213,"latency":3532.407},{"benchmarkId":"epoch-ai:epoch-frontiermath-tiers-1-3-v2-private:overall","benchmarkName":"FrontierMath-Tiers-1-3-v2-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":59.205776173285194,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tiers-1-3-v2-private:overall","index":62.05775791290037,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.958195251960623,"costPerTest":null,"latency":null},{"benchmarkId":"openHarmonyBench","benchmarkName":"OpenHarmony Bench v1.0","category":"coding","score":58.4,"unit":"percent","family":"openHarmonyBench","index":65.23553655607611,"aboveFit":false,"run":null,"source":"OpenHarmony Bench authors","sourceUrl":"https://arxiv.org/abs/2608.16022","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"cursorBench32","benchmarkName":"cursorBench32","category":"coding","score":55,"unit":"percent","family":"cursorBench32","index":61.02714845532829,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hleWithTools","benchmarkName":"Humanity's Last Exam with tools","category":"agentic","score":54.7,"unit":"percent","family":"hleWithTools","index":64.0462867038348,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-agentic-coding:overall","benchmarkName":"LiveBench Agentic Coding","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":51.76766666666666,"unit":"percent","family":"livebench:livebench-agentic-coding:overall","index":62.549803305598346,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Agentic%20Coding","measuredAt":"2026-06-25","stderr":null,"costPerTest":1.4573083333333334,"latency":null},{"benchmarkId":"aaSciCode","benchmarkName":"Artificial Analysis SciCode","category":"coding","score":51.2,"unit":"percent","family":"scicode-family","index":62.5887851389456,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/scicode","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"terminalBenchHard","benchmarkName":"Terminal-Bench Hard","category":"agentic","score":50.8,"unit":"percent","family":"terminalBenchHard","index":60.78140658622181,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/terminalbench-hard","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:fabv2:overall","benchmarkName":"Finance Agent (v2)","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":49.699,"unit":"percent","family":"vals-ai:fabv2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/fabv2","measuredAt":"2026-10-29","stderr":0.878,"costPerTest":0.713981,"latency":473.158},{"benchmarkId":"nl2Repo","benchmarkName":"NL2Repo","category":"coding","score":48.9,"unit":"percent","family":"nl2Repo","index":61.25927652498958,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"toolathlon","benchmarkName":"Toolathlon","category":"agentic","score":48.2,"unit":"percent","family":"toolathlon","index":61.2211900592577,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:skillsbench:overall","benchmarkName":"SkillsBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":45.077,"unit":"percent","family":"vals-ai:skillsbench:overall","index":58.69465374912853,"aboveFit":false,"run":"OpenHands","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/skillsbench","measuredAt":"2026-10-29","stderr":4.392,"costPerTest":0.811017,"latency":1191.665},{"benchmarkId":"gdpvalAaNormalized","benchmarkName":"GDPval-AA normalized","category":"agentic","score":43.6,"unit":"percent","family":"gdpvalAaNormalized","index":65.20676172695666,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaEnterpriseOpsGym","benchmarkName":"Artificial Analysis EnterpriseOps-Gym","category":"agentic","score":42.7,"unit":"percent","family":"aaEnterpriseOpsGym","index":null,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/enterprise-ops-gym-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaItbench","benchmarkName":"Artificial Analysis ITBench-AA","category":"agentic","score":42.7,"unit":"percent","family":"aaItbench","index":61.82249468717953,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/itbench-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":41.1,"unit":"percent","family":"hle","index":65.62197613403802,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:medcode:overall","benchmarkName":"MedCode","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":40.771,"unit":"percent","family":"vals-ai:medcode:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medcode","measuredAt":"2026-10-29","stderr":2.166,"costPerTest":0.04501,"latency":95.265},{"benchmarkId":"hle","benchmarkName":"Humanity's Last Exam","category":"knowledge","score":40.5,"unit":"percent","family":"hle","index":65.02000048702016,"aboveFit":false,"run":null,"source":"Center for AI Safety, Scale AI, and thousands of expert contributors","sourceUrl":"https://lastexam.ai/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hleNoTools","benchmarkName":"Humanity's Last Exam without tools","category":"knowledge","score":40.5,"unit":"percent","family":"hle","index":65.02000048702016,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:code-migration:overall","benchmarkName":"Code Migration","category":"coding","domain":null,"eligible":true,"ranked":true,"score":37.87,"unit":"percent","family":"vals-ai:code-migration:overall","index":65.3071997931072,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/code-migration","measuredAt":"2026-10-29","stderr":4.143,"costPerTest":12.635818,"latency":6856.367},{"benchmarkId":"tau2-bench:banking-knowledge:overall","benchmarkName":"τ²-bench Banking","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":37.11340206185567,"unit":"percent","family":"tau2-bench","index":27.416895845522188,"aboveFit":false,"run":"xhigh effort · Sierra","source":"Sierra Research","sourceUrl":"https://github.com/sierra-research/tau2-bench","measuredAt":"2026-10-20","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-mirrorcode:overall","benchmarkName":"MirrorCode","category":"coding","domain":null,"eligible":true,"ranked":true,"score":36.666666666666664,"unit":"percent","family":"epoch-ai:epoch-mirrorcode:overall","index":65.66858813337045,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":11.524071884721684,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-simpleqa-verified:overall","benchmarkName":"SimpleQA Verified","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":34.2,"unit":"percent","family":"epoch-ai:epoch-simpleqa-verified:overall","index":52.41067142577705,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.5008706182121803,"costPerTest":null,"latency":null},{"benchmarkId":"apexAgentsAa","benchmarkName":"APEX-Agents-AA","category":"agentic","score":33.7,"unit":"percent","family":"apexAgentsAa","index":65.23479868057109,"aboveFit":false,"run":null,"source":"Artificial Analysis / Mercor","sourceUrl":"https://artificialanalysis.ai/evaluations/apex-agents-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"postTrainBench11","benchmarkName":"PostTrainBench v1.1","category":"coding","score":31.7,"unit":"percent","family":"postTrainBench11","index":67.34917131155613,"aboveFit":false,"run":null,"source":"PostTrainBench team","sourceUrl":"https://posttrainbench.com/?version=v1.1","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:legal_research:overall","benchmarkName":"Legal Research Bench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":31.25,"unit":"percent","family":"vals-ai:legal_research:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_research","measuredAt":"2026-11-09","stderr":3.222,"costPerTest":0.889209,"latency":1023.468},{"benchmarkId":"epoch-ai:epoch-frontiermath-tier-4-v2-private:overall","benchmarkName":"FrontierMath-Tier-4-v2-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":29.268292682926827,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tier-4-v2-private:overall","index":59.986354413128154,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":7.194088392074452,"costPerTest":null,"latency":null},{"benchmarkId":"omniscienceAccuracy","benchmarkName":"Artificial Analysis Omniscience Accuracy","category":"knowledge","score":24.3,"unit":"percent","family":"omniscienceAccuracy","index":41.314170879271316,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"arc-prize:v2-semi-private:overall","benchmarkName":"ARC-AGI-2 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":22.775000000000002,"unit":"percent","family":"arc-prize:v2-semi-private:overall","index":49.6453082169745,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.2501,"latency":null},{"benchmarkId":"epoch-ai:epoch-chess-puzzles:overall","benchmarkName":"Chess Puzzles","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":22,"unit":"percent","family":"epoch-ai:epoch-chess-puzzles:overall","index":50.683010018441045,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":4.163331998932265,"costPerTest":null,"latency":null},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":20.9,"unit":"percent","family":"critpt","index":77.30907733177966,"aboveFit":true,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"arc-prize:v2-public-eval:overall","benchmarkName":"ARC-AGI-2 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":20.830000000000002,"unit":"percent","family":"arc-prize:v2-public-eval:overall","index":null,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.2294,"latency":null},{"benchmarkId":"researchClawBench","benchmarkName":"ResearchClawBench","category":"agentic","score":20.7,"unit":"percent","family":"researchClawBench","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-mystery-game-puzzles:overall","benchmarkName":"Mystery Game Puzzles","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":15,"unit":"percent","family":"epoch-ai:epoch-mystery-game-puzzles:overall","index":50.29722165264777,"aboveFit":false,"run":"medium effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":3.588702812826367,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-ebr-bench:overall","benchmarkName":"EBR-bench","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":9.523809523809524,"unit":"percent","family":"epoch-ai:epoch-ebr-bench:overall","index":56.79751559883462,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"frontierBench","benchmarkName":"Terminal-Bench 3.0","category":"agentic","score":4.59,"unit":"percent","family":"frontierBench","index":56.5641833944908,"aboveFit":false,"run":null,"source":"Ryan Marten, Alex Shaw, Andy Konwinski, Harbor, and the Laude Institute","sourceUrl":"https://www.frontierbench.ai/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lmarena:agent-task-outcome:overall","benchmarkName":"Agent Arena task outcome","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":1.7219690051307857,"unit":"index","family":"agent-arena","index":67.94652978822171,"aboveFit":false,"run":"max effort","source":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset","measuredAt":"2026-11-11","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lmarena:agent-bash-recovery:overall","benchmarkName":"Agent Arena command recovery","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":0.6746311175996889,"unit":"index","family":"agent-arena","index":67.08545385211995,"aboveFit":false,"run":"max effort","source":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset","measuredAt":"2026-10-08","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:programbench:overall","benchmarkName":"ProgramBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":0.5,"unit":"percent","family":"vals-ai:programbench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/programbench","measuredAt":"2026-11-09","stderr":0.5,"costPerTest":12.57696,"latency":7588.932},{"benchmarkId":"mazeBench","benchmarkName":"MazeBench: visual spatial reasoning in a 3D open world","category":"agentic","score":0,"unit":"percent","family":"mazeBench","index":61.97866734834783,"aboveFit":false,"run":null,"source":"Jonathan Pappas and David Pappas","sourceUrl":"https://mazebench.com/blog?post=maze-bench-results","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:srebench:overall","benchmarkName":"SRE Bench","category":null,"domain":"Other","eligible":false,"ranked":true,"score":0,"unit":"percent","family":"vals-ai:srebench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/srebench","measuredAt":"2026-11-05","stderr":0,"costPerTest":26.575264,"latency":8960.277},{"benchmarkId":"lmarena:agent-steerability:overall","benchmarkName":"Agent Arena steerability","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":-1.0316048829528635,"unit":"index","family":"agent-arena","index":65.68266037078259,"aboveFit":false,"run":"max effort","source":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset","measuredAt":"2026-11-11","stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":0.007,"outputPerMillion":6.8,"batch":{"inputPerMillion":0.322,"outputPerMillion":0.989},"freeTier":true,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":111,"timeToFirstToken":0.62,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":{"elo":1469.4428527485325,"eloUpper":1473.4928180361717,"eloLower":1465.3928874608932,"votes":54173,"rank":32,"rankUpper":null,"rankLower":null,"arenaName":"GLM-5.2-max","organization":"zai","updatedAt":"2026-11-09"},"quality":{"coveragePercent":85,"benchmarkCount":50,"categoryCount":6,"confidence":"medium","state":"available"},"hosts":[{"host":"deepinfra","name":"Deep Infra","kind":"host","url":"https://deepinfra.com/models","inputPerMillion":0.75,"outputPerMillion":2.4,"cacheReadPerMillion":0.14,"contextTokens":1048576,"maxOutputTokens":32768,"deprecated":false},{"host":"coreweave","name":"CoreWeave","kind":"host","url":"https://docs.wandb.ai/inference","inputPerMillion":0.62,"outputPerMillion":2.42,"cacheReadPerMillion":0.14,"contextTokens":1048576,"maxOutputTokens":1048576,"deprecated":false},{"host":"vultr","name":"Vultr","kind":"cloud","url":"https://api.vultrinference.com/","inputPerMillion":0.75,"outputPerMillion":3,"cacheReadPerMillion":null,"contextTokens":1048576,"maxOutputTokens":131072,"deprecated":false},{"host":"gmi-cloud","name":"GMI Cloud","kind":"host","url":"https://docs.gmicloud.ai/inference-engine/api-reference/llm-api-reference","inputPerMillion":0.979,"outputPerMillion":3.08,"cacheReadPerMillion":0.182,"contextTokens":1000000,"maxOutputTokens":131072,"deprecated":false},{"host":"chutes","name":"Chutes","kind":"host","url":"https://llm.chutes.ai/v1/models","inputPerMillion":1.25,"outputPerMillion":3.95,"cacheReadPerMillion":0.125,"contextTokens":1048576,"maxOutputTokens":65535,"deprecated":false},{"host":"siliconflow","name":"SiliconFlow","kind":"host","url":"https://cloud.siliconflow.com/models","inputPerMillion":1.302,"outputPerMillion":4.092,"cacheReadPerMillion":0.26,"contextTokens":1049000,"maxOutputTokens":262000,"deprecated":false},{"host":"volcengine","name":"Volcengine Ark","kind":"cloud","url":"https://www.volcengine.com/docs/82379/1330310","inputPerMillion":1.18747,"outputPerMillion":4.15615,"cacheReadPerMillion":0.29687,"contextTokens":1000000,"maxOutputTokens":131072,"deprecated":false},{"host":"zhipu-ai","name":"Zhipu AI","kind":"maker","url":"https://docs.bigmodel.cn/cn/guide/start/pricing","inputPerMillion":1.19516,"outputPerMillion":4.18306,"cacheReadPerMillion":0.29879,"contextTokens":1000000,"maxOutputTokens":131072,"deprecated":false},{"host":"alibaba-cloud","name":"Alibaba Cloud","kind":"host","url":"https://www.alibabacloud.com/help/en/model-studio/models","inputPerMillion":1.4,"outputPerMillion":4.4,"cacheReadPerMillion":0.28,"contextTokens":1000000,"maxOutputTokens":131072,"deprecated":false},{"host":"arcee-ai","name":"Arcee AI","kind":"host","url":"https://docs.arcee.ai","inputPerMillion":1.4,"outputPerMillion":4.4,"cacheReadPerMillion":0.26,"contextTokens":262144,"maxOutputTokens":131072,"deprecated":false},{"host":"baseten","name":"Baseten","kind":"host","url":"https://docs.baseten.co/inference/model-apis/overview","inputPerMillion":1.4,"outputPerMillion":4.4,"cacheReadPerMillion":0.14,"contextTokens":1048576,"maxOutputTokens":262144,"deprecated":false},{"host":"cloudflare","name":"Cloudflare Workers AI","kind":"cloud","url":"https://developers.cloudflare.com/workers-ai/models/","inputPerMillion":1.4,"outputPerMillion":4.4,"cacheReadPerMillion":0.26,"contextTokens":262144,"maxOutputTokens":256000,"deprecated":true},{"host":"crusoe","name":"Crusoe","kind":"host","url":"https://docs.crusoecloud.com/managed-inference/overview","inputPerMillion":1.4,"outputPerMillion":4.4,"cacheReadPerMillion":0.26,"contextTokens":1000000,"maxOutputTokens":131072,"deprecated":true},{"host":"databricks","name":"Databricks","kind":"cloud","url":"https://docs.databricks.com/aws/en/machine-learning/foundation-models/","inputPerMillion":1.4,"outputPerMillion":4.4,"cacheReadPerMillion":0.26,"contextTokens":1000000,"maxOutputTokens":131072,"deprecated":false},{"host":"digitalocean","name":"DigitalOcean","kind":"cloud","url":"https://docs.digitalocean.com/products/gradient-ai-platform/details/models/","inputPerMillion":1.4,"outputPerMillion":4.4,"cacheReadPerMillion":0.21,"contextTokens":262144,"maxOutputTokens":131072,"deprecated":false},{"host":"friendli","name":"Friendli","kind":"host","url":"https://friendli.ai/docs/guides/serverless_endpoints/introduction","inputPerMillion":1.4,"outputPerMillion":4.4,"cacheReadPerMillion":0.26,"contextTokens":1048576,"maxOutputTokens":131072,"deprecated":false},{"host":"google-vertex","name":"Google Vertex AI","kind":"cloud","url":"https://cloud.google.com/vertex-ai/generative-ai/docs/models","inputPerMillion":1.4,"outputPerMillion":4.4,"cacheReadPerMillion":0.14,"contextTokens":1000000,"maxOutputTokens":64000,"deprecated":false},{"host":"mistral","name":"Mistral","kind":"host","url":"https://docs.mistral.ai/getting-started/models/","inputPerMillion":1.4,"outputPerMillion":4.4,"cacheReadPerMillion":0.14,"contextTokens":1048576,"maxOutputTokens":131072,"deprecated":true},{"host":"nebius","name":"Nebius","kind":"host","url":"https://docs.tokenfactory.nebius.com/","inputPerMillion":1.4,"outputPerMillion":4.4,"cacheReadPerMillion":0.14,"contextTokens":1048576,"maxOutputTokens":1048576,"deprecated":false},{"host":"novita-ai","name":"Novita AI","kind":"host","url":"https://novita.ai/docs/guides/introduction","inputPerMillion":1.4,"outputPerMillion":4.4,"cacheReadPerMillion":0.26,"contextTokens":1048576,"maxOutputTokens":131072,"deprecated":false},{"host":"together-ai","name":"Together AI","kind":"host","url":"https://docs.together.ai/docs/serverless-models","inputPerMillion":1.4,"outputPerMillion":4.4,"cacheReadPerMillion":0.26,"contextTokens":1048575,"maxOutputTokens":164000,"deprecated":false},{"host":"z-ai","name":"Z.AI","kind":"maker","url":"https://docs.z.ai/guides/overview/pricing","inputPerMillion":1.4,"outputPerMillion":4.4,"cacheReadPerMillion":0.26,"contextTokens":1000000,"maxOutputTokens":131072,"deprecated":false},{"host":"berget-ai","name":"Berget AI","kind":"host","url":"https://api.berget.ai","inputPerMillion":1.54,"outputPerMillion":4.84,"cacheReadPerMillion":null,"contextTokens":524288,"maxOutputTokens":32768,"deprecated":true},{"host":"regolo","name":"Regolo","kind":"host","url":"https://docs.regolo.ai/","inputPerMillion":2.31,"outputPerMillion":6,"cacheReadPerMillion":null,"contextTokens":96000,"maxOutputTokens":96000,"deprecated":false},{"host":"scaleway","name":"Scaleway","kind":"cloud","url":"https://www.scaleway.com/en/docs/generative-apis/","inputPerMillion":2.0187,"outputPerMillion":6.16825,"cacheReadPerMillion":null,"contextTokens":256000,"maxOutputTokens":16384,"deprecated":false},{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":0.007,"outputPerMillion":6.8,"cacheReadPerMillion":0.006,"contextTokens":1048576,"maxOutputTokens":943718,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:livebench","name":"LiveBench","url":"https://livebench.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:arc-prize-foundation","name":"ARC Prize Foundation","url":"https://arcprize.org","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:sierra-research","name":"Sierra Research","url":"https://github.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:lmarena","name":"LMArena","url":"https://huggingface.co","license":"CC BY 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"qwen3-7-max","providerId":"alibaba","slug":"qwen3-7-max","name":"Qwen3.7 Max","provider":"Alibaba","description":"Qwen3.7 Max is a reasoning model from Alibaba. 54 benchmarks count toward its index, in 7 categories.","logoUrl":null,"releaseDate":"2026-05-16","contextWindowTokens":1000000,"modalities":["text"],"openWeights":false,"details":{"apiId":"qwen/qwen3.7-max","maxOutputTokens":131072,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.295,"weightsUrl":null},"reasoning":true,"overallScore":60.53073941856152,"overallScoreRange":{"low":57.3302811329613,"high":63.73119770416174},"badges":[],"capabilityScores":{"agentic":60.48928136640412,"coding":59.80764157252949,"reasoning":60.41155015057611,"multimodal":null,"knowledge":58.57358098756344,"multilingual":57.225549825392605,"instruction":64.36576789326105,"math":62.13894793900508},"capabilityCounts":{"math":6,"knowledge":9,"agentic":16,"instruction":3,"coding":11,"reasoning":7,"multilingual":2},"benchmarks":[{"benchmarkId":"hmmtFeb2026","benchmarkName":"Harvard-MIT Mathematics Tournament February 2026","category":"math","score":97.1,"unit":"percent","family":"hmmtFeb2026","index":61.35312071220199,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":95.55555555555556,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":64.51274413550408,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":3.1067790907534736,"costPerTest":null,"latency":null},{"benchmarkId":"mmluRedux","benchmarkName":"MMLU-Redux","category":"knowledge","score":95,"unit":"percent","family":"mmluRedux","index":51.31744460011175,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"tau2Bench","benchmarkName":"τ²-Bench Tool-Agent-User Evaluation","category":"agentic","score":94.7,"unit":"percent","family":"tau2-bench","index":65.62981384431495,"aboveFit":false,"run":null,"source":"Victor Barres, Honghua Dong, Soham Ray, Xujie Si, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2506.07982","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"ifeval","benchmarkName":"Instruction-Following Eval","category":"instruction","score":94.3,"unit":"percent","family":"ifeval","index":53.09454352454052,"aboveFit":false,"run":null,"source":"Jeffrey Zhou, Tianjian Lu, Swaroop Mishra, Siddhartha Brahma, Sujoy Basu, Yi Luan, Denny Zhou, Le Hou","sourceUrl":"https://arxiv.org/abs/2311.07911","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gpqa","benchmarkName":"Graduate-Level Google-Proof Q&A","category":"knowledge","score":92.4,"unit":"percent","family":"gpqa-diamond","index":61.95585590043767,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gpqaDiamond","benchmarkName":"GPQA Diamond","category":"knowledge","score":92.4,"unit":"percent","family":"gpqa-diamond","index":61.95585590043767,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":92.3,"unit":"percent","family":"gpqa-diamond","index":61.86413863294568,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"liveCodeBenchV6","benchmarkName":"LiveCodeBench v6","category":"coding","score":91.6,"unit":"percent","family":"liveCodeBenchV6","index":59.625905779901885,"aboveFit":false,"run":null,"source":"LiveCodeBench maintainers","sourceUrl":"https://github.com/LiveCodeBench/LiveCodeBench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":90.9090909090909,"unit":"percent","family":"gpqa-diamond","index":60.58843482146635,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.0482086775424224,"costPerTest":null,"latency":null},{"benchmarkId":"mrcrv2","benchmarkName":"MRCRv2","category":"reasoning","score":90.4,"unit":"percent","family":"mrcrv2","index":66.4058991385846,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmmlu","benchmarkName":"MMMLU","category":"knowledge","score":90.3,"unit":"percent","family":"mmmlu","index":48.91800439593626,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:gpqa:overall","benchmarkName":"GPQA Diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":90.152,"unit":"percent","family":"gpqa-diamond","index":59.89405172721799,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","measuredAt":"2026-09-01","stderr":1.497,"costPerTest":0.06149,"latency":126.611},{"benchmarkId":"imoAnswerBench","benchmarkName":"IMOAnswerBench","category":"math","score":90,"unit":"percent","family":"imoAnswerBench","index":61.601470180263476,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmluPro","benchmarkName":"Massive Multitask Language Understanding Professional","category":"knowledge","score":89.6,"unit":"percent","family":"mmlu-pro","index":57.366003582635614,"aboveFit":false,"run":null,"source":"Yubo Wang, Xueguang Ma, Ge Zhang, Yuansheng Ni, Abhranil Chandra, Shiguang Guo, Weiming Ren, Aaran Arulraj, Xuan He, Ziyan Jiang, Tianle Li, Max Ku, Kai Wang, Alex Zhuang, Rongqi Fan, Xiang Yue, Wenhu Chen","sourceUrl":"https://arxiv.org/abs/2406.01574","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:mmlu_pro:overall","benchmarkName":"MMLU Pro","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":89.311,"unit":"percent","family":"mmlu-pro","index":57.00189817504641,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","measuredAt":"2026-09-01","stderr":0.305,"costPerTest":0.01678,"latency":29.15},{"benchmarkId":"maxife","benchmarkName":"MAXIFE","category":"multilingual","score":89.2,"unit":"percent","family":"maxife","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:lcb:overall","benchmarkName":"LiveCodeBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":87.057,"unit":"percent","family":"livecodebench","index":61.57854867679025,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","measuredAt":"2026-09-01","stderr":0.974,"costPerTest":0.086751,"latency":192.175},{"benchmarkId":"mmluProX","benchmarkName":"MMLU-ProX","category":"multilingual","score":87,"unit":"percent","family":"mmluProX","index":56.31837552744226,"aboveFit":false,"run":null,"source":"MMLU-ProX authors","sourceUrl":"https://arxiv.org/abs/2503.10497","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"polyMath","benchmarkName":"PolyMath","category":"multilingual","score":86.5,"unit":"percent","family":"polyMath","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"include","benchmarkName":"INCLUDE","category":"multilingual","score":86.2,"unit":"percent","family":"include","index":58.13272412334296,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-math:overall","benchmarkName":"LiveBench Mathematics","category":"math","domain":null,"eligible":true,"ranked":true,"score":85.24825,"unit":"percent","family":"livebench:livebench-math:overall","index":58.58695501427451,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Mathematics","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.12712480211081795,"latency":null},{"benchmarkId":"vals-ai:legal_bench:overall","benchmarkName":"LegalBench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":84.913,"unit":"percent","family":"vals-ai:legal_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","measuredAt":"2026-10-13","stderr":0.413,"costPerTest":0.005967,"latency":11.853},{"benchmarkId":"livebench:livebench-reasoning:overall","benchmarkName":"LiveBench Reasoning","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":83.3365,"unit":"percent","family":"livebench","index":69.57665655528692,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Reasoning","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.10212970297029704,"latency":null},{"benchmarkId":"aaIfBench","benchmarkName":"Artificial Analysis IFBench","category":"instruction","score":80.5,"unit":"percent","family":"ifbench-family","index":64.2517338858399,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"sweVerified","benchmarkName":"Software Engineering Benchmark Verified","category":"coding","score":80.4,"unit":"percent","family":"swe-bench-verified","index":60.52755678328048,"aboveFit":false,"run":null,"source":"Carlos E. Jimenez, John Yang, Alexander Wettig, Shunyu Yao, Kexin Pei, Ofir Press, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2310.06770","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-language:overall","benchmarkName":"LiveBench Language","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":79.73899999999999,"unit":"percent","family":"livebench:livebench-language:overall","index":64.7283698031222,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Language","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.036788,"latency":null},{"benchmarkId":"vals-ai:medscribe:overall","benchmarkName":"MedScribe","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":79.396,"unit":"percent","family":"vals-ai:medscribe:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medscribe","measuredAt":"2026-10-29","stderr":1.907,"costPerTest":0.06907,"latency":108.046},{"benchmarkId":"ifBench","benchmarkName":"Instruction Following Benchmark","category":"instruction","score":79.1,"unit":"percent","family":"ifbench-family","index":62.94341902507859,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":79,"unit":"percent","family":"lcr","index":61.49841098043786,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-swe-bench-verified:overall","benchmarkName":"SWE-Bench verified","category":"coding","domain":null,"eligible":true,"ranked":true,"score":77.27272727272727,"unit":"percent","family":"swe-bench-verified","index":58.2210130854842,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.906835447169652,"costPerTest":null,"latency":null},{"benchmarkId":"mcpAtlas","benchmarkName":"MCP Atlas","category":"agentic","score":76.4,"unit":"percent","family":"mcpAtlas","index":62.509706851493135,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:tax_eval_v2:overall","benchmarkName":"TaxEval v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":75.306,"unit":"percent","family":"vals-ai:tax_eval_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","measuredAt":"2026-09-01","stderr":0.839,"costPerTest":0.028122,"latency":57.677},{"benchmarkId":"bfclV4","benchmarkName":"Berkeley Function Calling Leaderboard v4","category":"agentic","score":75,"unit":"percent","family":"bfclV4","index":54.80309419967615,"aboveFit":false,"run":null,"source":"Shishir G. Patil et al.","sourceUrl":"https://gorilla.cs.berkeley.edu/leaderboard.html","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-coding:overall","benchmarkName":"LiveBench Coding","category":"coding","domain":null,"eligible":true,"ranked":true,"score":74.219,"unit":"percent","family":"livebench:livebench-coding:overall","index":59.206558493165176,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Coding","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.02653760683760684,"latency":null},{"benchmarkId":"livebench:livebench-instruction-following:overall","benchmarkName":"LiveBench Instruction Following","category":"instruction","domain":null,"eligible":true,"ranked":true,"score":74.0415,"unit":"percent","family":"livebench:livebench-instruction-following:overall","index":76.4051836997834,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=IF","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.0790895,"latency":null},{"benchmarkId":"superGpqa","benchmarkName":"SuperGPQA: Scaling LLM Evaluation Across 285 Graduate Disciplines","category":"knowledge","score":73.6,"unit":"percent","family":"superGpqa","index":57.416788945464525,"aboveFit":false,"run":null,"source":"Xiaoxuan Du, Yao Yao, Kexin Ma, Bowen Wang, Tianyu Zheng, Kaiyan Zhu, Yiming Zhang, Yutao Zhu, Jiawei Zhou, Jingren Zhou","sourceUrl":"https://arxiv.org/abs/2502.14739","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-data-analysis:overall","benchmarkName":"LiveBench Data Analysis","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":71.786,"unit":"percent","family":"livebench","index":54.61374618279611,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Data%20Analysis","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.12076333333333333,"latency":null},{"benchmarkId":"terminalBench2","benchmarkName":"Terminal-Bench 2.0","category":"agentic","score":69.7,"unit":"percent","family":"terminalBench2","index":63.47841965924014,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:swebench:overall","benchmarkName":"SWE-bench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":68.8,"unit":"percent","family":"swe-bench-verified","index":51.97188888098965,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/swebench","measuredAt":"2026-09-01","stderr":2.074,"costPerTest":3.227207,"latency":976.131},{"benchmarkId":"clawEval","benchmarkName":"Claw-Eval","category":"agentic","score":65.2,"unit":"percent","family":"clawEval","index":59.40318135537407,"aboveFit":false,"run":null,"source":"Bowen Ye, Rang Li, Qibin Yang, Yuanxin Liu, Linli Yao, Hanglong Lv, Zhihui Xie, Chenxin An, Lei Li, Lingpeng Kong, Qi Liu, Zhifang Sui, Tong Yang","sourceUrl":"https://arxiv.org/abs/2604.06132","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tiers-1-3-v2-private:overall","benchmarkName":"FrontierMath-Tiers-1-3-v2-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":64.56140350877193,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tiers-1-3-v2-private:overall","index":64.87670152601203,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.838347520543562,"costPerTest":null,"latency":null},{"benchmarkId":"qwenClawBench","benchmarkName":"QwenClawBench","category":"agentic","score":64.3,"unit":"percent","family":"qwenClawBench","index":60.79071777790381,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gertLabs","benchmarkName":"Gert Labs Composite Game Benchmark","category":"agentic","score":64.27,"unit":"percent","family":"gertLabs","index":67.58725475750853,"aboveFit":false,"run":null,"source":"Gert Labs","sourceUrl":"https://gertlabs.com/rankings","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:corp_fin_v2:overall","benchmarkName":"CorpFin v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":63.714,"unit":"percent","family":"vals-ai:corp_fin_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","measuredAt":"2026-08-12","stderr":0.946,"costPerTest":0.219883,"latency":19.934},{"benchmarkId":"vals-ai:terminal-bench-2-1:overall","benchmarkName":"Terminal-Bench 2.1","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":61.049,"unit":"percent","family":"terminal-bench-2-1","index":56.84595635306583,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2-1","measuredAt":"2026-10-01","stderr":0.375,"costPerTest":0.33233,"latency":935.196},{"benchmarkId":"swePro","benchmarkName":"SWE-bench Pro","category":"coding","score":60.6,"unit":"percent","family":"swePro","index":60.44694853390529,"aboveFit":false,"run":null,"source":"Xiang Deng, Jeff Da, Edwin Pan, Yannis Yiming He, Charles Ide, Kanak Garg, Niklas Lauffer, Andrew Park, Nitin Pasari, Chetan Rane, Karmini Sampath, Maya Krishnan, Srivatsa Kundurthy, Sean Hendryx, Zifan Wang, Chen Bo Calvin Zhang, Noah Jacobson, Bing Liu, Brad Kenstler","sourceUrl":"https://arxiv.org/abs/2509.16941","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:terminal-bench-2:overall","benchmarkName":"Terminal-Bench 2.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":59.176,"unit":"percent","family":"vals-ai:terminal-bench-2:overall","index":62.1818001777441,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2","measuredAt":"2026-06-04","stderr":1.35,"costPerTest":0.719164,"latency":664.843},{"benchmarkId":"nova63","benchmarkName":"NOVA-63","category":"multilingual","score":59,"unit":"percent","family":"nova63","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:emb:overall","benchmarkName":"EMB","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":57.041,"unit":"percent","family":"vals-ai:emb:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/emb","measuredAt":"2026-10-20","stderr":2.853,"costPerTest":2.857267,"latency":1957.165},{"benchmarkId":"epoch-ai:epoch-simpleqa-verified:overall","benchmarkName":"SimpleQA Verified","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":55.76730190571715,"unit":"percent","family":"epoch-ai:epoch-simpleqa-verified:overall","index":70.37813783848979,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.5737359397180177,"costPerTest":null,"latency":null},{"benchmarkId":"hleWithTools","benchmarkName":"Humanity's Last Exam with tools","category":"agentic","score":53.5,"unit":"percent","family":"hleWithTools","index":63.02378133020048,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"sciCode","benchmarkName":"Scientific Code Benchmark","category":"coding","score":53.5,"unit":"percent","family":"scicode-family","index":65.58727477569948,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"openHarmonyBench","benchmarkName":"OpenHarmony Bench v1.0","category":"coding","score":53.4,"unit":"percent","family":"openHarmonyBench","index":60.781406586221806,"aboveFit":false,"run":null,"source":"OpenHarmony Bench authors","sourceUrl":"https://arxiv.org/abs/2608.16022","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"terminalBenchHard","benchmarkName":"Terminal-Bench Hard","category":"agentic","score":50.8,"unit":"percent","family":"terminalBenchHard","index":60.78140658622181,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/terminalbench-hard","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaSciCode","benchmarkName":"Artificial Analysis SciCode","category":"coding","score":49.5,"unit":"percent","family":"scicode-family","index":60.37251019004056,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/scicode","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vitaBench","benchmarkName":"VITA-Bench","category":"agentic","score":47.9,"unit":"percent","family":"vitaBench","index":60.781406586221806,"aboveFit":false,"run":null,"source":"Meituan LongCat Team","sourceUrl":"https://vitabench.github.io/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:fabv2:overall","benchmarkName":"Finance Agent (v2)","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":47.776,"unit":"percent","family":"vals-ai:fabv2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/fabv2","measuredAt":"2026-10-29","stderr":0.727,"costPerTest":0.553214,"latency":635.178},{"benchmarkId":"vals-ai:vibe-code:overall","benchmarkName":"Vibe Code Bench v1.1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":47.671,"unit":"percent","family":"vals-ai:vibe-code:overall","index":59.44996835854364,"aboveFit":false,"run":"OpenHands","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","measuredAt":"2026-11-09","stderr":4.63,"costPerTest":11.400322,"latency":4689.627},{"benchmarkId":"nl2Repo","benchmarkName":"NL2Repo","category":"coding","score":47.2,"unit":"percent","family":"nl2Repo","index":60.060719378122435,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:ioi-v1:overall","benchmarkName":"IOI v1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":46.75,"unit":"percent","family":"vals-ai:ioi-v1:overall","index":63.401181693601124,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/ioi-v1","measuredAt":"2026-08-09","stderr":10.263,"costPerTest":4.859648,"latency":4380.294},{"benchmarkId":"aaEnterpriseOpsGym","benchmarkName":"Artificial Analysis EnterpriseOps-Gym","category":"agentic","score":45,"unit":"percent","family":"aaEnterpriseOpsGym","index":null,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/enterprise-ops-gym-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"apex","benchmarkName":"Apex","category":"math","score":44.5,"unit":"percent","family":"apex","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-agentic-coding:overall","benchmarkName":"LiveBench Agentic Coding","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":43.586000000000006,"unit":"percent","family":"livebench:livebench-agentic-coding:overall","index":54.112189819798175,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Agentic%20Coding","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.7961263888888889,"latency":null},{"benchmarkId":"aaItbench","benchmarkName":"Artificial Analysis ITBench-AA","category":"agentic","score":42.5,"unit":"percent","family":"aaItbench","index":61.548527350664315,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/itbench-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hle","benchmarkName":"Humanity's Last Exam","category":"knowledge","score":41.4,"unit":"percent","family":"hle","index":65.92296395754695,"aboveFit":false,"run":null,"source":"Center for AI Safety, Scale AI, and thousands of expert contributors","sourceUrl":"https://lastexam.ai/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":40.5,"unit":"percent","family":"hle","index":65.02000048702016,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:medcode:overall","benchmarkName":"MedCode","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":38.751,"unit":"percent","family":"vals-ai:medcode:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medcode","measuredAt":"2026-10-29","stderr":2.196,"costPerTest":0.042362,"latency":28.755},{"benchmarkId":"epoch-ai:epoch-frontiermath-tier-4-v2-private:overall","benchmarkName":"FrontierMath-Tier-4-v2-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":34.146341463414636,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tier-4-v2-private:overall","index":61.90269606577442,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":7.497768853141171,"costPerTest":null,"latency":null},{"benchmarkId":"gdpvalAaNormalized","benchmarkName":"GDPval-AA normalized","category":"agentic","score":31.4,"unit":"percent","family":"gdpvalAaNormalized","index":56.48404523431631,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"omniscienceAccuracy","benchmarkName":"Artificial Analysis Omniscience Accuracy","category":"knowledge","score":31.1,"unit":"percent","family":"omniscienceAccuracy","index":49.883911570876165,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-chess-puzzles:overall","benchmarkName":"Chess Puzzles","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":27,"unit":"percent","family":"epoch-ai:epoch-chess-puzzles:overall","index":54.47808599419153,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":4.461960433384737,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-mystery-game-puzzles:overall","benchmarkName":"Mystery Game Puzzles","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":26,"unit":"percent","family":"epoch-ai:epoch-mystery-game-puzzles:overall","index":58.81578551640466,"aboveFit":false,"run":"none effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":4.4084400227680804,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:legal_research:overall","benchmarkName":"Legal Research Bench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":25.481,"unit":"percent","family":"vals-ai:legal_research:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_research","measuredAt":"2026-11-09","stderr":3.029,"costPerTest":1.400701,"latency":808.802},{"benchmarkId":"researchClawBench","benchmarkName":"ResearchClawBench","category":"agentic","score":18.7,"unit":"percent","family":"researchClawBench","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:tax_agent_bench:overall","benchmarkName":"Tax Agent Bench","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":15.832,"unit":"percent","family":"vals-ai:tax_agent_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench","measuredAt":"2026-11-09","stderr":2.617,"costPerTest":0.435494,"latency":209.726},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":13.4,"unit":"percent","family":"critpt","index":62.789952456538,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:code-migration:overall","benchmarkName":"Code Migration","category":"coding","domain":null,"eligible":true,"ranked":true,"score":13.069,"unit":"percent","family":"vals-ai:code-migration:overall","index":52.13191422921859,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/code-migration","measuredAt":"2026-10-29","stderr":2.858,"costPerTest":1.932899,"latency":1531.852},{"benchmarkId":"epoch-ai:epoch-ebr-bench:overall","benchmarkName":"EBR-bench","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":9.523809523809524,"unit":"percent","family":"epoch-ai:epoch-ebr-bench:overall","index":56.79751559883462,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lmarena:agent-bash-recovery:overall","benchmarkName":"Agent Arena command recovery","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":-1.560714420881748,"unit":"index","family":"agent-arena","index":65.24764938140028,"aboveFit":false,"run":null,"source":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset","measuredAt":"2026-10-08","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lmarena:agent-task-outcome:overall","benchmarkName":"Agent Arena task outcome","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":-10.537651253451074,"unit":"index","family":"agent-arena","index":57.867199978722326,"aboveFit":false,"run":null,"source":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset","measuredAt":"2026-11-11","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lmarena:agent-steerability:overall","benchmarkName":"Agent Arena steerability","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":-12.783620440822219,"unit":"index","family":"agent-arena","index":56.02066120313176,"aboveFit":false,"run":null,"source":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset","measuredAt":"2026-11-11","stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":1.475,"outputPerMillion":4.425,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":68,"timeToFirstToken":1.283,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":{"elo":1474.796238733963,"eloUpper":1484.4994095011552,"eloLower":1465.0930679667708,"votes":3904,"rank":26,"rankUpper":null,"rankLower":null,"arenaName":"Qwen3.7-Max-Preview","organization":"alibaba","updatedAt":"2026-11-09"},"quality":{"coveragePercent":90,"benchmarkCount":54,"categoryCount":7,"confidence":"high","state":"available"},"hosts":[{"host":"novita-ai","name":"Novita AI","kind":"host","url":"https://novita.ai/docs/guides/introduction","inputPerMillion":1.25,"outputPerMillion":3.75,"cacheReadPerMillion":0.25,"contextTokens":1000000,"maxOutputTokens":65536,"deprecated":false},{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":1.475,"outputPerMillion":4.425,"cacheReadPerMillion":0.295,"contextTokens":1000000,"maxOutputTokens":131072,"deprecated":false},{"host":"deepinfra","name":"Deep Infra","kind":"host","url":"https://deepinfra.com/models","inputPerMillion":2.5,"outputPerMillion":7.5,"cacheReadPerMillion":0.5,"contextTokens":256000,"maxOutputTokens":131072,"deprecated":false},{"host":"gmi-cloud","name":"GMI Cloud","kind":"host","url":"https://docs.gmicloud.ai/inference-engine/api-reference/llm-api-reference","inputPerMillion":2.5,"outputPerMillion":7.5,"cacheReadPerMillion":0.25,"contextTokens":1000000,"maxOutputTokens":131072,"deprecated":false},{"host":"together-ai","name":"Together AI","kind":"host","url":"https://docs.together.ai/docs/serverless-models","inputPerMillion":2.5,"outputPerMillion":7.5,"cacheReadPerMillion":0.5,"contextTokens":1000000,"maxOutputTokens":131072,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:livebench","name":"LiveBench","url":"https://livebench.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:lmarena","name":"LMArena","url":"https://huggingface.co","license":"CC BY 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"sage-a1-5","providerId":"internscience","slug":"sage-a1-5","name":"Sage-A1.5","provider":"InternScience","description":"Sage-A1.5 is a reasoning model from InternScience. 9 benchmarks count toward its index, in 4 categories.","logoUrl":null,"releaseDate":"2026-10-15","contextWindowTokens":262000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":true,"overallScore":60.52518469856023,"overallScoreRange":{"low":51.13518469856023,"high":69.91518469856024},"badges":[],"capabilityScores":{"agentic":59.38329097942892,"coding":57.275615375651014,"reasoning":69.21867718917991,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":65.27969556215234,"math":null},"capabilityCounts":{"agentic":4,"instruction":1,"reasoning":2,"coding":2},"benchmarks":[{"benchmarkId":"gaia","benchmarkName":"General AI Assistants","category":"agentic","score":95.1,"unit":"percent","family":"gaia","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"pinchBench","benchmarkName":"PinchBench","category":"agentic","score":90,"unit":"percent","family":"pinchBench","index":49.847313385590816,"aboveFit":false,"run":null,"source":"Kilo Code","sourceUrl":"https://pinchbench.com/about","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"browseComp","benchmarkName":"BrowseComp","category":"agentic","score":81.7,"unit":"percent","family":"browseComp","index":64.45361220074204,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"ifBench","benchmarkName":"Instruction Following Benchmark","category":"instruction","score":81.6,"unit":"percent","family":"ifbench-family","index":65.27969556215234,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":81,"unit":"percent","family":"lcr","index":62.87057203160918,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mleBenchLite","benchmarkName":"MLE-Bench Lite","category":"agentic","score":65.2,"unit":"percent","family":"mleBenchLite","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"swePro","benchmarkName":"SWE-bench Pro","category":"coding","score":61.2,"unit":"percent","family":"swePro","index":60.95791452261808,"aboveFit":false,"run":null,"source":"Xiang Deng, Jeff Da, Edwin Pan, Yannis Yiming He, Charles Ide, Kanak Garg, Niklas Lauffer, Andrew Park, Nitin Pasari, Chetan Rane, Karmini Sampath, Maya Krishnan, Srivatsa Kundurthy, Sean Hendryx, Zifan Wang, Chen Bo Calvin Zhang, Noah Jacobson, Bing Liu, Brad Kenstler","sourceUrl":"https://arxiv.org/abs/2509.16941","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hleWithTools","benchmarkName":"Humanity's Last Exam with tools","category":"agentic","score":55.1,"unit":"percent","family":"hleWithTools","index":64.38712182837956,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"frontierScienceResearch","benchmarkName":"FrontierScience Research","category":"knowledge","score":48.3,"unit":"percent","family":"frontierScienceResearch","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vitaBench","benchmarkName":"VITA-Bench","category":"agentic","score":45.4,"unit":"percent","family":"vitaBench","index":58.84511650300327,"aboveFit":false,"run":null,"source":"Meituan LongCat Team","sourceUrl":"https://vitabench.github.io/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"sciCode","benchmarkName":"Scientific Code Benchmark","category":"coding","score":44.3,"unit":"percent","family":"scicode-family","index":53.59331622868394,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":20,"unit":"percent","family":"critpt","index":75.56678234675066,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":60,"benchmarkCount":9,"categoryCount":4,"confidence":"low","state":"available"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"deepseekv4flashvisionexp","providerId":"deepseek","slug":"deepseek-deepseek-v4-flash-vision-exp","name":"DeepSeek V4 Flash Vision Exp","provider":"DeepSeek","description":"DeepSeek V4 Flash Vision Exp is a model from DeepSeek. 8 benchmarks count toward its index, in 6 categories.","logoUrl":null,"releaseDate":"2026-08-21T11:26:03.000Z","contextWindowTokens":1048576,"modalities":["text","image"],"openWeights":true,"details":{"apiId":"deepseek/deepseek-v4-flash-vision-exp","maxOutputTokens":943717,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.014,"weightsUrl":"https://huggingface.co/deepseek-ai/DeepSeek-V4-Flash-Vision-Exp"},"reasoning":null,"overallScore":60.43614842685514,"overallScoreRange":{"low":53.2191901009619,"high":67.65310675274837},"badges":[],"capabilityScores":{"agentic":71.79818269363173,"coding":43.075784624620674,"reasoning":68.41249320408322,"multimodal":null,"knowledge":65.43022565149322,"multilingual":null,"instruction":70.16635105270767,"math":62.15278582667787},"capabilityCounts":{"math":1,"reasoning":1,"knowledge":1,"instruction":1,"agentic":2,"coding":2},"benchmarks":[{"benchmarkId":"livebench:livebench-math:overall","benchmarkName":"LiveBench Mathematics","category":"math","domain":null,"eligible":true,"ranked":true,"score":87.80675,"unit":"percent","family":"livebench:livebench-math:overall","index":62.15278582667787,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Mathematics","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.02944696569920844,"latency":null},{"benchmarkId":"livebench:livebench-reasoning:overall","benchmarkName":"LiveBench Reasoning","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":85.399,"unit":"percent","family":"livebench","index":72.24848917233919,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Reasoning","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.04243415841584158,"latency":null},{"benchmarkId":"livebench:livebench-language:overall","benchmarkName":"LiveBench Language","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":80.35966666666667,"unit":"percent","family":"livebench:livebench-language:overall","index":65.43022565149322,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Language","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.029698666666666672,"latency":null},{"benchmarkId":"livebench:livebench-data-analysis:overall","benchmarkName":"LiveBench Data Analysis","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":79.47666666666667,"unit":"percent","family":"livebench","index":64.57649723582723,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Data%20Analysis","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.03952533333333333,"latency":null},{"benchmarkId":"livebench:livebench-instruction-following:overall","benchmarkName":"LiveBench Instruction Following","category":"instruction","domain":null,"eligible":true,"ranked":true,"score":70.95824999999999,"unit":"percent","family":"livebench:livebench-instruction-following:overall","index":70.16635105270767,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=IF","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.03578,"latency":null},{"benchmarkId":"livebench:livebench-design:overall","benchmarkName":"LiveBench Design","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":70.7,"unit":"percent","family":"livebench-2-agentic","index":70.29457999324651,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Design","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.059424000000000005,"latency":null},{"benchmarkId":"livebench:livebench-research:overall","benchmarkName":"LiveBench Research","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":69.2,"unit":"percent","family":"livebench-2-agentic","index":69.19758335086583,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Research","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.24880454545454544,"latency":null},{"benchmarkId":"livebench:livebench-coding:overall","benchmarkName":"LiveBench Coding","category":"coding","domain":null,"eligible":true,"ranked":true,"score":68.2025,"unit":"percent","family":"livebench:livebench-coding:overall","index":50.74502751322684,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Coding","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.03525384615384615,"latency":null},{"benchmarkId":"livebench:livebench-agentic-coding:overall","benchmarkName":"LiveBench Agentic Coding","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":65.101,"unit":"percent","family":"livebench:livebench-agentic-coding:overall","index":76.30024288319082,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Agentic%20Coding","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.10912083333333333,"latency":null},{"benchmarkId":"livebench:livebench-browser:overall","benchmarkName":"LiveBench Browser Use","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":64,"unit":"percent","family":"livebench-2-agentic","index":65.39466165727944,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Browser","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.431712,"latency":null},{"benchmarkId":"livebench:livebench-long-horizon:overall","benchmarkName":"LiveBench Long Agentic","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":39,"unit":"percent","family":"livebench-2-agentic","index":47.11138428426802,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Long%20Agentic","measuredAt":"2026-10-14","stderr":null,"costPerTest":1.0406888888888888,"latency":null},{"benchmarkId":"livebench:livebench-coding-2:overall","benchmarkName":"LiveBench Coding 2.0","category":"coding","domain":null,"eligible":true,"ranked":true,"score":37.6,"unit":"percent","family":"livebench:livebench-coding-2:overall","index":35.40654173601451,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Coding","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.040134482758620686,"latency":null}],"price":{"inputPerMillion":0.44,"outputPerMillion":1.32,"batch":{"inputPerMillion":0.11,"outputPerMillion":0.33},"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":74,"timeToFirstToken":3.127,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":85,"benchmarkCount":8,"categoryCount":6,"confidence":"low","state":"available"},"hosts":[{"host":"deepinfra","name":"Deep Infra","kind":"host","url":"https://deepinfra.com/models","inputPerMillion":0.44,"outputPerMillion":1.32,"cacheReadPerMillion":0.014,"contextTokens":1048576,"maxOutputTokens":384000,"deprecated":false},{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":0.44,"outputPerMillion":1.32,"cacheReadPerMillion":0.014,"contextTokens":1048576,"maxOutputTokens":943717,"deprecated":false},{"host":"siliconflow","name":"SiliconFlow","kind":"host","url":"https://cloud.siliconflow.com/models","inputPerMillion":0.44,"outputPerMillion":1.32,"cacheReadPerMillion":0.028,"contextTokens":1000000,"maxOutputTokens":384000,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:livebench","name":"LiveBench","url":"https://livebench.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"reflection-beam","providerId":"reflection-ai","slug":"reflection-beam","name":"Beam 1.0 501B-A23B","provider":"Reflection AI","description":"Beam 1.0 501B-A23B is a reasoning model from Reflection AI in the Beam family. 27 benchmarks count toward its index, in 6 categories.","logoUrl":null,"releaseDate":"2026-10-16","contextWindowTokens":1000000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":true,"overallScore":60.18237998368361,"overallScoreRange":{"low":55.57046203275153,"high":64.7942979346157},"badges":[],"capabilityScores":{"agentic":59.33062099427735,"coding":62.022846750643694,"reasoning":63.15753747662049,"multimodal":null,"knowledge":56.51259903765037,"multilingual":null,"instruction":63.41067433249334,"math":55.62916527166053},"capabilityCounts":{"math":1,"knowledge":3,"coding":3,"agentic":15,"instruction":1,"reasoning":4},"benchmarks":[{"benchmarkId":"aime2026","benchmarkName":"AIME 2026","category":"math","score":97.7,"unit":"percent","family":"aime2026","index":55.62916527166053,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gpqaDiamond","benchmarkName":"GPQA Diamond","category":"knowledge","score":90.6,"unit":"percent","family":"gpqa-diamond","index":60.30494508558205,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"sweVerified","benchmarkName":"Software Engineering Benchmark Verified","category":"coding","score":81.6,"unit":"percent","family":"swe-bench-verified","index":61.41262587662091,"aboveFit":false,"run":null,"source":"Carlos E. Jimenez, John Yang, Alexander Wettig, Shunyu Yao, Kexin Pei, Ofir Press, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2310.06770","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mcpAtlas","benchmarkName":"MCP Atlas","category":"agentic","score":80.8,"unit":"percent","family":"mcpAtlas","index":65.47264747420897,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"deepSearchQa","benchmarkName":"DeepSearchQA","category":"agentic","score":80.5,"unit":"percent","family":"deepSearchQa","index":59.26623034211232,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"terminalBench21","benchmarkName":"Terminal-Bench 2.1","category":"agentic","score":80.1,"unit":"percent","family":"terminal-bench-2-1","index":67.98803590164052,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"ifBench","benchmarkName":"Instruction Following Benchmark","category":"instruction","score":79.6,"unit":"percent","family":"ifbench-family","index":63.41067433249334,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":78.2,"unit":"percent","family":"lcr","index":60.949546559969335,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"browseComp","benchmarkName":"BrowseComp","category":"agentic","score":76.8,"unit":"percent","family":"browseComp","index":60.58251515134536,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"longBenchV2","benchmarkName":"LongBench v2","category":"reasoning","score":67.5,"unit":"percent","family":"longBenchV2","index":60.37233651267623,"aboveFit":false,"run":null,"source":"LongBench v2 authors","sourceUrl":"https://arxiv.org/abs/2412.15204","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"swePro","benchmarkName":"SWE-bench Pro","category":"coding","score":66.1,"unit":"percent","family":"swePro","index":65.13080343043919,"aboveFit":false,"run":null,"source":"Xiang Deng, Jeff Da, Edwin Pan, Yannis Yiming He, Charles Ide, Kanak Garg, Niklas Lauffer, Andrew Park, Nitin Pasari, Chetan Rane, Karmini Sampath, Maya Krishnan, Srivatsa Kundurthy, Sean Hendryx, Zifan Wang, Chen Bo Calvin Zhang, Noah Jacobson, Bing Liu, Brad Kenstler","sourceUrl":"https://arxiv.org/abs/2509.16941","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hleWithTools","benchmarkName":"Humanity's Last Exam with tools","category":"agentic","score":51.1,"unit":"percent","family":"hleWithTools","index":60.97877058293187,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaCweBench","benchmarkName":"Artificial Analysis CWE-Bench-AA","category":"agentic","score":49.2,"unit":"percent","family":"aaCweBench","index":62.718972329422996,"aboveFit":false,"run":null,"source":"Artificial Analysis; dataset by Collinear AI","sourceUrl":"https://artificialanalysis.ai/evaluations/cwe-bench-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"sciCode","benchmarkName":"Scientific Code Benchmark","category":"coding","score":49.2,"unit":"percent","family":"scicode-family","index":59.98140284611614,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaAutomationBench","benchmarkName":"Artificial Analysis AutomationBench","category":"agentic","score":48.5,"unit":"percent","family":"aaAutomationBench","index":59.87670256324496,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/automationbench-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaSciCode","benchmarkName":"Artificial Analysis SciCode","category":"coding","score":48.5,"unit":"percent","family":"scicode-family","index":59.06881904362582,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/scicode","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaCyberGymE2e","benchmarkName":"Artificial Analysis CyberGym-E2E-AA","category":"agentic","score":45,"unit":"percent","family":"aaCyberGymE2e","index":95,"aboveFit":true,"run":null,"source":"Artificial Analysis; dataset by Berkeley RDI","sourceUrl":"https://artificialanalysis.ai/evaluations/cybergym-e2e-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"deepSwe","benchmarkName":"DeepSWE","category":"agentic","score":43.3,"unit":"percent","family":"deepSwe","index":52.55461186980438,"aboveFit":false,"run":null,"source":"Datacurve AI","sourceUrl":"https://deepswe.datacurve.ai/blog","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"tau3Bench","benchmarkName":"τ³-Bench Tool-Agent-User Evaluation","category":"agentic","score":36.1,"unit":"percent","family":"tau3Bench","index":20.999073264364007,"aboveFit":false,"run":null,"source":"Sierra Research","sourceUrl":"https://github.com/sierra-research/tau2-bench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hleNoTools","benchmarkName":"Humanity's Last Exam without tools","category":"knowledge","score":36,"unit":"percent","family":"hle","index":60.50518313438607,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":34.7,"unit":"percent","family":"hle","index":59.20090256584734,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gdpvalAaNormalized","benchmarkName":"GDPval-AA normalized","category":"agentic","score":33.9,"unit":"percent","family":"gdpvalAaNormalized","index":58.27148713854589,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaCyberIndex","benchmarkName":"Artificial Analysis Cyber Index","category":"agentic","score":33.7,"unit":"percent","family":"aaCyberIndex","index":61.8381703108941,"aboveFit":false,"run":null,"source":"Artificial Analysis with the Cyber Index Alliance (Collinear AI, IBM, NVIDIA, Pasta Lab, Vercel)","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-cyber-index","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"sweAtlasCodebaseQna","benchmarkName":"SWE-Atlas Codebase QnA","category":"coding","score":31.8,"unit":"percent","family":"sweAtlasCodebaseQna","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"omniscienceAccuracy","benchmarkName":"Artificial Analysis Omniscience Accuracy","category":"knowledge","score":30.7,"unit":"percent","family":"omniscienceAccuracy","index":49.37980917725235,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaMlcr","benchmarkName":"Medical Long Context Reasoning (MLCR-AA)","category":"reasoning","score":17.8,"unit":"percent","family":"aaMlcr","index":62.90425275887164,"aboveFit":false,"run":null,"source":"Wisedocs and Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mlcr-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":16.3,"unit":"percent","family":"critpt","index":68.40401407496478,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaGdpPdf","benchmarkName":"Artificial Analysis GDP.pdf","category":"agentic","score":11.6,"unit":"percent","family":"aaGdpPdf","index":56.63073463501786,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdp-pdf","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaDeepsecBench","benchmarkName":"Artificial Analysis DeepsecBench-AA","category":"agentic","score":6.9,"unit":"percent","family":"aaDeepsecBench","index":55.5917334055754,"aboveFit":false,"run":null,"source":"Artificial Analysis; dataset by Vercel","sourceUrl":"https://artificialanalysis.ai/evaluations/deepsecbench-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaTerminalBench4","benchmarkName":"Artificial Analysis Terminal-Bench v4.0","category":"agentic","score":3,"unit":"percent","family":"aaTerminalBench4","index":52.18962994505158,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/terminalbench-v4-0","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":345,"timeToFirstToken":6.36,"source":"BenchLM","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":85,"benchmarkCount":27,"categoryCount":6,"confidence":"medium","state":"available"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"seed-2-1-pro","providerId":"bytedance","slug":"seed-2-1-pro","name":"Seed 2.1 Pro","provider":"ByteDance","description":"Seed 2.1 Pro is a reasoning model from ByteDance in the Seed 2.1 family. 9 benchmarks count toward its index, in 4 categories.","logoUrl":null,"releaseDate":"2026-06-24","contextWindowTokens":256000,"modalities":["text"],"openWeights":false,"details":null,"reasoning":true,"overallScore":60.084466822352105,"overallScoreRange":{"low":51.194466822352105,"high":68.9744668223521},"badges":[],"capabilityScores":{"agentic":62.665852084936425,"coding":59.91971265496159,"reasoning":null,"multimodal":62.65761102080474,"knowledge":55.16286273134034,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"multimodal":6,"agentic":1,"knowledge":1,"coding":1},"benchmarks":[{"benchmarkId":"mathVision","benchmarkName":"MathVision","category":"multimodal","score":92.6,"unit":"percent","family":"mathVision","index":63.3664667240016,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"videoMme","benchmarkName":"Video-MME","category":"multimodal","score":89.2,"unit":"percent","family":"videoMme","index":null,"aboveFit":false,"run":null,"source":"Video-MME benchmark team","sourceUrl":"https://mme-benchmark.github.io/","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"charxiv","benchmarkName":"CharXiv Reasoning","category":"multimodal","score":85.4,"unit":"percent","family":"charxiv","index":61.41196657479317,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmmuPro","benchmarkName":"Massive Multi-discipline Multimodal Understanding Pro","category":"multimodal","score":81.6,"unit":"percent","family":"mmmu-pro","index":61.348055861559544,"aboveFit":false,"run":null,"source":"MMMU-Pro authors","sourceUrl":"https://arxiv.org/abs/2409.02813","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"babyVision","benchmarkName":"BabyVision","category":"multimodal","score":73.7,"unit":"percent","family":"babyVision","index":65.21710741352577,"aboveFit":false,"run":null,"source":"Meta AI","sourceUrl":"https://ai.meta.com/static-resource/muse-spark-1-1-evaluation-report","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"erqa","benchmarkName":"ERQA","category":"multimodal","score":72,"unit":"percent","family":"erqa","index":64.83732741364459,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"terminalBench21","benchmarkName":"Terminal-Bench 2.1","category":"agentic","score":71,"unit":"percent","family":"terminal-bench-2-1","index":62.665852084936425,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"superGpqa","benchmarkName":"SuperGPQA: Scaling LLM Evaluation Across 285 Graduate Disciplines","category":"knowledge","score":70.8,"unit":"percent","family":"superGpqa","index":55.16286273134034,"aboveFit":false,"run":null,"source":"Xiaoxuan Du, Yao Yao, Kexin Ma, Bowen Wang, Tianyu Zheng, Kaiyan Zhu, Yiming Zhang, Yutao Zhu, Jiawei Zhou, Jingren Zhou","sourceUrl":"https://arxiv.org/abs/2502.14739","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"nl2Repo","benchmarkName":"NL2Repo","category":"coding","score":47,"unit":"percent","family":"nl2Repo","index":59.91971265496159,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"zeroBench","benchmarkName":"ZeroBench","category":"multimodal","score":18,"unit":"percent","family":"zeroBench","index":59.76474213730373,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":65,"benchmarkCount":9,"categoryCount":4,"confidence":"low","state":"available"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"claude-opus-4-6-thinking","providerId":"anthropic","slug":"claude-opus-4-6-thinking","name":"Claude Opus 4.6 (Adaptive)","provider":"Anthropic","description":"Claude Opus 4.6 (Adaptive) is a reasoning model from Anthropic in the Claude Opus 4.6 family. 15 benchmarks count toward its index, in 7 categories.","logoUrl":null,"releaseDate":"2026-02-01","contextWindowTokens":1000000,"modalities":["text"],"openWeights":false,"details":null,"reasoning":true,"overallScore":59.99523202246599,"overallScoreRange":{"low":55.320155975254394,"high":64.67030806967759},"badges":[],"capabilityScores":{"agentic":63.46303128223056,"coding":60.020102143423614,"reasoning":61.02678812901554,"multimodal":58.79273287899809,"knowledge":62.6234391332073,"multilingual":null,"instruction":38.646143039511514,"math":57.29506683176784},"capabilityCounts":{"math":1,"agentic":3,"knowledge":4,"coding":3,"multimodal":1,"reasoning":2,"instruction":1},"benchmarks":[{"benchmarkId":"vals-ai:aime:overall","benchmarkName":"AIME","category":"math","domain":null,"eligible":true,"ranked":true,"score":95.625,"unit":"percent","family":"vals-ai:aime:overall","index":57.29506683176784,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/aime","measuredAt":"2026-04-16","stderr":0.644,"costPerTest":0.343821,"latency":295.678},{"benchmarkId":"vals-ai:medqa:overall","benchmarkName":"MedQA","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":95.408,"unit":"percent","family":"vals-ai:medqa:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medqa","measuredAt":"2026-04-16","stderr":0.191,"costPerTest":0.026693,"latency":30.085},{"benchmarkId":"aaGlobalMmluLite","benchmarkName":"Artificial Analysis Global-MMLU-Lite","category":"multilingual","score":92.2,"unit":"percent","family":"aaGlobalMmluLite","index":null,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/global-mmlu-lite","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"tau2Bench","benchmarkName":"τ²-Bench Tool-Agent-User Evaluation","category":"agentic","score":92.1,"unit":"percent","family":"tau2-bench","index":63.90452375331877,"aboveFit":false,"run":null,"source":"Victor Barres, Honghua Dong, Soham Ray, Xujie Si, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2506.07982","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:gpqa:overall","benchmarkName":"GPQA Diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":89.646,"unit":"percent","family":"gpqa-diamond","index":59.42996235370858,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","measuredAt":"2026-09-01","stderr":1.186,"costPerTest":0.294901,"latency":319.02},{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":89.6,"unit":"percent","family":"gpqa-diamond","index":59.387772410662265,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:mmlu_pro:overall","benchmarkName":"MMLU Pro","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":89.107,"unit":"percent","family":"mmlu-pro","index":56.74488259321872,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","measuredAt":"2026-09-01","stderr":0.455,"costPerTest":0.052129,"latency":49.451},{"benchmarkId":"vals-ai:medscribe:overall","benchmarkName":"MedScribe","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":86.13,"unit":"percent","family":"vals-ai:medscribe:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medscribe","measuredAt":"2026-10-29","stderr":1.944,"costPerTest":0.224735,"latency":127.564},{"benchmarkId":"vals-ai:legal_bench:overall","benchmarkName":"LegalBench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":85.301,"unit":"percent","family":"vals-ai:legal_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","measuredAt":"2026-10-13","stderr":0.368,"costPerTest":0.004043,"latency":3.755},{"benchmarkId":"vals-ai:lcb:overall","benchmarkName":"LiveCodeBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":84.676,"unit":"percent","family":"livecodebench","index":59.60293481053472,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","measuredAt":"2026-09-01","stderr":1.02,"costPerTest":0.34344,"latency":324.486},{"benchmarkId":"vals-ai:mmmu:overall","benchmarkName":"MMMU Pro","category":"multimodal","domain":null,"eligible":true,"ranked":true,"score":83.873,"unit":"percent","family":"mmmu-pro","index":64.30616594017542,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmmu","measuredAt":"2026-09-01","stderr":0.884,"costPerTest":0.116094,"latency":119.005},{"benchmarkId":"vals-ai:swebench:overall","benchmarkName":"SWE-bench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":78.2,"unit":"percent","family":"swe-bench-verified","index":58.90493011215636,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/swebench","measuredAt":"2026-09-01","stderr":1.848,"costPerTest":1.216469,"latency":350.76},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":78,"unit":"percent","family":"lcr","index":60.8123304548522,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:tax_eval_v2:overall","benchmarkName":"TaxEval v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":75.961,"unit":"percent","family":"vals-ai:tax_eval_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","measuredAt":"2026-09-01","stderr":0.834,"costPerTest":0.071368,"latency":79.254},{"benchmarkId":"aaMmmuPro","benchmarkName":"Artificial Analysis MMMU-Pro","category":"multimodal","score":75.4,"unit":"percent","family":"mmmu-pro","index":53.279299817820736,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmmu-pro","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:mortgage_tax:overall","benchmarkName":"MortgageTax","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":68.522,"unit":"percent","family":"vals-ai:mortgage_tax:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mortgage_tax","measuredAt":"2026-09-01","stderr":0.911,"costPerTest":0.060749,"latency":27.161},{"benchmarkId":"vals-ai:corp_fin_v2:overall","benchmarkName":"CorpFin v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":67.016,"unit":"percent","family":"vals-ai:corp_fin_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","measuredAt":"2026-08-12","stderr":0.926,"costPerTest":0.487461,"latency":20.581},{"benchmarkId":"vals-ai:case_law_v2:overall","benchmarkName":"CaseLaw v2","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":62.058,"unit":"percent","family":"vals-ai:case_law_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/case_law_v2","measuredAt":"2026-05-04","stderr":0.371,"costPerTest":0.231218,"latency":32.273},{"benchmarkId":"vals-ai:finance_agent:overall","benchmarkName":"Finance Agent v1.1","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":60.046,"unit":"percent","family":"vals-ai:finance_agent:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/finance_agent","measuredAt":"2026-06-04","stderr":2.775,"costPerTest":1.108198,"latency":289.732},{"benchmarkId":"vals-ai:terminal-bench-2:overall","benchmarkName":"Terminal-Bench 2.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":58.427,"unit":"percent","family":"vals-ai:terminal-bench-2:overall","index":61.730818847292845,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2","measuredAt":"2026-06-04","stderr":5.254,"costPerTest":1.140327,"latency":898.494},{"benchmarkId":"vals-ai:vibe-code:overall","benchmarkName":"Vibe Code Bench v1.1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":53.498,"unit":"percent","family":"vals-ai:vibe-code:overall","index":61.55244150757977,"aboveFit":false,"run":"OpenHands","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","measuredAt":"2026-11-09","stderr":4.678,"costPerTest":8.279062,"latency":1386.056},{"benchmarkId":"aaIfBench","benchmarkName":"Artificial Analysis IFBench","category":"instruction","score":53.1,"unit":"percent","family":"ifbench-family","index":38.646143039511514,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:medcode:overall","benchmarkName":"MedCode","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":49.129,"unit":"percent","family":"vals-ai:medcode:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medcode","measuredAt":"2026-10-29","stderr":2.085,"costPerTest":0.244127,"latency":156.295},{"benchmarkId":"omniscienceAccuracy","benchmarkName":"Artificial Analysis Omniscience Accuracy","category":"knowledge","score":47,"unit":"percent","family":"omniscienceAccuracy","index":69.9219817174228,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":39.9,"unit":"percent","family":"hle","index":64.41802484000227,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"apexAgentsAa","benchmarkName":"APEX-Agents-AA","category":"agentic","score":33,"unit":"percent","family":"apexAgentsAa","index":64.75375124608007,"aboveFit":false,"run":null,"source":"Artificial Analysis / Mercor","sourceUrl":"https://artificialanalysis.ai/evaluations/apex-agents-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":12.6,"unit":"percent","family":"critpt","index":61.241245803178884,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":95,"benchmarkCount":15,"categoryCount":7,"confidence":"medium","state":"available"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gpt-5-2-codex","providerId":"openai","slug":"gpt-5-2-codex","name":"GPT-5.2-Codex","provider":"OpenAI","description":"GPT-5.2-Codex is a reasoning model from OpenAI. 19 benchmarks count toward its index, in 7 categories.","logoUrl":null,"releaseDate":"2025-12-18","contextWindowTokens":400000,"modalities":["text","image"],"openWeights":false,"details":{"apiId":"openai/gpt-5.2-codex","maxOutputTokens":128000,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.175,"weightsUrl":null},"reasoning":true,"overallScore":59.855507434196674,"overallScoreRange":{"low":55.64585501750969,"high":64.06515985088366},"badges":[],"capabilityScores":{"agentic":58.09064054674676,"coding":61.371466769916466,"reasoning":60.02087284381246,"multimodal":54.45057085642797,"knowledge":60.05712511549525,"multilingual":null,"instruction":61.28857074396265,"math":63.500512383486196},"capabilityCounts":{"agentic":4,"knowledge":4,"math":1,"coding":4,"reasoning":3,"instruction":2,"multimodal":1},"benchmarks":[{"benchmarkId":"tau2Bench","benchmarkName":"τ²-Bench Tool-Agent-User Evaluation","category":"agentic","score":92.1,"unit":"percent","family":"tau2-bench","index":63.90452375331877,"aboveFit":false,"run":null,"source":"Victor Barres, Honghua Dong, Soham Ray, Xujie Si, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2506.07982","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":89.9,"unit":"percent","family":"gpqa-diamond","index":59.66292421313821,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-math:overall","benchmarkName":"LiveBench Mathematics","category":"math","domain":null,"eligible":true,"ranked":true,"score":88.77375,"unit":"percent","family":"livebench:livebench-math:overall","index":63.500512383486196,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Mathematics","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.10792902374670185,"latency":null},{"benchmarkId":"vals-ai:lcb:overall","benchmarkName":"LiveCodeBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":87.993,"unit":"percent","family":"livecodebench","index":62.355186467136804,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","measuredAt":"2026-09-01","stderr":0.942,"costPerTest":0.11667,"latency":168.271},{"benchmarkId":"livebench:livebench-coding:overall","benchmarkName":"LiveBench Coding","category":"coding","domain":null,"eligible":true,"ranked":true,"score":83.61949999999999,"unit":"percent","family":"livebench:livebench-coding:overall","index":72.42730510280705,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Coding","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.029464957264957264,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":82.3,"unit":"percent","family":"lcr","index":63.76247671487054,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-data-analysis:overall","benchmarkName":"LiveBench Data Analysis","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":78.204,"unit":"percent","family":"livebench","index":62.9278415700647,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Data%20Analysis","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.199264,"latency":null},{"benchmarkId":"livebench:livebench-reasoning:overall","benchmarkName":"LiveBench Reasoning","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":77.7115,"unit":"percent","family":"livebench","index":62.28984032696253,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Reasoning","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.16897277227722773,"latency":null},{"benchmarkId":"aaIfBench","benchmarkName":"Artificial Analysis IFBench","category":"instruction","score":77.6,"unit":"percent","family":"ifbench-family","index":61.54165310283433,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaMmmuPro","benchmarkName":"Artificial Analysis MMMU-Pro","category":"multimodal","score":76.3,"unit":"percent","family":"mmmu-pro","index":54.45057085642797,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmmu-pro","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-language:overall","benchmarkName":"LiveBench Language","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":73.67833333333333,"unit":"percent","family":"livebench:livebench-language:overall","index":57.87490952649415,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Language","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.07135466666666668,"latency":null},{"benchmarkId":"swebench-experiments:swe-bench-verified:overall","benchmarkName":"SWE-bench Verified","category":"coding","domain":null,"eligible":true,"ranked":true,"score":72.8,"unit":"percent","family":"swe-bench-verified","index":54.92211919212442,"aboveFit":false,"run":"mini-SWE-agent","source":"SWE-bench team","sourceUrl":"https://github.com/SWE-bench/experiments/tree/main/evaluation/verified","measuredAt":"2026-09-01","stderr":null,"costPerTest":0.4494238987,"latency":null},{"benchmarkId":"vals-ai:swebench:overall","benchmarkName":"SWE-bench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":72.4,"unit":"percent","family":"swe-bench-verified","index":54.62709616101095,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/swebench","measuredAt":"2026-09-01","stderr":2.001,"costPerTest":0.311527,"latency":188.405},{"benchmarkId":"livebench:livebench-instruction-following:overall","benchmarkName":"LiveBench Instruction Following","category":"instruction","domain":null,"eligible":true,"ranked":true,"score":66.44574999999999,"unit":"percent","family":"livebench:livebench-instruction-following:overall","index":61.03548838509097,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=IF","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.0708265,"latency":null},{"benchmarkId":"swebench-experiments:swe-bench-multilingual:overall","benchmarkName":"SWE-bench Multilingual","category":"coding","domain":null,"eligible":true,"ranked":true,"score":66.3,"unit":"percent","family":"swe-bench-multilingual","index":null,"aboveFit":false,"run":"mini-SWE-agent","source":"SWE-bench team","sourceUrl":"https://github.com/SWE-bench/experiments/tree/main/evaluation/multilingual","measuredAt":"2026-09-02","stderr":null,"costPerTest":0.6621715578333333,"latency":null},{"benchmarkId":"gertLabs","benchmarkName":"Gert Labs Composite Game Benchmark","category":"agentic","score":51.79,"unit":"percent","family":"gertLabs","index":57.48198926258672,"aboveFit":false,"run":null,"source":"Gert Labs","sourceUrl":"https://gertlabs.com/rankings","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-agentic-coding:overall","benchmarkName":"LiveBench Agentic Coding","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":49.394000000000005,"unit":"percent","family":"livebench:livebench-agentic-coding:overall","index":60.101881299797455,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Agentic%20Coding","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.5942972222222223,"latency":null},{"benchmarkId":"omniscienceAccuracy","benchmarkName":"Artificial Analysis Omniscience Accuracy","category":"knowledge","score":41.1,"unit":"percent","family":"omniscienceAccuracy","index":62.48647141147153,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:vibe-code:overall","benchmarkName":"Vibe Code Bench v1.1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":37.912,"unit":"percent","family":"vals-ai:vibe-code:overall","index":55.928767833154325,"aboveFit":false,"run":"OpenHands","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","measuredAt":"2026-11-09","stderr":4.576,"costPerTest":4.152495,"latency":1934.792},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":35.7,"unit":"percent","family":"hle","index":60.20419531087714,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"jobBench","benchmarkName":"JobBench","category":"agentic","score":26,"unit":"percent","family":"jobBench","index":50.87416787128409,"aboveFit":false,"run":null,"source":"Yuetai Li, Yichen Feng, Zhangchen Xu, Zixian Ma, Kaiyuan Zheng, Fengqing Jiang, Xinghua Sun, Rulin Shao, Zichen Chen, Yue Huang, Xinyang Han, Brian Lee, Kayla Xu, Shenglai Zeng, Hang Hua, Xiangliang Zhang, Basel Alomair, Ranjay Krishna, Luke Zettlemoyer, Pang Wei Koh, Bhaskar Ramasubramanian, Luyao Niu, Xiang Yue, Radha Poovendran","sourceUrl":"https://arxiv.org/abs/2605.26329","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":8.7,"unit":"percent","family":"critpt","index":53.691300868053226,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":1.75,"outputPerMillion":14,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":44,"timeToFirstToken":2.949,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":95,"benchmarkCount":19,"categoryCount":7,"confidence":"medium","state":"available"},"hosts":[{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":1.75,"outputPerMillion":14,"cacheReadPerMillion":0.175,"contextTokens":400000,"maxOutputTokens":128000,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:livebench","name":"LiveBench","url":"https://livebench.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:swe-bench-team","name":"SWE-bench team","url":"https://github.com","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"muse-spark","providerId":"meta","slug":"muse-spark","name":"Muse Spark","provider":"Meta","description":"Muse Spark is a reasoning model from Meta. 29 benchmarks count toward its index, in 7 categories.","logoUrl":null,"releaseDate":"2026-04-08","contextWindowTokens":262000,"modalities":["text"],"openWeights":false,"details":null,"reasoning":true,"overallScore":59.84240912692594,"overallScoreRange":{"low":56.339715508603014,"high":63.34510274524887},"badges":[],"capabilityScores":{"agentic":55.85341962524374,"coding":53.34016842136722,"reasoning":59.76846397316127,"multimodal":63.042717186854,"knowledge":67.75709131675424,"multilingual":null,"instruction":59.95298505762419,"math":60.21394530462718},"capabilityCounts":{"math":4,"agentic":8,"knowledge":5,"multimodal":6,"reasoning":2,"coding":3,"instruction":1},"benchmarks":[{"benchmarkId":"vals-ai:aime:overall","benchmarkName":"AIME","category":"math","domain":null,"eligible":true,"ranked":true,"score":96.875,"unit":"percent","family":"vals-ai:aime:overall","index":57.841374032311265,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/aime","measuredAt":"2026-04-16","stderr":0.378,"costPerTest":0.004561,"latency":156.746},{"benchmarkId":"tau2Bench","benchmarkName":"τ²-Bench Tool-Agent-User Evaluation","category":"agentic","score":91.5,"unit":"percent","family":"tau2-bench","index":63.50637988616581,"aboveFit":false,"run":null,"source":"Victor Barres, Honghua Dong, Soham Ray, Xujie Si, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2506.07982","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":89.8,"unit":"percent","family":"gpqa-diamond","index":59.571206945646225,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.1999999999999997,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:gpqa:overall","benchmarkName":"GPQA Diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":89.646,"unit":"percent","family":"gpqa-diamond","index":59.42996235370858,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","measuredAt":"2026-09-01","stderr":1.546,"costPerTest":0.003074,"latency":94.58},{"benchmarkId":"gpqaDiamond","benchmarkName":"GPQA Diamond","category":"knowledge","score":89.5,"unit":"percent","family":"gpqa-diamond","index":59.29605514317029,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":88.9,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":60.64058154223609,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":4.7,"costPerTest":null,"latency":null},{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":88.4,"unit":"percent","family":"gpqa-diamond","index":58.28716520075854,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:mmmu:overall","benchmarkName":"MMMU Pro","category":"multimodal","domain":null,"eligible":true,"ranked":true,"score":87.399,"unit":"percent","family":"mmmu-pro","index":68.89494558698559,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmmu","measuredAt":"2026-09-01","stderr":0.798,"costPerTest":0.001746,"latency":48.272},{"benchmarkId":"vals-ai:mmlu_pro:overall","benchmarkName":"MMLU Pro","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":87.316,"unit":"percent","family":"mmlu-pro","index":54.488436970408046,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","measuredAt":"2026-09-01","stderr":0.327,"costPerTest":0.00167,"latency":46.242},{"benchmarkId":"charxiv","benchmarkName":"CharXiv Reasoning","category":"multimodal","score":86.4,"unit":"percent","family":"charxiv","index":62.550465000359985,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:medscribe:overall","benchmarkName":"MedScribe","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":85.902,"unit":"percent","family":"vals-ai:medscribe:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medscribe","measuredAt":"2026-10-29","stderr":1.847,"costPerTest":0.007681,"latency":191.473},{"benchmarkId":"vals-ai:legal_bench:overall","benchmarkName":"LegalBench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":84.217,"unit":"percent","family":"vals-ai:legal_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","measuredAt":"2026-10-13","stderr":0.416,"costPerTest":0.001178,"latency":34.203},{"benchmarkId":"screenSpotPro","benchmarkName":"ScreenSpot Pro","category":"multimodal","score":84.1,"unit":"percent","family":"screenSpotPro","index":64.07550289780768,"aboveFit":false,"run":null,"source":"Kaixin Li, Ziyang Meng, Hongzhan Lin, Ziyang Luo, Yuchen Tian, Jing Ma, Zhiyong Huang, Tat-Seng Chua","sourceUrl":"https://arxiv.org/abs/2504.07981","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaMmmuPro","benchmarkName":"Artificial Analysis MMMU-Pro","category":"multimodal","score":80.5,"unit":"percent","family":"mmmu-pro","index":59.91650236992847,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmmu-pro","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmmuPro","benchmarkName":"Massive Multi-discipline Multimodal Understanding Pro","category":"multimodal","score":80.4,"unit":"percent","family":"mmmu-pro","index":59.78636114341656,"aboveFit":false,"run":null,"source":"MMMU-Pro authors","sourceUrl":"https://arxiv.org/abs/2409.02813","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"liveCodeBenchPro","benchmarkName":"LiveCodeBench Pro","category":"coding","score":80,"unit":"percent","family":"liveCodeBenchPro","index":null,"aboveFit":false,"run":null,"source":"LiveCodeBench Pro authors","sourceUrl":"https://arxiv.org/abs/2506.11928","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"medXpertQaMm","benchmarkName":"MedXpertQA Multimodal","category":"multimodal","score":78.4,"unit":"percent","family":"medXpertQaMm","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":78,"unit":"percent","family":"lcr","index":60.8123304548522,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:tax_eval_v2:overall","benchmarkName":"TaxEval v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":77.678,"unit":"percent","family":"vals-ai:tax_eval_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","measuredAt":"2026-09-01","stderr":0.806,"costPerTest":0.001922,"latency":57.522},{"benchmarkId":"sweVerified","benchmarkName":"Software Engineering Benchmark Verified","category":"coding","score":77.4,"unit":"percent","family":"swe-bench-verified","index":58.314884049929404,"aboveFit":false,"run":null,"source":"Carlos E. Jimenez, John Yang, Alexander Wettig, Shunyu Yao, Kexin Pei, Ofir Press, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2310.06770","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaIfBench","benchmarkName":"Artificial Analysis IFBench","category":"instruction","score":75.9,"unit":"percent","family":"ifbench-family","index":59.95298505762419,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"deepSearchQa","benchmarkName":"DeepSearchQA","category":"agentic","score":74.8,"unit":"percent","family":"deepSearchQa","index":55.2637807888055,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:swebench:overall","benchmarkName":"SWE-bench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":74.4,"unit":"percent","family":"swe-bench-verified","index":56.102211316578334,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/swebench","measuredAt":"2026-09-01","stderr":1.954,"costPerTest":0.444473,"latency":495.294},{"benchmarkId":"simpleVqa","benchmarkName":"SimpleVQA","category":"multimodal","score":71.3,"unit":"percent","family":"simpleVqa","index":61.59451343631456,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:corp_fin_v2:overall","benchmarkName":"CorpFin v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":65.113,"unit":"percent","family":"vals-ai:corp_fin_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","measuredAt":"2026-08-12","stderr":0.936,"costPerTest":0.011749,"latency":40.057},{"benchmarkId":"erqa","benchmarkName":"ERQA","category":"multimodal","score":64.7,"unit":"percent","family":"erqa","index":58.079318686800036,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"clawEval","benchmarkName":"Claw-Eval","category":"agentic","score":63.8,"unit":"percent","family":"clawEval","index":57.37721715640845,"aboveFit":false,"run":null,"source":"Bowen Ye, Rang Li, Qibin Yang, Yuanxin Liu, Linli Yao, Hanglong Lv, Zhihui Xie, Chenxin An, Lei Li, Lingpeng Kong, Qi Liu, Zhifang Sui, Tong Yang","sourceUrl":"https://arxiv.org/abs/2604.06132","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:case_law_v2:overall","benchmarkName":"CaseLaw v2","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":63.13,"unit":"percent","family":"vals-ai:case_law_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/case_law_v2","measuredAt":"2026-05-04","stderr":0.75,"costPerTest":0.006504,"latency":61.867},{"benchmarkId":"vals-ai:finance_agent:overall","benchmarkName":"Finance Agent v1.1","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":60.595,"unit":"percent","family":"vals-ai:finance_agent:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/finance_agent","measuredAt":"2026-06-04","stderr":2.839,"costPerTest":0.063451,"latency":418.139},{"benchmarkId":"vals-ai:terminal-bench-2:overall","benchmarkName":"Terminal-Bench 2.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":59.551,"unit":"percent","family":"vals-ai:terminal-bench-2:overall","index":62.40759189859753,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2","measuredAt":"2026-06-04","stderr":5.232,"costPerTest":0.070242,"latency":566.325},{"benchmarkId":"terminalBench2","benchmarkName":"Terminal-Bench 2.0","category":"agentic","score":59,"unit":"percent","family":"terminalBench2","index":55.867022805272306,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"medXpertQaText","benchmarkName":"MedXpertQA Text","category":"knowledge","score":52.6,"unit":"percent","family":"medXpertQaText","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"swePro","benchmarkName":"SWE-bench Pro","category":"coding","score":52.4,"unit":"percent","family":"swePro","index":53.46374668816383,"aboveFit":false,"run":null,"source":"Xiang Deng, Jeff Da, Edwin Pan, Yannis Yiming He, Charles Ide, Kanak Garg, Niklas Lauffer, Andrew Park, Nitin Pasari, Chetan Rane, Karmini Sampath, Maya Krishnan, Srivatsa Kundurthy, Sean Hendryx, Zifan Wang, Chen Bo Calvin Zhang, Noah Jacobson, Bing Liu, Brad Kenstler","sourceUrl":"https://arxiv.org/abs/2509.16941","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:medcode:overall","benchmarkName":"MedCode","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":51.31,"unit":"percent","family":"vals-ai:medcode:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medcode","measuredAt":"2026-10-29","stderr":2.244,"costPerTest":0.005341,"latency":123.095},{"benchmarkId":"hle","benchmarkName":"Humanity's Last Exam","category":"knowledge","score":50.4,"unit":"percent","family":"hle","index":74.9525986628151,"aboveFit":false,"run":null,"source":"Center for AI Safety, Scale AI, and thousands of expert contributors","sourceUrl":"https://lastexam.ai/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"omniscienceAccuracy","benchmarkName":"Artificial Analysis Omniscience Accuracy","category":"knowledge","score":49.6,"unit":"percent","family":"omniscienceAccuracy","index":73.19864727597759,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"terminalBenchHard","benchmarkName":"Terminal-Bench Hard","category":"agentic","score":45.5,"unit":"percent","family":"terminalBenchHard","index":56.42730214348893,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/terminalbench-hard","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"cyberGym","benchmarkName":"CyberGym","category":"agentic","score":43.5,"unit":"percent","family":"cyberGym","index":44.14136603989197,"aboveFit":false,"run":null,"source":"Zhun Wang, Tianneng Shi, Jingxuan He, Matthew Cai, Jialin Zhang, Dawn Song","sourceUrl":"https://www.cybergym.io/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hleNoTools","benchmarkName":"Humanity's Last Exam without tools","category":"knowledge","score":42.8,"unit":"percent","family":"hle","index":67.32757380058868,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"healthBenchHard","benchmarkName":"HealthBench Hard","category":"knowledge","score":42.8,"unit":"percent","family":"healthBenchHard","index":84.40778978835743,"aboveFit":true,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":40.7,"unit":"percent","family":"hle","index":65.2206590360261,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-2025-02-28-private:overall","benchmarkName":"FrontierMath-2025-02-28-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":39,"unit":"percent","family":"epoch-ai:epoch-frontiermath-2025-02-28-private:overall","index":64.34250078570871,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.9000000000000004,"costPerTest":null,"latency":null},{"benchmarkId":"zeroBench","benchmarkName":"ZeroBench","category":"multimodal","score":33,"unit":"percent","family":"zeroBench","index":72.04000072991327,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gdpvalAaNormalized","benchmarkName":"GDPval-AA normalized","category":"agentic","score":24.9,"unit":"percent","family":"gdpvalAaNormalized","index":51.836696283319405,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:vibe-code:overall","benchmarkName":"Vibe Code Bench v1.1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":19.674,"unit":"percent","family":"vals-ai:vibe-code:overall","index":49.348210892683944,"aboveFit":false,"run":"OpenHands","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","measuredAt":"2026-11-09","stderr":3.966,"costPerTest":0.579705,"latency":981.298},{"benchmarkId":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-private:overall","benchmarkName":"FrontierMath-Tier-4-2025-07-01-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":14.6,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-private:overall","index":58.03132485825266,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":5.1,"costPerTest":null,"latency":null},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":11.3,"unit":"percent","family":"critpt","index":58.724597491470334,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":{"elo":1474.125948560341,"eloUpper":1479.8585772587971,"eloLower":1468.3933198618847,"votes":14009,"rank":28,"rankUpper":null,"rankLower":null,"arenaName":"Muse Spark","organization":"meta","updatedAt":"2026-11-09"},"quality":{"coveragePercent":95,"benchmarkCount":29,"categoryCount":7,"confidence":"medium","state":"available"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"reflection-beam-fp8","providerId":"reflection-ai","slug":"reflection-beam-fp8","name":"Beam 1.0 501B-A23B FP8","provider":"Reflection AI","description":"Beam 1.0 501B-A23B FP8 is a reasoning model from Reflection AI in the Beam family. 17 benchmarks count toward its index, in 6 categories.","logoUrl":null,"releaseDate":"2026-10-16","contextWindowTokens":1000000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":true,"overallScore":59.23504459041645,"overallScoreRange":{"low":53.81324353357895,"high":64.65684564725396},"badges":[],"capabilityScores":{"agentic":55.35536755205313,"coding":61.38286917208941,"reasoning":64.26571179802654,"multimodal":null,"knowledge":60.267488208746094,"multilingual":null,"instruction":64.4386360088058,"math":55.84514742641585},"capabilityCounts":{"math":1,"knowledge":2,"agentic":7,"instruction":1,"coding":3,"reasoning":3},"benchmarks":[{"benchmarkId":"aime2026","benchmarkName":"AIME 2026","category":"math","score":98,"unit":"percent","family":"aime2026","index":55.84514742641585,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gpqaDiamond","benchmarkName":"GPQA Diamond","category":"knowledge","score":90.3,"unit":"percent","family":"gpqa-diamond","index":60.029793283106116,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mcpAtlas","benchmarkName":"MCP Atlas","category":"agentic","score":81.5,"unit":"percent","family":"mcpAtlas","index":65.94402439145922,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"ifBench","benchmarkName":"Instruction Following Benchmark","category":"instruction","score":80.7,"unit":"percent","family":"ifbench-family","index":64.4386360088058,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"sweVerified","benchmarkName":"Software Engineering Benchmark Verified","category":"coding","score":80.2,"unit":"percent","family":"swe-bench-verified","index":60.380045267723744,"aboveFit":false,"run":null,"source":"Carlos E. Jimenez, John Yang, Alexander Wettig, Shunyu Yao, Kexin Pei, Ofir Press, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2310.06770","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"deepSearchQa","benchmarkName":"DeepSearchQA","category":"agentic","score":79.6,"unit":"percent","family":"deepSearchQa","index":58.63426462316913,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":79.4,"unit":"percent","family":"lcr","index":61.77284319067213,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"terminalBench21","benchmarkName":"Terminal-Bench 2.1","category":"agentic","score":78.9,"unit":"percent","family":"terminal-bench-2-1","index":67.28620946427294,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"browseComp","benchmarkName":"BrowseComp","category":"agentic","score":76.7,"unit":"percent","family":"browseComp","index":60.503513170745435,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"longBenchV2","benchmarkName":"LongBench v2","category":"reasoning","score":67.6,"unit":"percent","family":"longBenchV2","index":60.49080648007396,"aboveFit":false,"run":null,"source":"LongBench v2 authors","sourceUrl":"https://arxiv.org/abs/2412.15204","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"swePro","benchmarkName":"SWE-bench Pro","category":"coding","score":65.9,"unit":"percent","family":"swePro","index":64.9604814342016,"aboveFit":false,"run":null,"source":"Xiang Deng, Jeff Da, Edwin Pan, Yannis Yiming He, Charles Ide, Kanak Garg, Niklas Lauffer, Andrew Park, Nitin Pasari, Chetan Rane, Karmini Sampath, Maya Krishnan, Srivatsa Kundurthy, Sean Hendryx, Zifan Wang, Chen Bo Calvin Zhang, Noah Jacobson, Bing Liu, Brad Kenstler","sourceUrl":"https://arxiv.org/abs/2509.16941","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hleWithTools","benchmarkName":"Humanity's Last Exam with tools","category":"agentic","score":51.1,"unit":"percent","family":"hleWithTools","index":60.97877058293187,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"sciCode","benchmarkName":"Scientific Code Benchmark","category":"coding","score":48.3,"unit":"percent","family":"scicode-family","index":58.80808081434287,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"deepSwe","benchmarkName":"DeepSWE","category":"agentic","score":43,"unit":"percent","family":"deepSwe","index":52.3365202446062,"aboveFit":false,"run":null,"source":"Datacurve AI","sourceUrl":"https://deepswe.datacurve.ai/blog","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"tau3Bench","benchmarkName":"τ³-Bench Tool-Agent-User Evaluation","category":"agentic","score":36.8,"unit":"percent","family":"tau3Bench","index":21.804270387187117,"aboveFit":false,"run":null,"source":"Sierra Research","sourceUrl":"https://github.com/sierra-research/tau2-bench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hleNoTools","benchmarkName":"Humanity's Last Exam without tools","category":"knowledge","score":36,"unit":"percent","family":"hle","index":60.50518313438607,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"sweAtlasCodebaseQna","benchmarkName":"SWE-Atlas Codebase QnA","category":"coding","score":32.2,"unit":"percent","family":"sweAtlasCodebaseQna","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":17.4,"unit":"percent","family":"critpt","index":70.53348572333354,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":85,"benchmarkCount":17,"categoryCount":6,"confidence":"medium","state":"available"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"qwen3-8-27b","providerId":"alibaba","slug":"qwen3-8-27b","name":"Qwen3.8-27B","provider":"Alibaba","description":"Qwen3.8-27B is a reasoning model from Alibaba. 59 benchmarks count toward its index, in 8 categories.","logoUrl":null,"releaseDate":"2026-08-05","contextWindowTokens":1048575,"modalities":["text","image","video"],"openWeights":true,"details":{"apiId":"qwen/qwen3.8-27b","maxOutputTokens":235929,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.0375,"weightsUrl":"https://huggingface.co/Qwen/Qwen3.8-27B"},"reasoning":true,"overallScore":59.18112635717594,"overallScoreRange":{"low":57.07597166661602,"high":61.28628104773586},"badges":[],"capabilityScores":{"agentic":57.38527699076469,"coding":60.33596250080948,"reasoning":61.2319873196066,"multimodal":63.626525666694015,"knowledge":51.23322664612123,"multilingual":60.56376398028179,"instruction":68.46172805215804,"math":56.30586653969561},"capabilityCounts":{"knowledge":5,"coding":14,"multimodal":9,"reasoning":6,"math":2,"agentic":20,"instruction":2,"multilingual":1},"benchmarks":[{"benchmarkId":"mathVisionPython","benchmarkName":"MathVision with Python","category":"multimodal","score":94.6,"unit":"percent","family":"mathVisionPython","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"omniDocBench15","benchmarkName":"OmniDocBench 1.5","category":"multimodal","score":91.1,"unit":"percent","family":"omniDocBench15","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":90.5,"unit":"percent","family":"gpqa-diamond","index":60.213227818090076,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"liveCodeBenchV6","benchmarkName":"LiveCodeBench v6","category":"coding","score":90.3,"unit":"percent","family":"liveCodeBenchV6","index":58.55853148873203,"aboveFit":false,"run":null,"source":"LiveCodeBench maintainers","sourceUrl":"https://github.com/LiveCodeBench/LiveCodeBench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"charxiv","benchmarkName":"CharXiv Reasoning","category":"multimodal","score":90.2,"unit":"percent","family":"charxiv","index":66.87675901751388,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mathVision","benchmarkName":"MathVision","category":"multimodal","score":90,"unit":"percent","family":"mathVision","index":60.88952303525008,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"extractBenchLlamaIndex","benchmarkName":"ExtractBench: schema-guided enterprise document extraction (LlamaIndex leaderboard)","category":"multimodal","score":89.75,"unit":"percent","family":"extractBenchLlamaIndex","index":71.16027091070515,"aboveFit":false,"run":null,"source":"Boyang Zhang, Adrian Lyjak, Eli Stewart, Zhaoqi Li and Simon Suo (LlamaIndex)","sourceUrl":"https://arxiv.org/abs/2607.29677","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gpqa","benchmarkName":"Graduate-Level Google-Proof Q&A","category":"knowledge","score":89.2,"unit":"percent","family":"gpqa-diamond","index":59.02090334069436,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gpqaDiamond","benchmarkName":"GPQA Diamond","category":"knowledge","score":89.2,"unit":"percent","family":"gpqa-diamond","index":59.02090334069436,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:gpqa:overall","benchmarkName":"GPQA Diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":88.889,"unit":"percent","family":"gpqa-diamond","index":58.735662638794295,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","measuredAt":"2026-09-01","stderr":1.579,"costPerTest":0.046624,"latency":116.361},{"benchmarkId":"arc-prize:v1-public-eval:overall","benchmarkName":"ARC-AGI-1 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":87.5,"unit":"percent","family":"arc-prize:v1-public-eval:overall","index":null,"aboveFit":false,"run":"xhigh effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.14246719631250002,"latency":null},{"benchmarkId":"arc-prize:v1-semi-private:overall","benchmarkName":"ARC-AGI-1 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":87.5,"unit":"percent","family":"arc-prize:v1-semi-private:overall","index":67.21426787045047,"aboveFit":false,"run":"xhigh effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.21523628600000005,"latency":null},{"benchmarkId":"livebench:livebench-coding-2:overall","benchmarkName":"LiveBench Coding 2.0","category":"coding","domain":null,"eligible":true,"ranked":true,"score":86.8,"unit":"percent","family":"livebench:livebench-coding-2:overall","index":71.04333547590505,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Coding","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.6900000000000001,"latency":null},{"benchmarkId":"livebench:livebench-math:overall","benchmarkName":"LiveBench Mathematics","category":"math","domain":null,"eligible":true,"ranked":true,"score":86.21275,"unit":"percent","family":"livebench:livebench-math:overall","index":59.93119727284908,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Mathematics","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.015334828496042216,"latency":null},{"benchmarkId":"vals-ai:swebench:overall","benchmarkName":"SWE-bench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":86,"unit":"percent","family":"swe-bench-verified","index":64.65787921886916,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/swebench","measuredAt":"2026-09-01","stderr":1.553,"costPerTest":1.077188,"latency":599.776},{"benchmarkId":"realWorldQa","benchmarkName":"RealWorldQA","category":"multimodal","score":85.9,"unit":"percent","family":"realWorldQa","index":60.70278667048427,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"babyVisionPython","benchmarkName":"BabyVision with Python","category":"multimodal","score":85.6,"unit":"percent","family":"babyVisionPython","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:mmlu_pro:overall","benchmarkName":"MMLU Pro","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":84.337,"unit":"percent","family":"mmlu-pro","index":50.73525354754205,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","measuredAt":"2026-09-01","stderr":0.356,"costPerTest":0.010543,"latency":69.848},{"benchmarkId":"osWorldVerified","benchmarkName":"OSWorld-Verified","category":"agentic","score":84.3,"unit":"percent","family":"osWorldVerified","index":68.53278084000104,"aboveFit":false,"run":null,"source":"Tianbao Xie, Danyang Zhang, Jixuan Chen, Xiaochuan Li, Siheng Zhao, Ruisheng Cao, Toh Jing Hua, Zhoujun Cheng, Dongchan Shin, Fangyu Lei, Yitao Liu, Yiheng Xu, Shuyan Zhou, Silvio Savarese, Caiming Xiong, Victor Zhong, Tao Yu","sourceUrl":"https://os-world.github.io/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:lcb:overall","benchmarkName":"LiveCodeBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":83.996,"unit":"percent","family":"livecodebench","index":59.038710774812856,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","measuredAt":"2026-09-01","stderr":1.035,"costPerTest":0.044983,"latency":149.298},{"benchmarkId":"vals-ai:mmmu:overall","benchmarkName":"MMMU Pro","category":"multimodal","domain":null,"eligible":true,"ranked":true,"score":83.931,"unit":"percent","family":"mmmu-pro","index":64.38164785155233,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmmu","measuredAt":"2026-09-01","stderr":0.883,"costPerTest":0.012642,"latency":43.483},{"benchmarkId":"vals-ai:medscribe:overall","benchmarkName":"MedScribe","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":83.849,"unit":"percent","family":"vals-ai:medscribe:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medscribe","measuredAt":"2026-10-29","stderr":1.977,"costPerTest":0.046732,"latency":89.705},{"benchmarkId":"charxivNoTools","benchmarkName":"CharXiv Reasoning without tools","category":"multimodal","score":83.7,"unit":"percent","family":"charxivNoTools","index":64.17540993655344,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vulcanBench","benchmarkName":"VulcanBench v3","category":"coding","score":82.6,"unit":"percent","family":"vulcanBench","index":62.527678725370734,"aboveFit":false,"run":null,"source":"VulcanBench contributors","sourceUrl":"https://github.com/morganlinton/VulcanBench/tree/main","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:legal_bench:overall","benchmarkName":"LegalBench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":82.428,"unit":"percent","family":"vals-ai:legal_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","measuredAt":"2026-10-13","stderr":0.474,"costPerTest":0.004666,"latency":47.806},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":82,"unit":"percent","family":"lcr","index":63.55665255719484,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"androidWorld","benchmarkName":"AndroidWorld","category":"agentic","score":81.9,"unit":"percent","family":"androidWorld","index":61.72876508843363,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-reasoning:overall","benchmarkName":"LiveBench Reasoning","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":80.02875,"unit":"percent","family":"livebench","index":65.29168475453315,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Reasoning","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.016406930693069303,"latency":null},{"benchmarkId":"aaTerminalBench21","benchmarkName":"Artificial Analysis Terminal-Bench v2.1","category":"agentic","score":79.8,"unit":"percent","family":"aaTerminalBench21","index":62.33127305744256,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/terminalbench-v2-1","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"ifBench","benchmarkName":"Instruction Following Benchmark","category":"instruction","score":79.5,"unit":"percent","family":"ifbench-family","index":63.31722327101039,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-data-analysis:overall","benchmarkName":"LiveBench Data Analysis","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":76.58566666666667,"unit":"percent","family":"livebench","index":60.83139755296754,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Data%20Analysis","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.01988466666666667,"latency":null},{"benchmarkId":"aaMmmuPro","benchmarkName":"Artificial Analysis MMMU-Pro","category":"multimodal","score":76.3,"unit":"percent","family":"mmmu-pro","index":54.45057085642797,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmmu-pro","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-design:overall","benchmarkName":"LiveBench Design","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":75.9,"unit":"percent","family":"livebench-2-agentic","index":74.09750168683289,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Design","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.38361199999999995,"latency":null},{"benchmarkId":"livebench:livebench-coding:overall","benchmarkName":"LiveBench Coding","category":"coding","domain":null,"eligible":true,"ranked":true,"score":75.689,"unit":"percent","family":"livebench:livebench-coding:overall","index":61.27394826138746,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Coding","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.018570085470085467,"latency":null},{"benchmarkId":"livebench:livebench-language:overall","benchmarkName":"LiveBench Language","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":74.34566666666667,"unit":"percent","family":"livebench:livebench-language:overall","index":58.62953649128403,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Language","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.031849333333333334,"latency":null},{"benchmarkId":"terminalBench21","benchmarkName":"Terminal-Bench 2.1","category":"agentic","score":73,"unit":"percent","family":"terminal-bench-2-1","index":63.835562813882376,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-instruction-following:overall","benchmarkName":"LiveBench Instruction Following","category":"instruction","domain":null,"eligible":true,"ranked":true,"score":72.65825000000001,"unit":"percent","family":"livebench:livebench-instruction-following:overall","index":73.60623283330568,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=IF","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.0125445,"latency":null},{"benchmarkId":"vals-ai:tax_eval_v2:overall","benchmarkName":"TaxEval v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":70.85,"unit":"percent","family":"vals-ai:tax_eval_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","measuredAt":"2026-09-01","stderr":0.893,"costPerTest":0.046465,"latency":94.491},{"benchmarkId":"coworkBench","benchmarkName":"CoWorkBench","category":"agentic","score":70.7,"unit":"percent","family":"coworkBench","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"parseBenchLlamaIndex","benchmarkName":"ParseBench: document parsing for AI agents (LlamaIndex leaderboard)","category":"multimodal","score":69.03,"unit":"percent","family":"parseBenchLlamaIndex","index":70.03330498191903,"aboveFit":false,"run":null,"source":"Boyang Zhang, Sebastián G. Acosta, Preston Carlson, Sacha Bron, Pierre-Loïc Doulcet, Daniel B. Ospina and Simon Suo (LlamaIndex)","sourceUrl":"https://arxiv.org/abs/2604.08538","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-portuguese:overall","benchmarkName":"EuroEval Portuguese","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":68.48750000000001,"unit":"percent","family":"euroeval","index":65.14201443371431,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/portuguese","measuredAt":null,"stderr":0.4914710062658834,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:cyber:overall","benchmarkName":"CyberBench v1.1","category":null,"domain":"Other","eligible":false,"ranked":true,"score":67.798,"unit":"percent","family":"vals-ai:cyber:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/cyber","measuredAt":"2026-11-05","stderr":5.68,"costPerTest":0.990223,"latency":2496.156},{"benchmarkId":"euroeval:euroeval-french:overall","benchmarkName":"EuroEval French","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":66.81375,"unit":"percent","family":"euroeval","index":62.87112080261738,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/french","measuredAt":null,"stderr":0.4162349847141636,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-italian:overall","benchmarkName":"EuroEval Italian","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":66.155,"unit":"percent","family":"euroeval","index":61.977348626674086,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/italian","measuredAt":null,"stderr":0.49370727663262165,"costPerTest":null,"latency":null},{"benchmarkId":"babyVision","benchmarkName":"BabyVision","category":"multimodal","score":65.7,"unit":"percent","family":"babyVision","index":60.564644710937586,"aboveFit":false,"run":null,"source":"Meta AI","sourceUrl":"https://ai.meta.com/static-resource/muse-spark-1-1-evaluation-report","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"erqa","benchmarkName":"ERQA","category":"multimodal","score":65.5,"unit":"percent","family":"erqa","index":58.81992238289259,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":65.36,"unit":"percent","family":"euroeval","index":60.89871655096263,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.4618352917812733,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:mortgage_tax:overall","benchmarkName":"MortgageTax","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":64.944,"unit":"percent","family":"vals-ai:mortgage_tax:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mortgage_tax","measuredAt":"2026-09-01","stderr":0.947,"costPerTest":0.011604,"latency":79.952},{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":64.86625000000001,"unit":"percent","family":"euroeval","index":60.22881140960097,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":0.6616751941096175,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:vibe-code:overall","benchmarkName":"Vibe Code Bench v1.1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":64.85,"unit":"percent","family":"vals-ai:vibe-code:overall","index":65.64842146087616,"aboveFit":false,"run":"OpenHands","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","measuredAt":"2026-11-09","stderr":4.825,"costPerTest":23.456372,"latency":2619.407},{"benchmarkId":"webArenaVerified","benchmarkName":"WebArena-Verified Browser Agent Benchmark","category":"agentic","score":64.8,"unit":"percent","family":"webArenaVerified","index":null,"aboveFit":false,"run":null,"source":"Amine El Hattami, Megh Thakkar, Nicolas Chapados, Christopher Pal","sourceUrl":"https://openreview.net/forum?id=94tlGxmqkN","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vision2Web","benchmarkName":"Vision2Web","category":"multimodal","score":62.9,"unit":"percent","family":"vision2Web","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-dutch:overall","benchmarkName":"EuroEval Dutch","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":62.10444444444444,"unit":"percent","family":"euroeval","index":56.48167674406597,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/dutch","measuredAt":null,"stderr":0.45014126726376374,"costPerTest":null,"latency":null},{"benchmarkId":"swePro","benchmarkName":"SWE-bench Pro","category":"coding","score":61.7,"unit":"percent","family":"swePro","index":61.38371951321207,"aboveFit":false,"run":null,"source":"Xiang Deng, Jeff Da, Edwin Pan, Yannis Yiming He, Charles Ide, Kanak Garg, Niklas Lauffer, Andrew Park, Nitin Pasari, Chetan Rane, Karmini Sampath, Maya Krishnan, Srivatsa Kundurthy, Sean Hendryx, Zifan Wang, Chen Bo Calvin Zhang, Noah Jacobson, Bing Liu, Brad Kenstler","sourceUrl":"https://arxiv.org/abs/2509.16941","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-agentic-coding:overall","benchmarkName":"LiveBench Agentic Coding","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":61.36366666666667,"unit":"percent","family":"livebench:livebench-agentic-coding:overall","index":72.44599466959164,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Agentic%20Coding","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.9474347222222222,"latency":null},{"benchmarkId":"vals-ai:emb:overall","benchmarkName":"EMB","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":59.661,"unit":"percent","family":"vals-ai:emb:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/emb","measuredAt":"2026-10-20","stderr":3.029,"costPerTest":4.556437,"latency":1370.794},{"benchmarkId":"livebench:livebench-research:overall","benchmarkName":"LiveBench Research","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":59.4,"unit":"percent","family":"livebench-2-agentic","index":62.030538620645345,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Research","measuredAt":"2026-10-14","stderr":null,"costPerTest":1.1274636363636363,"latency":null},{"benchmarkId":"euroeval:euroeval-spanish:overall","benchmarkName":"EuroEval Spanish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":59.036249999999995,"unit":"percent","family":"euroeval","index":52.31884285438363,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/spanish","measuredAt":null,"stderr":0.5714523711561621,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:terminal-bench-2-1:overall","benchmarkName":"Terminal-Bench 2.1","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":58.427,"unit":"percent","family":"terminal-bench-2-1","index":55.31246558741768,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2-1","measuredAt":"2026-10-01","stderr":5.254,"costPerTest":0.677989,"latency":892.585},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":57.1525,"unit":"percent","family":"euroeval","index":49.76302754291012,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.6841509336396465,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:fabv2:overall","benchmarkName":"Finance Agent (v2)","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":48.553,"unit":"percent","family":"vals-ai:fabv2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/fabv2","measuredAt":"2026-10-29","stderr":2.096,"costPerTest":0.748532,"latency":621.733},{"benchmarkId":"aaAutomationBench","benchmarkName":"Artificial Analysis AutomationBench","category":"agentic","score":48.2,"unit":"percent","family":"aaAutomationBench","index":59.60290577982219,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/automationbench-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaTau3Banking","benchmarkName":"Artificial Analysis Tau3-Banking","category":"agentic","score":48,"unit":"percent","family":"aaTau3Banking","index":73.90477131819009,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/tau3-banking","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaSciCode","benchmarkName":"Artificial Analysis SciCode","category":"coding","score":46.6,"unit":"percent","family":"scicode-family","index":56.591805865437834,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/scicode","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gdpvalAaNormalized","benchmarkName":"GDPval-AA normalized","category":"agentic","score":45.9,"unit":"percent","family":"gdpvalAaNormalized","index":66.85120827884788,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaEnterpriseOpsGym","benchmarkName":"Artificial Analysis EnterpriseOps-Gym","category":"agentic","score":44.2,"unit":"percent","family":"aaEnterpriseOpsGym","index":null,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/enterprise-ops-gym-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"agentsLastExam","benchmarkName":"Agents' Last Exam","category":"agentic","score":42.9,"unit":"percent","family":"agentsLastExam","index":76.67250396303038,"aboveFit":true,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"arc-prize:v2-semi-private:overall","benchmarkName":"ARC-AGI-2 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":42.36111111111111,"unit":"percent","family":"arc-prize:v2-semi-private:overall","index":58.637971485260266,"aboveFit":false,"run":"xhigh effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.4469396510416667,"latency":null},{"benchmarkId":"nl2Repo","benchmarkName":"NL2Repo","category":"coding","score":42.3,"unit":"percent","family":"nl2Repo","index":56.60605466068182,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"deepSwe","benchmarkName":"DeepSWE","category":"agentic","score":42.2,"unit":"percent","family":"deepSwe","index":51.754942577411036,"aboveFit":false,"run":null,"source":"Datacurve AI","sourceUrl":"https://deepswe.datacurve.ai/blog","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:ioi:overall","benchmarkName":"IOI","category":"coding","domain":null,"eligible":true,"ranked":true,"score":39.056,"unit":"percent","family":"vals-ai:ioi:overall","index":60.7791314588791,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/ioi","measuredAt":"2026-10-29","stderr":4.938,"costPerTest":3.553383,"latency":5924.968},{"benchmarkId":"arc-prize:v2-public-eval:overall","benchmarkName":"ARC-AGI-2 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":38.75,"unit":"percent","family":"arc-prize:v2-public-eval:overall","index":null,"aboveFit":false,"run":"xhigh effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.4728417612499997,"latency":null},{"benchmarkId":"vals-ai:skillsbench:overall","benchmarkName":"SkillsBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":38.106,"unit":"percent","family":"vals-ai:skillsbench:overall","index":53.506332468555826,"aboveFit":false,"run":"OpenHands","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/skillsbench","measuredAt":"2026-10-29","stderr":4.38,"costPerTest":0.81809,"latency":401.009},{"benchmarkId":"vals-ai:legal_research:overall","benchmarkName":"Legal Research Bench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":36.058,"unit":"percent","family":"vals-ai:legal_research:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_research","measuredAt":"2026-11-09","stderr":3.337,"costPerTest":2.159719,"latency":1600.721},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":33.9,"unit":"percent","family":"hle","index":58.398268369823505,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"jobBench","benchmarkName":"JobBench","category":"agentic","score":33.4,"unit":"percent","family":"jobBench","index":54.967102472366115,"aboveFit":false,"run":null,"source":"Yuetai Li, Yichen Feng, Zhangchen Xu, Zixian Ma, Kaiyuan Zheng, Fengqing Jiang, Xinghua Sun, Rulin Shao, Zichen Chen, Yue Huang, Xinyang Han, Brian Lee, Kayla Xu, Shenglai Zeng, Hang Hua, Xiangliang Zhang, Basel Alomair, Ranjay Krishna, Luke Zettlemoyer, Pang Wei Koh, Bhaskar Ramasubramanian, Luyao Niu, Xiang Yue, Radha Poovendran","sourceUrl":"https://arxiv.org/abs/2605.26329","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-browser:overall","benchmarkName":"LiveBench Browser Use","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":32,"unit":"percent","family":"livebench-2-agentic","index":41.992066619824826,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Browser","measuredAt":"2026-10-14","stderr":null,"costPerTest":1.290276,"latency":null},{"benchmarkId":"vals-ai:tax_agent_bench:overall","benchmarkName":"Tax Agent Bench","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":30.894,"unit":"percent","family":"vals-ai:tax_agent_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench","measuredAt":"2026-11-09","stderr":3.323,"costPerTest":0.81533,"latency":1333.825},{"benchmarkId":"hle","benchmarkName":"Humanity's Last Exam","category":"knowledge","score":30.8,"unit":"percent","family":"hle","index":55.28806086023114,"aboveFit":false,"run":null,"source":"Center for AI Safety, Scale AI, and thousands of expert contributors","sourceUrl":"https://lastexam.ai/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hleNoTools","benchmarkName":"Humanity's Last Exam without tools","category":"knowledge","score":30.8,"unit":"percent","family":"hle","index":55.28806086023114,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:medcode:overall","benchmarkName":"MedCode","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":28.698,"unit":"percent","family":"vals-ai:medcode:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medcode","measuredAt":"2026-10-29","stderr":1.971,"costPerTest":0.050145,"latency":89.379},{"benchmarkId":"livebench:livebench-long-horizon:overall","benchmarkName":"LiveBench Long Agentic","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":27.9,"unit":"percent","family":"livebench-2-agentic","index":38.99360913065095,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Long%20Agentic","measuredAt":"2026-10-14","stderr":null,"costPerTest":6.343303448275862,"latency":null},{"benchmarkId":"aaCweBench","benchmarkName":"Artificial Analysis CWE-Bench-AA","category":"agentic","score":26.7,"unit":"percent","family":"aaCweBench","index":43.63262816905138,"aboveFit":false,"run":null,"source":"Artificial Analysis; dataset by Collinear AI","sourceUrl":"https://artificialanalysis.ai/evaluations/cwe-bench-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaMlcr","benchmarkName":"Medical Long Context Reasoning (MLCR-AA)","category":"reasoning","score":22.8,"unit":"percent","family":"aaMlcr","index":67.6186049280368,"aboveFit":false,"run":null,"source":"Wisedocs and Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mlcr-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"omniscienceAccuracy","benchmarkName":"Artificial Analysis Omniscience Accuracy","category":"knowledge","score":17.3,"unit":"percent","family":"omniscienceAccuracy","index":32.49237899085456,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaGdpPdf","benchmarkName":"Artificial Analysis GDP.pdf","category":"agentic","score":16.4,"unit":"percent","family":"aaGdpPdf","index":62.16837154757575,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdp-pdf","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:proof_bench:overall","benchmarkName":"ProofBench v1.1","category":"math","domain":null,"eligible":true,"ranked":true,"score":16,"unit":"percent","family":"vals-ai:proof_bench:overall","index":52.68053580654214,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/proof_bench","measuredAt":"2026-11-09","stderr":3.685,"costPerTest":2.298832,"latency":1453.872},{"benchmarkId":"bug-hunt-bench","benchmarkName":"Bug Hunt Bench","category":"coding","score":15,"unit":"percent","family":"bug-hunt-bench","index":60.378022187446916,"aboveFit":false,"run":null,"source":"Pawel Huryn","sourceUrl":"https://bughunt.productcompass.pm/method","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:code-migration:overall","benchmarkName":"Code Migration","category":"coding","domain":null,"eligible":true,"ranked":true,"score":14.157,"unit":"percent","family":"vals-ai:code-migration:overall","index":52.709903451165715,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/code-migration","measuredAt":"2026-10-29","stderr":4.189,"costPerTest":21.677954,"latency":17150.223},{"benchmarkId":"aaCyberIndex","benchmarkName":"Artificial Analysis Cyber Index","category":"agentic","score":12.7,"unit":"percent","family":"aaCyberIndex","index":40.29352311645131,"aboveFit":false,"run":null,"source":"Artificial Analysis with the Cyber Index Alliance (Collinear AI, IBM, NVIDIA, Pasta Lab, Vercel)","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-cyber-index","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaDeepsecBench","benchmarkName":"Artificial Analysis DeepsecBench-AA","category":"agentic","score":11.5,"unit":"percent","family":"aaDeepsecBench","index":59.130290621892186,"aboveFit":false,"run":null,"source":"Artificial Analysis; dataset by Vercel","sourceUrl":"https://artificialanalysis.ai/evaluations/deepsecbench-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaTerminalBench4","benchmarkName":"Artificial Analysis Terminal-Bench v4.0","category":"agentic","score":5.6,"unit":"percent","family":"aaTerminalBench4","index":53.44564694643363,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/terminalbench-v4-0","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":5.4,"unit":"percent","family":"critpt","index":47.3028859229469,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mazeBench","benchmarkName":"MazeBench: visual spatial reasoning in a 3D open world","category":"agentic","score":0,"unit":"percent","family":"mazeBench","index":61.97866734834783,"aboveFit":false,"run":null,"source":"Jonathan Pappas and David Pappas","sourceUrl":"https://mazebench.com/blog?post=maze-bench-results","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaCyberGymE2e","benchmarkName":"Artificial Analysis CyberGym-E2E-AA","category":"agentic","score":0,"unit":"percent","family":"aaCyberGymE2e","index":5,"aboveFit":false,"run":null,"source":"Artificial Analysis; dataset by Berkeley RDI","sourceUrl":"https://artificialanalysis.ai/evaluations/cybergym-e2e-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:programbench:overall","benchmarkName":"ProgramBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":0,"unit":"percent","family":"vals-ai:programbench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/programbench","measuredAt":"2026-11-09","stderr":0,"costPerTest":17.990872,"latency":21400.403},{"benchmarkId":"vals-ai:terminal-bench-science:overall","benchmarkName":"Terminal-Bench-Science","category":null,"domain":"Science","eligible":false,"ranked":true,"score":0,"unit":"percent","family":"vals-ai:terminal-bench-science:overall","index":null,"aboveFit":false,"run":"Mini-SWE-agent","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-science","measuredAt":"2026-11-09","stderr":0,"costPerTest":11.135637,"latency":14549.775},{"benchmarkId":"lmarena:agent-task-outcome:overall","benchmarkName":"Agent Arena task outcome","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":-2.691039276240015,"unit":"index","family":"agent-arena","index":64.31834514776747,"aboveFit":false,"run":null,"source":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset","measuredAt":"2026-11-11","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lmarena:agent-bash-recovery:overall","benchmarkName":"Agent Arena command recovery","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":-5.901495043678318,"unit":"index","family":"agent-arena","index":61.678847199519986,"aboveFit":false,"run":null,"source":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset","measuredAt":"2026-10-08","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lmarena:agent-steerability:overall","benchmarkName":"Agent Arena steerability","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":-5.999233512112712,"unit":"index","family":"agent-arena","index":61.59849085587745,"aboveFit":false,"run":null,"source":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset","measuredAt":"2026-11-11","stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":0.15,"outputPerMillion":1.875,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":113,"timeToFirstToken":0.566,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":{"elo":1436.1038822396554,"eloUpper":1440.7188045152084,"eloLower":1431.4889599641028,"votes":32659,"rank":88,"rankUpper":null,"rankLower":null,"arenaName":"Qwen3.8-27B","organization":"alibaba","updatedAt":"2026-11-09"},"quality":{"coveragePercent":100,"benchmarkCount":59,"categoryCount":8,"confidence":"high","state":"available"},"hosts":[{"host":"vultr","name":"Vultr","kind":"cloud","url":"https://api.vultrinference.com/","inputPerMillion":0.15,"outputPerMillion":1,"cacheReadPerMillion":null,"contextTokens":262144,"maxOutputTokens":32768,"deprecated":false},{"host":"chutes","name":"Chutes","kind":"host","url":"https://llm.chutes.ai/v1/models","inputPerMillion":0.039,"outputPerMillion":1.25,"cacheReadPerMillion":0.0039,"contextTokens":262144,"maxOutputTokens":65536,"deprecated":false},{"host":"cerebras","name":"Cerebras","kind":"host","url":"https://inference-docs.cerebras.ai/models/overview","inputPerMillion":0.99,"outputPerMillion":1.49,"cacheReadPerMillion":0.99,"contextTokens":131072,"maxOutputTokens":40960,"deprecated":false},{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":0.15,"outputPerMillion":1.875,"cacheReadPerMillion":0.0375,"contextTokens":1048575,"maxOutputTokens":235929,"deprecated":false},{"host":"regolo","name":"Regolo","kind":"host","url":"https://docs.regolo.ai/","inputPerMillion":0.58,"outputPerMillion":2.42,"cacheReadPerMillion":null,"contextTokens":120000,"maxOutputTokens":120000,"deprecated":false},{"host":"deepinfra","name":"Deep Infra","kind":"host","url":"https://deepinfra.com/models","inputPerMillion":0.2,"outputPerMillion":2.5,"cacheReadPerMillion":0.05,"contextTokens":262144,"maxOutputTokens":32768,"deprecated":false},{"host":"coreweave","name":"CoreWeave","kind":"host","url":"https://docs.wandb.ai/inference","inputPerMillion":0.4,"outputPerMillion":3,"cacheReadPerMillion":0.15,"contextTokens":262144,"maxOutputTokens":262144,"deprecated":false},{"host":"novita-ai","name":"Novita AI","kind":"host","url":"https://novita.ai/docs/guides/introduction","inputPerMillion":0.42,"outputPerMillion":3,"cacheReadPerMillion":0.085,"contextTokens":1000000,"maxOutputTokens":131072,"deprecated":false},{"host":"nebius","name":"Nebius","kind":"host","url":"https://docs.tokenfactory.nebius.com/","inputPerMillion":0.45,"outputPerMillion":3,"cacheReadPerMillion":0.1,"contextTokens":262144,"maxOutputTokens":262144,"deprecated":false},{"host":"ovhcloud","name":"OVHcloud","kind":"cloud","url":"https://www.ovhcloud.com/en/public-cloud/ai-endpoints/catalog//","inputPerMillion":0.47,"outputPerMillion":3.19,"cacheReadPerMillion":null,"contextTokens":262144,"maxOutputTokens":262144,"deprecated":false},{"host":"cloudflare","name":"Cloudflare Workers AI","kind":"cloud","url":"https://developers.cloudflare.com/workers-ai/models/","inputPerMillion":0.45,"outputPerMillion":3.2,"cacheReadPerMillion":0.05,"contextTokens":262144,"maxOutputTokens":262144,"deprecated":false},{"host":"berget-ai","name":"Berget AI","kind":"host","url":"https://api.berget.ai","inputPerMillion":0.46,"outputPerMillion":3.48,"cacheReadPerMillion":null,"contextTokens":262144,"maxOutputTokens":65536,"deprecated":false},{"host":"scaleway","name":"Scaleway","kind":"cloud","url":"https://www.scaleway.com/en/docs/generative-apis/","inputPerMillion":0.684,"outputPerMillion":3.762,"cacheReadPerMillion":0.137,"contextTokens":262144,"maxOutputTokens":32768,"deprecated":false},{"host":"groq","name":"Groq","kind":"host","url":"https://console.groq.com/docs/models","inputPerMillion":0.8,"outputPerMillion":4,"cacheReadPerMillion":null,"contextTokens":131072,"maxOutputTokens":16384,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:arc-prize-foundation","name":"ARC Prize Foundation","url":"https://arcprize.org","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:livebench","name":"LiveBench","url":"https://livebench.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:lmarena","name":"LMArena","url":"https://huggingface.co","license":"CC BY 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gpt-5-2","providerId":"openai","slug":"gpt-5-2","name":"GPT-5.2","provider":"OpenAI","description":"GPT-5.2 is a reasoning model from OpenAI. 51 benchmarks count toward its index, in 8 categories.","logoUrl":null,"releaseDate":"2025-12-11","contextWindowTokens":400000,"modalities":["file","image","text"],"openWeights":false,"details":{"apiId":"openai/gpt-5.2","maxOutputTokens":128000,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.175,"weightsUrl":null},"reasoning":true,"overallScore":59.045279153663806,"overallScoreRange":{"low":56.781026257790515,"high":61.3095320495371},"badges":[],"capabilityScores":{"agentic":55.30994836521483,"coding":61.03530818195575,"reasoning":63.77826022263975,"multimodal":51.51238976125241,"knowledge":60.564108193753135,"multilingual":51.71411112460471,"instruction":55.53077161932799,"math":62.626952410882836},"capabilityCounts":{"math":8,"knowledge":6,"multimodal":9,"agentic":9,"reasoning":9,"coding":7,"instruction":2,"multilingual":1},"benchmarks":[{"benchmarkId":"aime2025","benchmarkName":"American Invitational Mathematics Examination 2025","category":"math","score":100,"unit":"percent","family":"aime2025","index":53.269803393382645,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaAime2025","benchmarkName":"Artificial Analysis AIME 2025","category":"math","score":99,"unit":"percent","family":"aaAime2025","index":null,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/aime-2025","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:aime:overall","benchmarkName":"AIME","category":"math","domain":null,"eligible":true,"ranked":true,"score":96.875,"unit":"percent","family":"vals-ai:aime:overall","index":57.841374032311265,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/aime","measuredAt":"2026-04-16","stderr":0.208,"costPerTest":0.135323,"latency":120.31},{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":96.1,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":64.8294986715143,"aboveFit":false,"run":"xhigh effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.7,"costPerTest":null,"latency":null},{"benchmarkId":"arc-prize:v1-public-eval:overall","benchmarkName":"ARC-AGI-1 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":95,"unit":"percent","family":"arc-prize:v1-public-eval:overall","index":null,"aboveFit":false,"run":"xhigh effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.65,"latency":null},{"benchmarkId":"vals-ai:medqa:overall","benchmarkName":"MedQA","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":94.133,"unit":"percent","family":"vals-ai:medqa:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medqa","measuredAt":"2026-04-16","stderr":0.214,"costPerTest":0.021634,"latency":19.483},{"benchmarkId":"vals-ai:mgsm:overall","benchmarkName":"MGSM","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":94,"unit":"percent","family":"vals-ai:mgsm:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mgsm","measuredAt":"2026-01-09","stderr":1.416,"costPerTest":0.010576,"latency":10.596},{"benchmarkId":"livebench:livebench-math:overall","benchmarkName":"LiveBench Mathematics","category":"math","domain":null,"eligible":true,"ranked":true,"score":93.166,"unit":"percent","family":"livebench:livebench-math:overall","index":69.62207595033342,"aboveFit":false,"run":"high effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Mathematics","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.10874327176781001,"latency":null},{"benchmarkId":"gpqa","benchmarkName":"Graduate-Level Google-Proof Q&A","category":"knowledge","score":92.4,"unit":"percent","family":"gpqa-diamond","index":61.95585590043767,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"ai2dTest","benchmarkName":"AI2D test split","category":"multimodal","score":92.2,"unit":"percent","family":"ai2dTest","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"countBench","benchmarkName":"CountBench","category":"multimodal","score":91.9,"unit":"percent","family":"countBench","index":43.003565153027786,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:gpqa:overall","benchmarkName":"GPQA Diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":91.666,"unit":"percent","family":"gpqa-diamond","index":61.282651157046544,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","measuredAt":"2026-09-01","stderr":1.837,"costPerTest":0.149037,"latency":173.214},{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":91.4,"unit":"percent","family":"gpqa-diamond","index":61.038683225517886,"aboveFit":false,"run":"xhigh effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.7999999999999998,"costPerTest":null,"latency":null},{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":90.3,"unit":"percent","family":"gpqa-diamond","index":60.029793283106116,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"tau2-bench:telecom:overall","benchmarkName":"τ²-bench Telecom","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":89.69,"unit":"percent","family":"tau2-bench","index":62.3053125535877,"aboveFit":false,"run":"high effort · Sierra","source":"Sierra Research","sourceUrl":"https://github.com/sierra-research/tau2-bench","measuredAt":"2026-03-02","stderr":null,"costPerTest":0.09312,"latency":null},{"benchmarkId":"aaMmluPro","benchmarkName":"Artificial Analysis MMLU-Pro","category":"knowledge","score":87.4,"unit":"percent","family":"mmlu-pro","index":54.59426691586651,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmlu-pro","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"dynaMath","benchmarkName":"DynaMath","category":"multimodal","score":86.8,"unit":"percent","family":"dynaMath","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmmu","benchmarkName":"Massive Multi-discipline Multimodal Understanding","category":"multimodal","score":86.7,"unit":"percent","family":"mmmu","index":55.95644160079212,"aboveFit":false,"run":null,"source":"MMMU authors","sourceUrl":"https://arxiv.org/abs/2401.05508","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:mmmu:overall","benchmarkName":"MMMU Pro","category":"multimodal","domain":null,"eligible":true,"ranked":true,"score":86.667,"unit":"percent","family":"mmmu-pro","index":67.94231180891836,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmmu","measuredAt":"2026-09-01","stderr":0.822,"costPerTest":0.061121,"latency":96.324},{"benchmarkId":"vals-ai:mmlu_pro:overall","benchmarkName":"MMLU Pro","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":86.234,"unit":"percent","family":"mmlu-pro","index":53.125246482478865,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","measuredAt":"2026-09-01","stderr":0.344,"costPerTest":0.03698,"latency":35.413},{"benchmarkId":"arc-prize:v1-semi-private:overall","benchmarkName":"ARC-AGI-1 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":86.17,"unit":"percent","family":"arc-prize:v1-semi-private:overall","index":66.60281885349453,"aboveFit":false,"run":"xhigh effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.96,"latency":null},{"benchmarkId":"videoMmeWithSub","benchmarkName":"Video-MME with subtitle","category":"multimodal","score":86,"unit":"percent","family":"videoMmeWithSub","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"videoMmmu","benchmarkName":"VideoMMMU","category":"multimodal","score":85.9,"unit":"percent","family":"videoMmmu","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"videoMmeNoSub","benchmarkName":"Video-MME without subtitle","category":"multimodal","score":85.8,"unit":"percent","family":"videoMmeNoSub","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"omniDocBench15","benchmarkName":"OmniDocBench 1.5","category":"multimodal","score":85.7,"unit":"percent","family":"omniDocBench15","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mlvuAvg","benchmarkName":"MLVU mean average","category":"multimodal","score":85.6,"unit":"percent","family":"mlvuAvg","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:lcb:overall","benchmarkName":"LiveCodeBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":85.361,"unit":"percent","family":"livecodebench","index":60.1713075524016,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","measuredAt":"2026-09-01","stderr":1.027,"costPerTest":0.135162,"latency":151.513},{"benchmarkId":"tau2Bench","benchmarkName":"τ²-Bench Tool-Agent-User Evaluation","category":"agentic","score":84.8,"unit":"percent","family":"tau2-bench","index":59.06044003629105,"aboveFit":false,"run":null,"source":"Victor Barres, Honghua Dong, Soham Ray, Xujie Si, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2506.07982","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:medscribe:overall","benchmarkName":"MedScribe","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":84.387,"unit":"percent","family":"vals-ai:medscribe:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medscribe","measuredAt":"2026-10-29","stderr":1.856,"costPerTest":0.115422,"latency":126.205},{"benchmarkId":"realWorldQa","benchmarkName":"RealWorldQA","category":"multimodal","score":83.3,"unit":"percent","family":"realWorldQa","index":56.1890694271189,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-reasoning:overall","benchmarkName":"LiveBench Reasoning","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":83.2115,"unit":"percent","family":"livebench","index":69.4147273057686,"aboveFit":false,"run":"high effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Reasoning","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.11387079207920792,"latency":null},{"benchmarkId":"mathVision","benchmarkName":"MathVision","category":"multimodal","score":83,"unit":"percent","family":"mathVision","index":54.220828488611374,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"tau2-bench:airline:overall","benchmarkName":"τ²-bench Airline","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":83,"unit":"percent","family":"tau2-bench","index":57.86600843483216,"aboveFit":false,"run":"high effort · Sierra","source":"Sierra Research","sourceUrl":"https://github.com/sierra-research/tau2-bench","measuredAt":"2026-03-02","stderr":null,"costPerTest":0.11379,"latency":null},{"benchmarkId":"vals-ai:legal_bench:overall","benchmarkName":"LegalBench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":82.764,"unit":"percent","family":"vals-ai:legal_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","measuredAt":"2026-10-13","stderr":0.395,"costPerTest":0.017566,"latency":17.138},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":82.7,"unit":"percent","family":"lcr","index":64.0369089251048,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"charxiv","benchmarkName":"CharXiv Reasoning","category":"multimodal","score":82.1,"unit":"percent","family":"charxiv","index":57.654921770422675,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"tau2-bench:retail:overall","benchmarkName":"τ²-bench Retail","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":81.58,"unit":"percent","family":"tau2-bench","index":56.92373461590348,"aboveFit":false,"run":"high effort · Sierra","source":"Sierra Research","sourceUrl":"https://github.com/sierra-research/tau2-bench","measuredAt":"2026-04-30","stderr":null,"costPerTest":0.07156,"latency":null},{"benchmarkId":"sweVerified","benchmarkName":"Software Engineering Benchmark Verified","category":"coding","score":80,"unit":"percent","family":"swe-bench-verified","index":60.232533752167,"aboveFit":false,"run":null,"source":"Carlos E. Jimenez, John Yang, Alexander Wettig, Shunyu Yao, Kexin Pei, Ofir Press, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2310.06770","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-language:overall","benchmarkName":"LiveBench Language","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":79.809,"unit":"percent","family":"livebench:livebench-language:overall","index":64.80752647775051,"aboveFit":false,"run":"high effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Language","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.041860666666666664,"latency":null},{"benchmarkId":"mmmuPro","benchmarkName":"Massive Multi-discipline Multimodal Understanding Pro","category":"multimodal","score":79.5,"unit":"percent","family":"mmmu-pro","index":58.61509010480931,"aboveFit":false,"run":null,"source":"MMMU-Pro authors","sourceUrl":"https://arxiv.org/abs/2409.02813","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"weMath","benchmarkName":"We-Math","category":"multimodal","score":79,"unit":"percent","family":"weMath","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-data-analysis:overall","benchmarkName":"LiveBench Data Analysis","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":78.16333333333334,"unit":"percent","family":"livebench","index":62.87516058755476,"aboveFit":false,"run":"high effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Data%20Analysis","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.166278,"latency":null},{"benchmarkId":"livebench:livebench-coding:overall","benchmarkName":"LiveBench Coding","category":"coding","domain":null,"eligible":true,"ranked":true,"score":76.0715,"unit":"percent","family":"livebench:livebench-coding:overall","index":61.81189151740449,"aboveFit":false,"run":"high effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Coding","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.028731623931623932,"latency":null},{"benchmarkId":"vStar","benchmarkName":"V*","category":"multimodal","score":75.9,"unit":"percent","family":"vStar","index":32.70412943711976,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:swebench:overall","benchmarkName":"SWE-bench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":75.8,"unit":"percent","family":"swe-bench-verified","index":57.1347919254755,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/swebench","measuredAt":"2026-09-01","stderr":1.917,"costPerTest":1.453372,"latency":640.472},{"benchmarkId":"vals-ai:tax_eval_v2:overall","benchmarkName":"TaxEval v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":75.756,"unit":"percent","family":"vals-ai:tax_eval_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","measuredAt":"2026-09-01","stderr":0.845,"costPerTest":0.066435,"latency":60.479},{"benchmarkId":"aaIfBench","benchmarkName":"Artificial Analysis IFBench","category":"instruction","score":75.4,"unit":"percent","family":"ifbench-family","index":59.48572975020944,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-swe-bench-verified:overall","benchmarkName":"SWE-Bench verified","category":"coding","domain":null,"eligible":true,"ranked":true,"score":73.7603305785124,"unit":"percent","family":"swe-bench-verified","index":55.630418287483636,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.001783206492828,"costPerTest":null,"latency":null},{"benchmarkId":"swebench-experiments:swe-bench-verified:overall","benchmarkName":"SWE-bench Verified","category":"coding","domain":null,"eligible":true,"ranked":true,"score":72.8,"unit":"percent","family":"swe-bench-verified","index":54.92211919212442,"aboveFit":false,"run":"high effort · mini-SWE-agent","source":"SWE-bench team","sourceUrl":"https://github.com/SWE-bench/experiments/tree/main/evaluation/verified","measuredAt":"2026-09-01","stderr":null,"costPerTest":0.4735670177,"latency":null},{"benchmarkId":"ccOcr","benchmarkName":"CC-OCR","category":"multimodal","score":70.3,"unit":"percent","family":"ccOcr","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tiers-1-3-v2-private:overall","benchmarkName":"FrontierMath-Tiers-1-3-v2-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":67.4,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tiers-1-3-v2-private:overall","index":66.37080164685646,"aboveFit":false,"run":"xhigh effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.8000000000000003,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:mortgage_tax:overall","benchmarkName":"MortgageTax","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":67.13,"unit":"percent","family":"vals-ai:mortgage_tax:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mortgage_tax","measuredAt":"2026-09-01","stderr":0.918,"costPerTest":0.050478,"latency":70.666},{"benchmarkId":"swe-bench:swe-bench-multilingual:overall","benchmarkName":"SWE-bench Multilingual","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":66.7,"unit":"percent","family":"swe-bench-multilingual","index":null,"aboveFit":false,"run":"high effort · mini-SWE-agent","source":"SWE-bench team","sourceUrl":"https://www.swebench.com/","measuredAt":"2026-02-20","stderr":null,"costPerTest":0.5364109905,"latency":null},{"benchmarkId":"swebench-experiments:swe-bench-multilingual:overall","benchmarkName":"SWE-bench Multilingual","category":"coding","domain":null,"eligible":true,"ranked":true,"score":66.7,"unit":"percent","family":"swe-bench-multilingual","index":null,"aboveFit":false,"run":"high effort · mini-SWE-agent","source":"SWE-bench team","sourceUrl":"https://github.com/SWE-bench/experiments/tree/main/evaluation/multilingual","measuredAt":"2026-09-02","stderr":null,"costPerTest":0.5364109905,"latency":null},{"benchmarkId":"vals-ai:case_law_v2:overall","benchmarkName":"CaseLaw v2","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":66.024,"unit":"percent","family":"vals-ai:case_law_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/case_law_v2","measuredAt":"2026-05-04","stderr":0.567,"costPerTest":0.103778,"latency":86.227},{"benchmarkId":"vals-ai:corp_fin_v2:overall","benchmarkName":"CorpFin v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":65.889,"unit":"percent","family":"vals-ai:corp_fin_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","measuredAt":"2026-08-12","stderr":0.934,"costPerTest":0.153756,"latency":26.038},{"benchmarkId":"browseComp","benchmarkName":"BrowseComp","category":"agentic","score":65.8,"unit":"percent","family":"browseComp","index":51.89229728535281,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:terminal-bench:overall","benchmarkName":"Terminal-Bench 1.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":63.75,"unit":"percent","family":"vals-ai:terminal-bench:overall","index":64.35270176105807,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench","measuredAt":"2026-01-12","stderr":5.375,"costPerTest":1.395387,"latency":1429.571},{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":63.50250000000001,"unit":"percent","family":"euroeval","index":58.37851644953621,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":0.6224661135355081,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-portuguese:overall","benchmarkName":"EuroEval Portuguese","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":62.53375,"unit":"percent","family":"euroeval","index":57.06414560256075,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/portuguese","measuredAt":null,"stderr":0.3671426834352007,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-french:overall","benchmarkName":"EuroEval French","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":61.99374999999999,"unit":"percent","family":"euroeval","index":56.33148985302089,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/french","measuredAt":null,"stderr":0.3545375509025807,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-instruction-following:overall","benchmarkName":"LiveBench Instruction Following","category":"instruction","domain":null,"eligible":true,"ranked":true,"score":61.77074999999999,"unit":"percent","family":"livebench:livebench-instruction-following:overall","index":51.57581348844654,"aboveFit":false,"run":"high effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=IF","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.1017,"latency":null},{"benchmarkId":"euroeval:euroeval-italian:overall","benchmarkName":"EuroEval Italian","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":60.1125,"unit":"percent","family":"euroeval","index":53.779066466313765,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/italian","measuredAt":null,"stderr":0.4148795005781799,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-2025-02-28-public:overall","benchmarkName":"FrontierMath-2025-02-28-Public","category":"math","domain":null,"eligible":false,"ranked":true,"score":60,"unit":"percent","family":"epoch-ai:epoch-frontiermath-2025-02-28-public:overall","index":null,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":16.329931618554518,"costPerTest":null,"latency":null},{"benchmarkId":"arc-prize:v2-public-eval:overall","benchmarkName":"ARC-AGI-2 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":59.809999999999995,"unit":"percent","family":"arc-prize:v2-public-eval:overall","index":null,"aboveFit":false,"run":"xhigh effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":2.054,"latency":null},{"benchmarkId":"erqa","benchmarkName":"ERQA","category":"multimodal","score":59.8,"unit":"percent","family":"erqa","index":53.54312104823315,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:finance_agent:overall","benchmarkName":"Finance Agent v1.1","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":58.535,"unit":"percent","family":"vals-ai:finance_agent:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/finance_agent","measuredAt":"2026-06-04","stderr":2.87,"costPerTest":0.977848,"latency":585.414},{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":57.06857142857143,"unit":"percent","family":"euroeval","index":49.64915578289566,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.4411395262642569,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-dutch:overall","benchmarkName":"EuroEval Dutch","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":56.23,"unit":"percent","family":"euroeval","index":48.511407304112865,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/dutch","measuredAt":null,"stderr":0.5323022440421847,"costPerTest":null,"latency":null},{"benchmarkId":"simpleVqa","benchmarkName":"SimpleVQA","category":"multimodal","score":55.8,"unit":"percent","family":"simpleVqa","index":47.06072996908204,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"swePro","benchmarkName":"SWE-bench Pro","category":"coding","score":55.6,"unit":"percent","family":"swePro","index":56.18889862796538,"aboveFit":false,"run":null,"source":"Xiang Deng, Jeff Da, Edwin Pan, Yannis Yiming He, Charles Ide, Kanak Garg, Niklas Lauffer, Andrew Park, Nitin Pasari, Chetan Rane, Karmini Sampath, Maya Krishnan, Srivatsa Kundurthy, Sean Hendryx, Zifan Wang, Chen Bo Calvin Zhang, Noah Jacobson, Bing Liu, Brad Kenstler","sourceUrl":"https://arxiv.org/abs/2509.16941","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:ioi-v1:overall","benchmarkName":"IOI v1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":54.833,"unit":"percent","family":"vals-ai:ioi-v1:overall","index":67.4229358519386,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/ioi-v1","measuredAt":"2026-08-09","stderr":8.318,"costPerTest":3.871992,"latency":6739.612},{"benchmarkId":"euroeval:euroeval-spanish:overall","benchmarkName":"EuroEval Spanish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":53.52625,"unit":"percent","family":"euroeval","index":44.843040669264006,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/spanish","measuredAt":null,"stderr":0.43218015630058726,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:vibe-code:overall","benchmarkName":"Vibe Code Bench v1.1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":53.499,"unit":"percent","family":"vals-ai:vibe-code:overall","index":61.552802323290955,"aboveFit":false,"run":"OpenHands","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","measuredAt":"2026-11-09","stderr":5.067,"costPerTest":17.745182,"latency":4971.341},{"benchmarkId":"arc-prize:v2-semi-private:overall","benchmarkName":"ARC-AGI-2 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":52.910000000000004,"unit":"percent","family":"arc-prize:v2-semi-private:overall","index":63.481332430890966,"aboveFit":false,"run":"xhigh effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":1.8982,"latency":null},{"benchmarkId":"vals-ai:terminal-bench-2:overall","benchmarkName":"Terminal-Bench 2.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":51.685,"unit":"percent","family":"vals-ai:terminal-bench-2:overall","index":57.67138476197594,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2","measuredAt":"2026-06-04","stderr":5.327,"costPerTest":0.469958,"latency":742.684},{"benchmarkId":"livebench:livebench-agentic-coding:overall","benchmarkName":"LiveBench Agentic Coding","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":50.25266666666667,"unit":"percent","family":"livebench:livebench-agentic-coding:overall","index":60.987409608594405,"aboveFit":false,"run":"high effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Agentic%20Coding","measuredAt":"2026-06-25","stderr":null,"costPerTest":1.4209444444444443,"latency":null},{"benchmarkId":"vals-ai:medcode:overall","benchmarkName":"MedCode","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":49.749,"unit":"percent","family":"vals-ai:medcode:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medcode","measuredAt":"2026-10-29","stderr":2.262,"costPerTest":0.018852,"latency":156.651},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":49.58875,"unit":"percent","family":"euroeval","index":39.50075916220257,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.5131702933724828,"costPerTest":null,"latency":null},{"benchmarkId":"lifeVal","benchmarkName":"LifeVal: everyday life tasks","category":"agentic","score":49.3,"unit":"percent","family":"lifeVal","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-chess-puzzles:overall","benchmarkName":"Chess Puzzles","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":48,"unit":"percent","family":"epoch-ai:epoch-chess-puzzles:overall","index":70.41740509234356,"aboveFit":false,"run":"xhigh effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":5.021167315686783,"costPerTest":null,"latency":null},{"benchmarkId":"osWorldVerified","benchmarkName":"OSWorld-Verified","category":"agentic","score":47.3,"unit":"percent","family":"osWorldVerified","index":38.66426963196939,"aboveFit":false,"run":null,"source":"Tianbao Xie, Danyang Zhang, Jixuan Chen, Xiaochuan Li, Siheng Zhao, Ruisheng Cao, Toh Jing Hua, Zhoujun Cheng, Dongchan Shin, Fangyu Lei, Yitao Liu, Yiheng Xu, Shuyan Zhou, Silvio Savarese, Caiming Xiong, Victor Zhong, Tao Yu","sourceUrl":"https://os-world.github.io/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"terminalBenchHard","benchmarkName":"Terminal-Bench Hard","category":"agentic","score":47,"unit":"percent","family":"terminalBenchHard","index":57.65959585369635,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/terminalbench-hard","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gertLabs","benchmarkName":"Gert Labs Composite Game Benchmark","category":"agentic","score":46.54,"unit":"percent","family":"gertLabs","index":53.23097613371336,"aboveFit":false,"run":null,"source":"Gert Labs","sourceUrl":"https://gertlabs.com/rankings","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"omniscienceAccuracy","benchmarkName":"Artificial Analysis Omniscience Accuracy","category":"knowledge","score":44.3,"unit":"percent","family":"omniscienceAccuracy","index":66.51929056046204,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-2025-02-28-private:overall","benchmarkName":"FrontierMath-2025-02-28-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":40.699999999999996,"unit":"percent","family":"epoch-ai:epoch-frontiermath-2025-02-28-private:overall","index":65.84322275055155,"aboveFit":false,"run":"xhigh effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.9000000000000004,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-furniture-assembly:overall","benchmarkName":"Furniture Assembly","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":38.333333333333336,"unit":"percent","family":"epoch-ai:epoch-furniture-assembly:overall","index":64.53362403941502,"aboveFit":false,"run":"xhigh effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":6.329764084940143,"costPerTest":null,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":37.7,"unit":"percent","family":"hle","index":62.210780800936725,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-simpleqa-verified:overall","benchmarkName":"SimpleQA Verified","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":37.1,"unit":"percent","family":"epoch-ai:epoch-simpleqa-verified:overall","index":54.82662743291462,"aboveFit":false,"run":"xhigh effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.5283736211823096,"costPerTest":null,"latency":null},{"benchmarkId":"jobBench","benchmarkName":"JobBench","category":"agentic","score":34.3,"unit":"percent","family":"jobBench","index":55.464891815740955,"aboveFit":false,"run":null,"source":"Yuetai Li, Yichen Feng, Zhangchen Xu, Zixian Ma, Kaiyuan Zheng, Fengqing Jiang, Xinghua Sun, Rulin Shao, Zichen Chen, Yue Huang, Xinyang Han, Brian Lee, Kayla Xu, Shenglai Zeng, Hang Hua, Xiangliang Zhang, Basel Alomair, Ranjay Krishna, Luke Zettlemoyer, Pang Wei Koh, Bhaskar Ramasubramanian, Luyao Niu, Xiang Yue, Radha Poovendran","sourceUrl":"https://arxiv.org/abs/2605.26329","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-mirrorcode:overall","benchmarkName":"MirrorCode","category":"coding","domain":null,"eligible":true,"ranked":true,"score":32.22222222222222,"unit":"percent","family":"epoch-ai:epoch-mirrorcode:overall","index":63.71671629420966,"aboveFit":false,"run":"xhigh effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":9.955810654328078,"costPerTest":null,"latency":null},{"benchmarkId":"tau2-bench:banking-knowledge:overall","benchmarkName":"τ²-bench Banking","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":32.22,"unit":"percent","family":"tau2-bench","index":24.169765811453015,"aboveFit":false,"run":"high effort · Sierra","source":"Sierra Research","sourceUrl":"https://github.com/sierra-research/tau2-bench","measuredAt":"2026-10-20","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tier-4-v2-private:overall","benchmarkName":"FrontierMath-Tier-4-v2-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":31.7,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tier-4-v2-private:overall","index":60.94165072697232,"aboveFit":false,"run":"xhigh effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":7.3,"costPerTest":null,"latency":null},{"benchmarkId":"bioBasicsBench","benchmarkName":"BioBasicsBench: everyday wet-lab science","category":"reasoning","score":31.2,"unit":"percent","family":"bioBasicsBench","index":null,"aboveFit":false,"run":null,"source":"OpenAI, Ligo","sourceUrl":"https://openai.com/index/measuring-how-ai-agents-handle-wet-lab-science/","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-ebr-bench:overall","benchmarkName":"EBR-bench","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":23.015900000000002,"unit":"percent","family":"epoch-ai:epoch-ebr-bench:overall","index":62.98940540307805,"aboveFit":false,"run":"xhigh effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-mystery-game-puzzles:overall","benchmarkName":"Mystery Game Puzzles","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":23,"unit":"percent","family":"epoch-ai:epoch-mystery-game-puzzles:overall","index":56.492540826289144,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":4.229525846816508,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-private:overall","benchmarkName":"FrontierMath-Tier-4-2025-07-01-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":18.8,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-private:overall","index":62.29719211514071,"aboveFit":false,"run":"xhigh effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":5.6000000000000005,"costPerTest":null,"latency":null},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":11.6,"unit":"percent","family":"critpt","index":59.30536248648,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-public:overall","benchmarkName":"FrontierMath-Tier-4-2025-07-01-Public","category":"math","domain":null,"eligible":false,"ranked":true,"score":0,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-public:overall","index":null,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":1.75,"outputPerMillion":14,"batch":{"inputPerMillion":0.875,"outputPerMillion":7},"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":45,"timeToFirstToken":2.508,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":{"elo":1412.0825201781754,"eloUpper":1415.2363509495967,"eloLower":1408.9286894067536,"votes":82951,"rank":139,"rankUpper":null,"rankLower":null,"arenaName":"GPT-5.2","organization":"openai","updatedAt":"2026-11-09"},"quality":{"coveragePercent":100,"benchmarkCount":51,"categoryCount":8,"confidence":"high","state":"available"},"hosts":[{"host":"databricks","name":"Databricks","kind":"cloud","url":"https://docs.databricks.com/aws/en/machine-learning/foundation-models/","inputPerMillion":1.75,"outputPerMillion":14,"cacheReadPerMillion":0.175,"contextTokens":400000,"maxOutputTokens":128000,"deprecated":false},{"host":"digitalocean","name":"DigitalOcean","kind":"cloud","url":"https://docs.digitalocean.com/products/gradient-ai-platform/details/models/","inputPerMillion":1.75,"outputPerMillion":14,"cacheReadPerMillion":0.175,"contextTokens":400000,"maxOutputTokens":128000,"deprecated":false},{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":1.75,"outputPerMillion":14,"cacheReadPerMillion":0.175,"contextTokens":400000,"maxOutputTokens":128000,"deprecated":false},{"host":"sap-ai-core","name":"SAP AI Core","kind":"cloud","url":"https://help.sap.com/docs/sap-ai-core","inputPerMillion":1.75,"outputPerMillion":14,"cacheReadPerMillion":0.175,"contextTokens":400000,"maxOutputTokens":128000,"deprecated":false},{"host":"snowflake","name":"Snowflake Cortex","kind":"cloud","url":"https://docs.snowflake.com/en/user-guide/snowflake-cortex/cortex-rest-api","inputPerMillion":1.75,"outputPerMillion":14,"cacheReadPerMillion":0.18,"contextTokens":400000,"maxOutputTokens":128000,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:livebench","name":"LiveBench","url":"https://livebench.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:sierra-research","name":"Sierra Research","url":"https://github.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:arc-prize-foundation","name":"ARC Prize Foundation","url":"https://arcprize.org","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:swe-bench-team","name":"SWE-bench team","url":"https://github.com","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"apodex-1-1","providerId":"apodex","slug":"apodex-1-1","name":"Apodex 1.1","provider":"Apodex","description":"Apodex 1.1 is a reasoning model from Apodex. 14 benchmarks count toward its index, in 5 categories.","logoUrl":null,"releaseDate":"2026-08-24","contextWindowTokens":null,"modalities":["text"],"openWeights":false,"details":null,"reasoning":true,"overallScore":58.886760364543846,"overallScoreRange":{"low":52.56514608281994,"high":65.20837464626774},"badges":[],"capabilityScores":{"agentic":63.519040584720585,"coding":56.84694846382307,"reasoning":53.72920720385067,"multimodal":58.22466642527356,"knowledge":57.43904539405989,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"agentic":6,"knowledge":3,"reasoning":2,"multimodal":1,"coding":2},"benchmarks":[{"benchmarkId":"deepSearchQa","benchmarkName":"DeepSearchQA","category":"agentic","score":92.4,"unit":"percent","family":"deepSearchQa","index":67.62222151480552,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":86.4,"unit":"percent","family":"gpqa-diamond","index":56.452819850918964,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":79.3,"unit":"percent","family":"lcr","index":61.70423513811355,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaMmmuPro","benchmarkName":"Artificial Analysis MMMU-Pro","category":"multimodal","score":79.2,"unit":"percent","family":"mmmu-pro","index":58.22466642527356,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmmu-pro","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"sweVerified","benchmarkName":"Software Engineering Benchmark Verified","category":"coding","score":77.7,"unit":"percent","family":"swe-bench-verified","index":58.53615132326451,"aboveFit":false,"run":null,"source":"Carlos E. Jimenez, John Yang, Alexander Wettig, Shunyu Yao, Kexin Pei, Ofir Press, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2310.06770","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"terminalBench21","benchmarkName":"Terminal-Bench 2.1","category":"agentic","score":70.8,"unit":"percent","family":"terminal-bench-2-1","index":62.54888101204183,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"frontierScienceResearch","benchmarkName":"FrontierScience Research","category":"knowledge","score":63.3,"unit":"percent","family":"frontierScienceResearch","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hleWithTools","benchmarkName":"Humanity's Last Exam with tools","category":"agentic","score":56.1,"unit":"percent","family":"hleWithTools","index":65.23920963974147,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hle","benchmarkName":"Humanity's Last Exam","category":"knowledge","score":56.1,"unit":"percent","family":"hle","index":80.67136730948494,"aboveFit":true,"run":null,"source":"Center for AI Safety, Scale AI, and thousands of expert contributors","sourceUrl":"https://lastexam.ai/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaSciCode","benchmarkName":"Artificial Analysis SciCode","category":"coding","score":45.5,"unit":"percent","family":"scicode-family","index":55.157745604381624,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/scicode","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"apexAgents","benchmarkName":"APEX-Agents","category":"agentic","score":38.5,"unit":"percent","family":"apexAgents","index":63.55818369352543,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"bioMysteryBenchHumanDifficult","benchmarkName":"BioMysteryBench Human Difficult","category":"knowledge","score":35.3,"unit":"percent","family":"bioMysteryBenchHumanDifficult","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gdpvalAaNormalized","benchmarkName":"GDPval-AA normalized","category":"agentic","score":34.4,"unit":"percent","family":"gdpvalAaNormalized","index":58.6289755193918,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":34.1,"unit":"percent","family":"hle","index":58.59892691882946,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"apexAgentsAa","benchmarkName":"APEX-Agents-AA","category":"agentic","score":31.2,"unit":"percent","family":"apexAgentsAa","index":63.51677212881744,"aboveFit":false,"run":null,"source":"Artificial Analysis / Mercor","sourceUrl":"https://artificialanalysis.ai/evaluations/apex-agents-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"imo2026","benchmarkName":"International Mathematical Olympiad 2026","category":"math","score":30.5,"unit":"percent","family":"imo2026","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"omniscienceAccuracy","benchmarkName":"Artificial Analysis Omniscience Accuracy","category":"knowledge","score":28.2,"unit":"percent","family":"omniscienceAccuracy","index":46.229169217103504,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":4.6,"unit":"percent","family":"critpt","index":45.75417926958779,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":80,"benchmarkCount":14,"categoryCount":5,"confidence":"low","state":"available"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"beam10","providerId":"unknown","slug":"unknown-beam-1-0","name":"Beam 1.0","provider":"Unknown","description":"Beam 1.0 is a model. 9 benchmarks count toward its index, in 4 categories.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":58.79982235694077,"overallScoreRange":{"low":49.90982235694077,"high":67.68982235694077},"badges":[],"capabilityScores":{"agentic":62.340607232128576,"coding":63.28404059499576,"reasoning":55.84943028891552,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":53.37102520852528},"capabilityCounts":{"coding":3,"reasoning":3,"agentic":2,"math":1},"benchmarks":[{"benchmarkId":"livebench:livebench-coding-2:overall","benchmarkName":"LiveBench Coding 2.0","category":"coding","domain":null,"eligible":true,"ranked":true,"score":83.9,"unit":"percent","family":"livebench:livebench-coding-2:overall","index":68.94279275546435,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Coding","measuredAt":"2026-10-14","stderr":null,"costPerTest":1.8515137931034482,"latency":null},{"benchmarkId":"livebench:livebench-reasoning:overall","benchmarkName":"LiveBench Reasoning","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":80.2,"unit":"percent","family":"livebench","index":65.51352782637323,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Reasoning","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.039643495049504957,"latency":null},{"benchmarkId":"arc-prize:v1-public-eval:overall","benchmarkName":"ARC-AGI-1 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":79.125,"unit":"percent","family":"arc-prize:v1-public-eval:overall","index":null,"aboveFit":false,"run":"xhigh effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.23341662649999984,"latency":null},{"benchmarkId":"arc-prize:v1-semi-private:overall","benchmarkName":"ARC-AGI-1 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":74,"unit":"percent","family":"arc-prize:v1-semi-private:overall","index":61.00783048029613,"aboveFit":false,"run":"xhigh effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.3058129759999999,"latency":null},{"benchmarkId":"livebench:livebench-research:overall","benchmarkName":"LiveBench Research","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":72.8,"unit":"percent","family":"livebench-2-agentic","index":71.83037529257948,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Research","measuredAt":"2026-10-14","stderr":null,"costPerTest":2.385131818181818,"latency":null},{"benchmarkId":"livebench:livebench-design:overall","benchmarkName":"LiveBench Design","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":67.9,"unit":"percent","family":"livebench-2-agentic","index":68.24685292746923,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Design","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.374884,"latency":null},{"benchmarkId":"vals-ai:vibe-code:overall","benchmarkName":"Vibe Code Bench v1.1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":66.436,"unit":"percent","family":"vals-ai:vibe-code:overall","index":66.22067517880501,"aboveFit":false,"run":"OpenHands","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","measuredAt":"2026-11-09","stderr":4.062,"costPerTest":null,"latency":145.263},{"benchmarkId":"livebench:livebench-data-analysis:overall","benchmarkName":"LiveBench Data Analysis","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":66.4,"unit":"percent","family":"livebench","index":47.63653867955076,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Data%20Analysis","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.07495849333333333,"latency":null},{"benchmarkId":"vals-ai:biomysterybench:overall","benchmarkName":"BioMysteryBench","category":null,"domain":"Science","eligible":false,"ranked":true,"score":42.593,"unit":"percent","family":"vals-ai:biomysterybench:overall","index":null,"aboveFit":false,"run":"Terminus 2","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/biomysterybench","measuredAt":"2026-11-09","stderr":1.614,"costPerTest":null,"latency":440.046},{"benchmarkId":"vals-ai:legal_research:overall","benchmarkName":"Legal Research Bench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":28.846,"unit":"percent","family":"vals-ai:legal_research:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_research","measuredAt":"2026-11-09","stderr":3.149,"costPerTest":null,"latency":94.568},{"benchmarkId":"arc-prize:v2-public-eval:overall","benchmarkName":"ARC-AGI-2 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":26.25,"unit":"percent","family":"arc-prize:v2-public-eval:overall","index":null,"aboveFit":false,"run":"xhigh effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.6270107858333334,"latency":null},{"benchmarkId":"arc-prize:v2-semi-private:overall","benchmarkName":"ARC-AGI-2 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":23.47222222222222,"unit":"percent","family":"arc-prize:v2-semi-private:overall","index":49.96542713348843,"aboveFit":false,"run":"xhigh effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.6095804058333331,"latency":null},{"benchmarkId":"vals-ai:ioi:overall","benchmarkName":"IOI","category":"coding","domain":null,"eligible":true,"ranked":true,"score":22.833,"unit":"percent","family":"vals-ai:ioi:overall","index":54.68865385071791,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/ioi","measuredAt":"2026-10-29","stderr":8.012,"costPerTest":null,"latency":432.586},{"benchmarkId":"vals-ai:mysterymechanism:overall","benchmarkName":"MysteryMechanism","category":null,"domain":"Science","eligible":false,"ranked":true,"score":22.072,"unit":"percent","family":"vals-ai:mysterymechanism:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mysterymechanism","measuredAt":"2026-11-11","stderr":2.79,"costPerTest":null,"latency":144.311},{"benchmarkId":"vals-ai:proof_bench:overall","benchmarkName":"ProofBench v1.1","category":"math","domain":null,"eligible":true,"ranked":true,"score":18,"unit":"percent","family":"vals-ai:proof_bench:overall","index":53.37102520852528,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/proof_bench","measuredAt":"2026-11-09","stderr":3.861,"costPerTest":null,"latency":266.658},{"benchmarkId":"livebench:livebench-long-horizon:overall","benchmarkName":"LiveBench Long Agentic","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":17.3,"unit":"percent","family":"livebench-2-agentic","index":31.241499524494106,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Long%20Agentic","measuredAt":"2026-10-14","stderr":null,"costPerTest":9.888996551724137,"latency":null},{"benchmarkId":"vals-ai:tax_agent_bench:overall","benchmarkName":"Tax Agent Bench","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":8.808,"unit":"percent","family":"vals-ai:tax_agent_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench","measuredAt":"2026-11-09","stderr":1.902,"costPerTest":null,"latency":57.987},{"benchmarkId":"vals-ai:hlab:overall","benchmarkName":"HLAB-Vals-1.1","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":4.583,"unit":"percent","family":"vals-ai:hlab:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/hlab","measuredAt":"2026-11-09","stderr":1.775,"costPerTest":null,"latency":116.883},{"benchmarkId":"vals-ai:terminal-bench-4:overall","benchmarkName":"Terminal-Bench 4.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":4.04,"unit":"percent","family":"terminal-bench-4","index":56.43436153678791,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-4","measuredAt":"2026-11-09","stderr":1.336,"costPerTest":null,"latency":1154.552},{"benchmarkId":"vals-ai:terminal-bench-science:overall","benchmarkName":"Terminal-Bench-Science","category":null,"domain":"Science","eligible":false,"ranked":true,"score":1.429,"unit":"percent","family":"vals-ai:terminal-bench-science:overall","index":null,"aboveFit":false,"run":"Mini-SWE-agent","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-science","measuredAt":"2026-11-09","stderr":1.429,"costPerTest":null,"latency":4658.837}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":65,"benchmarkCount":9,"categoryCount":4,"confidence":"low","state":"available"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:livebench","name":"LiveBench","url":"https://livebench.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:arc-prize-foundation","name":"ARC Prize Foundation","url":"https://arcprize.org","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"reflection-beam-nvfp4","providerId":"reflection-ai","slug":"reflection-beam-nvfp4","name":"Beam 1.0 501B-A23B NVFP4","provider":"Reflection AI","description":"Beam 1.0 501B-A23B NVFP4 is a reasoning model from Reflection AI in the Beam family. 17 benchmarks count toward its index, in 6 categories.","logoUrl":null,"releaseDate":"2026-10-16","contextWindowTokens":1000000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":true,"overallScore":58.451985009853836,"overallScoreRange":{"low":53.03018395301633,"high":63.87378606669134},"badges":[],"capabilityScores":{"agentic":54.98250258955123,"coding":60.81224523527435,"reasoning":62.65707750772699,"multimodal":null,"knowledge":59.20692622426832,"multilingual":null,"instruction":63.03687008656155,"math":55.485177168490324},"capabilityCounts":{"math":1,"knowledge":2,"agentic":7,"coding":3,"instruction":1,"reasoning":3},"benchmarks":[{"benchmarkId":"aime2026","benchmarkName":"AIME 2026","category":"math","score":97.5,"unit":"percent","family":"aime2026","index":55.485177168490324,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gpqaDiamond","benchmarkName":"GPQA Diamond","category":"knowledge","score":89.3,"unit":"percent","family":"gpqa-diamond","index":59.11262060818633,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"deepSearchQa","benchmarkName":"DeepSearchQA","category":"agentic","score":81.1,"unit":"percent","family":"deepSearchQa","index":59.68754082140777,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"sweVerified","benchmarkName":"Software Engineering Benchmark Verified","category":"coding","score":80.7,"unit":"percent","family":"swe-bench-verified","index":60.74882405661559,"aboveFit":false,"run":null,"source":"Carlos E. Jimenez, John Yang, Alexander Wettig, Shunyu Yao, Kexin Pei, Ofir Press, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2310.06770","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mcpAtlas","benchmarkName":"MCP Atlas","category":"agentic","score":80.5,"unit":"percent","family":"mcpAtlas","index":65.27062879538744,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"ifBench","benchmarkName":"Instruction Following Benchmark","category":"instruction","score":79.2,"unit":"percent","family":"ifbench-family","index":63.03687008656155,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":78.7,"unit":"percent","family":"lcr","index":61.29258682276216,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"terminalBench21","benchmarkName":"Terminal-Bench 2.1","category":"agentic","score":78.4,"unit":"percent","family":"terminal-bench-2-1","index":66.99378178203646,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"browseComp","benchmarkName":"BrowseComp","category":"agentic","score":75.6,"unit":"percent","family":"browseComp","index":59.63449138414617,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"longBenchV2","benchmarkName":"LongBench v2","category":"reasoning","score":67.2,"unit":"percent","family":"longBenchV2","index":60.01692661048303,"aboveFit":false,"run":null,"source":"LongBench v2 authors","sourceUrl":"https://arxiv.org/abs/2412.15204","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"swePro","benchmarkName":"SWE-bench Pro","category":"coding","score":65.6,"unit":"percent","family":"swePro","index":64.7049984398452,"aboveFit":false,"run":null,"source":"Xiang Deng, Jeff Da, Edwin Pan, Yannis Yiming He, Charles Ide, Kanak Garg, Niklas Lauffer, Andrew Park, Nitin Pasari, Chetan Rane, Karmini Sampath, Maya Krishnan, Srivatsa Kundurthy, Sean Hendryx, Zifan Wang, Chen Bo Calvin Zhang, Noah Jacobson, Bing Liu, Brad Kenstler","sourceUrl":"https://arxiv.org/abs/2509.16941","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hleWithTools","benchmarkName":"Humanity's Last Exam with tools","category":"agentic","score":51.1,"unit":"percent","family":"hleWithTools","index":60.97877058293187,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"sciCode","benchmarkName":"Scientific Code Benchmark","category":"coding","score":46.9,"unit":"percent","family":"scicode-family","index":56.982913209362245,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"deepSwe","benchmarkName":"DeepSWE","category":"agentic","score":42.7,"unit":"percent","family":"deepSwe","index":52.118428619408014,"aboveFit":false,"run":null,"source":"Datacurve AI","sourceUrl":"https://deepswe.datacurve.ai/blog","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"tau3Bench","benchmarkName":"τ³-Bench Tool-Agent-User Evaluation","category":"agentic","score":35.4,"unit":"percent","family":"tau3Bench","index":20.193876141540887,"aboveFit":false,"run":null,"source":"Sierra Research","sourceUrl":"https://github.com/sierra-research/tau2-bench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hleNoTools","benchmarkName":"Humanity's Last Exam without tools","category":"knowledge","score":34.8,"unit":"percent","family":"hle","index":59.30123184035032,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"sweAtlasCodebaseQna","benchmarkName":"SWE-Atlas Codebase QnA","category":"coding","score":31.4,"unit":"percent","family":"sweAtlasCodebaseQna","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":15.4,"unit":"percent","family":"critpt","index":66.66171908993577,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":85,"benchmarkCount":17,"categoryCount":6,"confidence":"medium","state":"available"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"kimi-2-6","providerId":"moonshot-ai","slug":"kimi-2-6","name":"Kimi K2.6","provider":"Moonshot AI","description":"Kimi K2.6 is a reasoning model from Moonshot AI. 46 benchmarks count toward its index, in 7 categories.","logoUrl":null,"releaseDate":"2026-04-20","contextWindowTokens":262144,"modalities":["text","image"],"openWeights":true,"details":{"apiId":"moonshotai/kimi-k2.6","maxOutputTokens":235929,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.15,"weightsUrl":"https://huggingface.co/moonshotai/Kimi-K2.6"},"reasoning":true,"overallScore":58.203949501232856,"overallScoreRange":{"low":55.31981307020817,"high":61.08808593225754},"badges":[],"capabilityScores":{"agentic":59.456656004151114,"coding":59.30437135629875,"reasoning":53.771532523614766,"multimodal":57.556087425845654,"knowledge":56.07956706450874,"multilingual":null,"instruction":60.046436119107135,"math":59.955127765116785},"capabilityCounts":{"multimodal":4,"math":7,"agentic":16,"knowledge":5,"coding":7,"reasoning":6,"instruction":1},"benchmarks":[{"benchmarkId":"vStar","benchmarkName":"V*","category":"multimodal","score":96.9,"unit":"percent","family":"vStar","index":57.60734703162754,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aime2026","benchmarkName":"AIME 2026","category":"math","score":96.4,"unit":"percent","family":"aime2026","index":54.693242601054166,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":96.11111111111111,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":64.83596304980023,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.3777636174410794,"costPerTest":null,"latency":null},{"benchmarkId":"tau2Bench","benchmarkName":"τ²-Bench Tool-Agent-User Evaluation","category":"agentic","score":95.9,"unit":"percent","family":"tau2-bench","index":66.42610157862087,"aboveFit":false,"run":null,"source":"Victor Barres, Honghua Dong, Soham Ray, Xujie Si, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2506.07982","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hmmtFeb2026","benchmarkName":"Harvard-MIT Mathematics Tournament February 2026","category":"math","score":92.7,"unit":"percent","family":"hmmtFeb2026","index":58.27391441201564,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"deepSearchQa","benchmarkName":"DeepSearchQA","category":"agentic","score":92.5,"unit":"percent","family":"deepSearchQa","index":67.69243992802143,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":91.1,"unit":"percent","family":"gpqa-diamond","index":60.76353142304194,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":90.78282828282829,"unit":"percent","family":"gpqa-diamond","index":60.47263019079467,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.7174516286666601,"costPerTest":null,"latency":null},{"benchmarkId":"gpqa","benchmarkName":"Graduate-Level Google-Proof Q&A","category":"knowledge","score":90.5,"unit":"percent","family":"gpqa-diamond","index":60.213227818090076,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gpqaDiamond","benchmarkName":"GPQA Diamond","category":"knowledge","score":90.5,"unit":"percent","family":"gpqa-diamond","index":60.213227818090076,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-2025-02-28-public:overall","benchmarkName":"FrontierMath-2025-02-28-Public","category":"math","domain":null,"eligible":false,"ranked":true,"score":90,"unit":"percent","family":"epoch-ai:epoch-frontiermath-2025-02-28-public:overall","index":null,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":10,"costPerTest":null,"latency":null},{"benchmarkId":"liveCodeBenchV6","benchmarkName":"LiveCodeBench v6","category":"coding","score":89.6,"unit":"percent","family":"liveCodeBenchV6","index":57.983791485794406,"aboveFit":false,"run":null,"source":"LiveCodeBench maintainers","sourceUrl":"https://github.com/LiveCodeBench/LiveCodeBench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:gpqa:overall","benchmarkName":"GPQA Diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":89.142,"unit":"percent","family":"gpqa-diamond","index":58.967707325549,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","measuredAt":"2026-09-01","stderr":1.997,"costPerTest":0.080129,"latency":544.744},{"benchmarkId":"vals-ai:mmlu_pro:overall","benchmarkName":"MMLU Pro","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":87.572,"unit":"percent","family":"mmlu-pro","index":54.81096632799573,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","measuredAt":"2026-09-01","stderr":0.33,"costPerTest":0.018214,"latency":156.347},{"benchmarkId":"mathVision","benchmarkName":"MathVision","category":"multimodal","score":87.4,"unit":"percent","family":"mathVision","index":58.41257934649857,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:lcb:overall","benchmarkName":"LiveCodeBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":86.771,"unit":"percent","family":"livecodebench","index":61.341242685295455,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","measuredAt":"2026-09-01","stderr":0.972,"costPerTest":0.06543,"latency":529.063},{"benchmarkId":"vals-ai:mmmu:overall","benchmarkName":"MMMU Pro","category":"multimodal","domain":null,"eligible":true,"ranked":true,"score":86.301,"unit":"percent","family":"mmmu-pro","index":67.46599491988475,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmmu","measuredAt":"2026-09-01","stderr":0.827,"costPerTest":0.034867,"latency":241.908},{"benchmarkId":"mmAnswerBench","benchmarkName":"MMAnswerBench","category":"math","score":86,"unit":"percent","family":"mmAnswerBench","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:legal_bench:overall","benchmarkName":"LegalBench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":84.738,"unit":"percent","family":"vals-ai:legal_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","measuredAt":"2026-10-13","stderr":0.449,"costPerTest":0.0065,"latency":50.067},{"benchmarkId":"browseComp","benchmarkName":"BrowseComp","category":"agentic","score":83.2,"unit":"percent","family":"browseComp","index":65.63864190974103,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":81,"unit":"percent","family":"lcr","index":62.87057203160918,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"wideResearch","benchmarkName":"WideResearch","category":"agentic","score":80.8,"unit":"percent","family":"wideResearch","index":63.08654324314704,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"charxiv","benchmarkName":"CharXiv Reasoning","category":"multimodal","score":80.4,"unit":"percent","family":"charxiv","index":55.719474446959104,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"sweVerified","benchmarkName":"Software Engineering Benchmark Verified","category":"coding","score":80.2,"unit":"percent","family":"swe-bench-verified","index":60.380045267723744,"aboveFit":false,"run":null,"source":"Carlos E. Jimenez, John Yang, Alexander Wettig, Shunyu Yao, Kexin Pei, Ofir Press, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2310.06770","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmmuProPython","benchmarkName":"MMMU-Pro with Python","category":"multimodal","score":80.1,"unit":"percent","family":"mmmuProPython","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmmuPro","benchmarkName":"Massive Multi-discipline Multimodal Understanding Pro","category":"multimodal","score":79.4,"unit":"percent","family":"mmmu-pro","index":58.4849488782974,"aboveFit":false,"run":null,"source":"MMMU-Pro authors","sourceUrl":"https://arxiv.org/abs/2409.02813","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaMmmuPro","benchmarkName":"Artificial Analysis MMMU-Pro","category":"multimodal","score":79.4,"unit":"percent","family":"mmmu-pro","index":58.4849488782974,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmmu-pro","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:medscribe:overall","benchmarkName":"MedScribe","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":78.149,"unit":"percent","family":"vals-ai:medscribe:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medscribe","measuredAt":"2026-10-29","stderr":1.792,"costPerTest":0.055962,"latency":494.142},{"benchmarkId":"epoch-ai:epoch-swe-bench-verified:overall","benchmarkName":"SWE-Bench verified","category":"coding","domain":null,"eligible":true,"ranked":true,"score":76.65289256198346,"unit":"percent","family":"swe-bench-verified","index":57.763849297601745,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.924895909142592,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:swebench:overall","benchmarkName":"SWE-bench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":76.2,"unit":"percent","family":"swe-bench-verified","index":57.42981495658897,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/swebench","measuredAt":"2026-09-01","stderr":1.906,"costPerTest":0.27474,"latency":519.829},{"benchmarkId":"aaIfBench","benchmarkName":"Artificial Analysis IFBench","category":"instruction","score":76,"unit":"percent","family":"ifbench-family","index":60.046436119107135,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:tax_eval_v2:overall","benchmarkName":"TaxEval v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":74.652,"unit":"percent","family":"vals-ai:tax_eval_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","measuredAt":"2026-09-01","stderr":0.846,"costPerTest":0.029274,"latency":267.484},{"benchmarkId":"osWorldVerified","benchmarkName":"OSWorld-Verified","category":"agentic","score":73.1,"unit":"percent","family":"osWorldVerified","index":59.49150177162388,"aboveFit":false,"run":null,"source":"Tianbao Xie, Danyang Zhang, Jixuan Chen, Xiaochuan Li, Siheng Zhao, Ruisheng Cao, Toh Jing Hua, Zhoujun Cheng, Dongchan Shin, Fangyu Lei, Yitao Liu, Yiheng Xu, Shuyan Zhou, Silvio Savarese, Caiming Xiong, Victor Zhong, Tao Yu","sourceUrl":"https://os-world.github.io/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:corp_fin_v2:overall","benchmarkName":"CorpFin v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":66.744,"unit":"percent","family":"vals-ai:corp_fin_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","measuredAt":"2026-08-12","stderr":0.928,"costPerTest":0.07328,"latency":79.59},{"benchmarkId":"terminalBench2","benchmarkName":"Terminal-Bench 2.0","category":"agentic","score":66.7,"unit":"percent","family":"terminalBench2","index":61.34438315812766,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:mortgage_tax:overall","benchmarkName":"MortgageTax","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":65.818,"unit":"percent","family":"vals-ai:mortgage_tax:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mortgage_tax","measuredAt":"2026-09-01","stderr":0.93,"costPerTest":0.02869,"latency":109.76},{"benchmarkId":"clawEval","benchmarkName":"Claw-Eval","category":"agentic","score":62.3,"unit":"percent","family":"clawEval","index":55.20654122894529,"aboveFit":false,"run":null,"source":"Bowen Ye, Rang Li, Qibin Yang, Yuanxin Liu, Linli Yao, Hanglong Lv, Zhihui Xie, Chenxin An, Lei Li, Lingpeng Kong, Qi Liu, Zhifang Sui, Tong Yang","sourceUrl":"https://arxiv.org/abs/2604.06132","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:case_law_v2:overall","benchmarkName":"CaseLaw v2","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":61.201,"unit":"percent","family":"vals-ai:case_law_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/case_law_v2","measuredAt":"2026-05-04","stderr":2.087,"costPerTest":0.025569,"latency":46.087},{"benchmarkId":"swePro","benchmarkName":"SWE-bench Pro","category":"coding","score":58.6,"unit":"percent","family":"swePro","index":58.743728571529324,"aboveFit":false,"run":null,"source":"Xiang Deng, Jeff Da, Edwin Pan, Yannis Yiming He, Charles Ide, Kanak Garg, Niklas Lauffer, Andrew Park, Nitin Pasari, Chetan Rane, Karmini Sampath, Maya Krishnan, Srivatsa Kundurthy, Sean Hendryx, Zifan Wang, Chen Bo Calvin Zhang, Noah Jacobson, Bing Liu, Brad Kenstler","sourceUrl":"https://arxiv.org/abs/2509.16941","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:emb:overall","benchmarkName":"EMB","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":57.855,"unit":"percent","family":"vals-ai:emb:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/emb","measuredAt":"2026-10-20","stderr":2.739,"costPerTest":2.195977,"latency":2387.544},{"benchmarkId":"vals-ai:terminal-bench-2:overall","benchmarkName":"Terminal-Bench 2.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":57.303,"unit":"percent","family":"vals-ai:terminal-bench-2:overall","index":61.05404579598816,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2","measuredAt":"2026-06-04","stderr":5.273,"costPerTest":0.192786,"latency":795.458},{"benchmarkId":"epoch-ai:epoch-frontiermath-tiers-1-3-v2-private:overall","benchmarkName":"FrontierMath-Tiers-1-3-v2-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":57.19298245614035,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tiers-1-3-v2-private:overall","index":60.998320495389876,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.9360921879029935,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:finance_agent:overall","benchmarkName":"Finance Agent v1.1","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":57.056,"unit":"percent","family":"vals-ai:finance_agent:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/finance_agent","measuredAt":"2026-06-04","stderr":2.856,"costPerTest":0.492709,"latency":1505.876},{"benchmarkId":"gertLabs","benchmarkName":"Gert Labs Composite Game Benchmark","category":"agentic","score":56.82,"unit":"percent","family":"gertLabs","index":61.554864698440625,"aboveFit":false,"run":null,"source":"Gert Labs","sourceUrl":"https://gertlabs.com/rankings","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:public-benefits-bench:overall","benchmarkName":"Public Benefits Bench v1.1","category":null,"domain":"Public benefits","eligible":false,"ranked":true,"score":56.631,"unit":"percent","family":"vals-ai:public-benefits-bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/public-benefits-bench","measuredAt":"2026-10-12","stderr":1.289,"costPerTest":0.906347,"latency":595.296},{"benchmarkId":"mcpAtlas","benchmarkName":"MCP Atlas","category":"agentic","score":55.9,"unit":"percent","family":"mcpAtlas","index":48.7050971320216,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:terminal-bench-2-1:overall","benchmarkName":"Terminal-Bench 2.1","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":53.558,"unit":"percent","family":"terminal-bench-2-1","index":52.46480481779876,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2-1","measuredAt":"2026-10-01","stderr":2.085,"costPerTest":0.138449,"latency":932.358},{"benchmarkId":"vals-ai:public-benefits-bench-v1:overall","benchmarkName":"Public Benefits Bench v1","category":null,"domain":"Public benefits","eligible":false,"ranked":true,"score":53.383,"unit":"percent","family":"vals-ai:public-benefits-bench-v1:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/public-benefits-bench-v1","measuredAt":"2026-06-09","stderr":1.298,"costPerTest":0.611734,"latency":578.919},{"benchmarkId":"sciCode","benchmarkName":"Scientific Code Benchmark","category":"coding","score":52.2,"unit":"percent","family":"scicode-family","index":63.89247628536034,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaSciCode","benchmarkName":"Artificial Analysis SciCode","category":"coding","score":51.5,"unit":"percent","family":"scicode-family","index":62.97989248287002,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/scicode","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"toolathlon","benchmarkName":"Toolathlon","category":"agentic","score":50,"unit":"percent","family":"toolathlon","index":62.66965468193693,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"cursorBench31","benchmarkName":"cursorBench31","category":"coding","score":47.6,"unit":"percent","family":"cursorBench31","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:fabv2:overall","benchmarkName":"Finance Agent (v2)","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":44.899,"unit":"percent","family":"vals-ai:fabv2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/fabv2","measuredAt":"2026-10-29","stderr":0.733,"costPerTest":1.141587,"latency":1629.639},{"benchmarkId":"terminalBenchHard","benchmarkName":"Terminal-Bench Hard","category":"agentic","score":43.9,"unit":"percent","family":"terminalBenchHard","index":55.11285551926768,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/terminalbench-hard","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:medcode:overall","benchmarkName":"MedCode","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":40.142,"unit":"percent","family":"vals-ai:medcode:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medcode","measuredAt":"2026-10-29","stderr":2.041,"costPerTest":0.041295,"latency":305.421},{"benchmarkId":"epoch-ai:epoch-frontiermath-2025-02-28-private:overall","benchmarkName":"FrontierMath-2025-02-28-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":38.96551724137931,"unit":"percent","family":"epoch-ai:epoch-frontiermath-2025-02-28-private:overall","index":64.31206017790257,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.868659061400048,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:vibe-code:overall","benchmarkName":"Vibe Code Bench v1.1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":37.891,"unit":"percent","family":"vals-ai:vibe-code:overall","index":55.92119070321958,"aboveFit":false,"run":"OpenHands","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","measuredAt":"2026-11-09","stderr":4.912,"costPerTest":1.925351,"latency":2967.768},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":37.5,"unit":"percent","family":"hle","index":62.01012225193077,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-simpleqa-verified:overall","benchmarkName":"SimpleQA Verified","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":34.9,"unit":"percent","family":"epoch-ai:epoch-simpleqa-verified:overall","index":52.993833220603356,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.5080663991563052,"costPerTest":null,"latency":null},{"benchmarkId":"hle","benchmarkName":"Humanity's Last Exam","category":"knowledge","score":34.7,"unit":"percent","family":"hle","index":59.20090256584734,"aboveFit":false,"run":null,"source":"Center for AI Safety, Scale AI, and thousands of expert contributors","sourceUrl":"https://lastexam.ai/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"omniscienceAccuracy","benchmarkName":"Artificial Analysis Omniscience Accuracy","category":"knowledge","score":32.6,"unit":"percent","family":"omniscienceAccuracy","index":51.77429554696547,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"apexAgentsAa","benchmarkName":"APEX-Agents-AA","category":"agentic","score":28.5,"unit":"percent","family":"apexAgentsAa","index":61.661303452923505,"aboveFit":false,"run":null,"source":"Artificial Analysis / Mercor","sourceUrl":"https://artificialanalysis.ai/evaluations/apex-agents-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:code-migration:overall","benchmarkName":"Code Migration","category":"coding","domain":null,"eligible":true,"ranked":true,"score":27.768,"unit":"percent","family":"vals-ai:code-migration:overall","index":59.940612366535575,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/code-migration","measuredAt":"2026-10-29","stderr":4.144,"costPerTest":5.112053,"latency":5398.566},{"benchmarkId":"gdpvalAaNormalized","benchmarkName":"GDPval-AA normalized","category":"agentic","score":26.9,"unit":"percent","family":"gdpvalAaNormalized","index":53.266649806703064,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tier-4-v2-private:overall","benchmarkName":"FrontierMath-Tier-4-v2-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":25.64102564102564,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tier-4-v2-private:overall","index":58.56138241500658,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":7.083413480167724,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-furniture-assembly:overall","benchmarkName":"Furniture Assembly","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":21.666666666666668,"unit":"percent","family":"epoch-ai:epoch-furniture-assembly:overall","index":56.764021126231064,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":5.2301098424620545,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-chess-puzzles:overall","benchmarkName":"Chess Puzzles","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":20,"unit":"percent","family":"epoch-ai:epoch-chess-puzzles:overall","index":49.164979628140856,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":4.020151261036848,"costPerTest":null,"latency":null},{"benchmarkId":"researchClawBench","benchmarkName":"ResearchClawBench","category":"agentic","score":18,"unit":"percent","family":"researchClawBench","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:legal_research:overall","benchmarkName":"Legal Research Bench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":15.865,"unit":"percent","family":"vals-ai:legal_research:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_research","measuredAt":"2026-11-09","stderr":2.539,"costPerTest":1.520305,"latency":1853.133},{"benchmarkId":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-private:overall","benchmarkName":"FrontierMath-Tier-4-2025-07-01-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":14.580000000000002,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-private:overall","index":58.01101120464843,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":5.1499999999999995,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:tax_agent_bench:overall","benchmarkName":"Tax Agent Bench","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":12.404,"unit":"percent","family":"vals-ai:tax_agent_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench","measuredAt":"2026-11-09","stderr":2.376,"costPerTest":0.565511,"latency":924.425},{"benchmarkId":"epoch-ai:epoch-mystery-game-puzzles:overall","benchmarkName":"Mystery Game Puzzles","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":12,"unit":"percent","family":"epoch-ai:epoch-mystery-game-puzzles:overall","index":47.973976962532255,"aboveFit":false,"run":"none effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":3.2659863237109037,"costPerTest":null,"latency":null},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":8,"unit":"percent","family":"critpt","index":52.33618254636401,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"osWorld2","benchmarkName":"OSWorld 2.0","category":"agentic","score":4.6,"unit":"percent","family":"osWorld2","index":55.931067343110335,"aboveFit":false,"run":null,"source":"Mengqi Yuan, Zilong Zhou, Xinzhuang Xiong, Weiming Wu, Jiayang Sun, Jiamin Song, Kaiqian Cui, Bowen Wang, Haoyuan Wu, Yitong Li, Dunjie Lu, Haikong Lu, Qi Zhen, Xinyuan Wang, Jiaqi Deng, Yuhao Yang, Cheng Chen, Boyuan Zheng, Alex Su, Xiao Yu, Hao Zou, Saaket Agashe, Xing Han Lu, Manpreet Kaur, Zhengyang Qi, Vincent Sunn Chen, Frederic Sala, Dayiheng Liu, Junyang Lin, Zhou Yu, Yu Su, Siva Reddy, Xin Eric Wang, Peng Qi, Tianbao Xie, Tao Yu","sourceUrl":"https://arxiv.org/abs/2606.29537","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-ebr-bench:overall","benchmarkName":"EBR-bench","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":2.380952380952381,"unit":"percent","family":"epoch-ai:epoch-ebr-bench:overall","index":53.51946284681125,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:programbench:overall","benchmarkName":"ProgramBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":0,"unit":"percent","family":"vals-ai:programbench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/programbench","measuredAt":"2026-11-09","stderr":0,"costPerTest":null,"latency":4794.524}],"price":{"inputPerMillion":0.65,"outputPerMillion":3.41,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":81,"timeToFirstToken":1.0245,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":{"elo":1455.3543366334475,"eloUpper":1459.7730526389046,"eloLower":1450.9356206279904,"votes":39573,"rank":46,"rankUpper":null,"rankLower":null,"arenaName":"Kimi K2.6","organization":"moonshot","updatedAt":"2026-11-09"},"quality":{"coveragePercent":95,"benchmarkCount":46,"categoryCount":7,"confidence":"high","state":"available"},"hosts":[{"host":"chutes","name":"Chutes","kind":"host","url":"https://llm.chutes.ai/v1/models","inputPerMillion":0.39,"outputPerMillion":1.8,"cacheReadPerMillion":0.039,"contextTokens":262144,"maxOutputTokens":65535,"deprecated":false},{"host":"siliconflow","name":"SiliconFlow","kind":"host","url":"https://cloud.siliconflow.com/models","inputPerMillion":0.77,"outputPerMillion":3.4,"cacheReadPerMillion":0.14,"contextTokens":262000,"maxOutputTokens":262000,"deprecated":false},{"host":"novita-ai","name":"Novita AI","kind":"host","url":"https://novita.ai/docs/guides/introduction","inputPerMillion":0.8,"outputPerMillion":3.4,"cacheReadPerMillion":0.16,"contextTokens":262144,"maxOutputTokens":262144,"deprecated":false},{"host":"coreweave","name":"CoreWeave","kind":"host","url":"https://docs.wandb.ai/inference","inputPerMillion":0.65,"outputPerMillion":3.41,"cacheReadPerMillion":0.15,"contextTokens":262144,"maxOutputTokens":262144,"deprecated":false},{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":0.65,"outputPerMillion":3.41,"cacheReadPerMillion":0.15,"contextTokens":262144,"maxOutputTokens":235929,"deprecated":false},{"host":"crusoe","name":"Crusoe","kind":"host","url":"https://docs.crusoecloud.com/managed-inference/overview","inputPerMillion":0.7,"outputPerMillion":3.5,"cacheReadPerMillion":0.35,"contextTokens":262144,"maxOutputTokens":262144,"deprecated":true},{"host":"deepinfra","name":"Deep Infra","kind":"host","url":"https://deepinfra.com/models","inputPerMillion":0.75,"outputPerMillion":3.5,"cacheReadPerMillion":0.15,"contextTokens":262144,"maxOutputTokens":16384,"deprecated":false},{"host":"gmi-cloud","name":"GMI Cloud","kind":"host","url":"https://docs.gmicloud.ai/inference-engine/api-reference/llm-api-reference","inputPerMillion":0.855,"outputPerMillion":3.6,"cacheReadPerMillion":0.144,"contextTokens":65536,"maxOutputTokens":65536,"deprecated":false},{"host":"cloudflare","name":"Cloudflare Workers AI","kind":"cloud","url":"https://developers.cloudflare.com/workers-ai/models/","inputPerMillion":0.95,"outputPerMillion":4,"cacheReadPerMillion":0.16,"contextTokens":262144,"maxOutputTokens":256000,"deprecated":true},{"host":"digitalocean","name":"DigitalOcean","kind":"cloud","url":"https://docs.digitalocean.com/products/gradient-ai-platform/details/models/","inputPerMillion":0.95,"outputPerMillion":4,"cacheReadPerMillion":0.19,"contextTokens":262144,"maxOutputTokens":262144,"deprecated":false},{"host":"azure","name":"Microsoft Azure","kind":"cloud","url":"https://learn.microsoft.com/en-us/azure/ai-services/openai/concepts/models","inputPerMillion":0.95,"outputPerMillion":4,"cacheReadPerMillion":null,"contextTokens":262144,"maxOutputTokens":262144,"deprecated":false},{"host":"moonshot-ai","name":"Moonshot AI","kind":"maker","url":"https://platform.moonshot.ai/docs/api/chat","inputPerMillion":0.95,"outputPerMillion":4,"cacheReadPerMillion":0.16,"contextTokens":262144,"maxOutputTokens":262144,"deprecated":false},{"host":"privatemode","name":"Privatemode","kind":"host","url":"https://docs.privatemode.ai/api/overview","inputPerMillion":1.791,"outputPerMillion":8.9436,"cacheReadPerMillion":0.1733,"contextTokens":256000,"maxOutputTokens":262144,"deprecated":true}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"claude-opus-4-6","providerId":"anthropic","slug":"claude-opus-4-6","name":"Claude Opus 4.6","provider":"Anthropic","description":"Claude Opus 4.6 is a standard model from Anthropic. 42 benchmarks count toward its index, in 7 categories.","logoUrl":null,"releaseDate":"2026-02-01","contextWindowTokens":1000000,"modalities":["text","image","file"],"openWeights":false,"details":{"apiId":"anthropic/claude-opus-4.6","maxOutputTokens":128000,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.5,"weightsUrl":null},"reasoning":false,"overallScore":57.98263957705643,"overallScoreRange":{"low":54.9875544081694,"high":60.97772474594346},"badges":[],"capabilityScores":{"agentic":57.76879599123939,"coding":59.18579228787411,"reasoning":55.3069930273339,"multimodal":52.641361578174816,"knowledge":61.80189558028913,"multilingual":null,"instruction":30.702802813460732,"math":63.775066206487814},"capabilityCounts":{"knowledge":7,"math":5,"agentic":12,"coding":6,"multimodal":4,"reasoning":7,"instruction":1},"benchmarks":[{"benchmarkId":"aime2025Arcee","benchmarkName":"AIME25 first-party comparison snapshot","category":"math","score":99.8,"unit":"percent","family":"aime2025Arcee","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"superGpqa","benchmarkName":"SuperGPQA: Scaling LLM Evaluation Across 285 Graduate Disciplines","category":"knowledge","score":95,"unit":"percent","family":"superGpqa","index":74.64322501055656,"aboveFit":false,"run":null,"source":"Xiaoxuan Du, Yao Yao, Kexin Ma, Bowen Wang, Tianyu Zheng, Kaiyan Zhu, Yiming Zhang, Yutao Zhu, Jiawei Zhou, Jingren Zhou","sourceUrl":"https://arxiv.org/abs/2502.14739","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gpqa","benchmarkName":"Graduate-Level Google-Proof Q&A","category":"knowledge","score":91.3,"unit":"percent","family":"gpqa-diamond","index":60.9469659580259,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":91.11111111111111,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":61.9269928211348,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":4.290254662948546,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-2025-02-28-public:overall","benchmarkName":"FrontierMath-2025-02-28-Public","category":"math","domain":null,"eligible":false,"ranked":true,"score":90,"unit":"percent","family":"epoch-ai:epoch-frontiermath-2025-02-28-public:overall","index":null,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":10,"costPerTest":null,"latency":null},{"benchmarkId":"gpqaDiamond","benchmarkName":"GPQA Diamond","category":"knowledge","score":89.2,"unit":"percent","family":"gpqa-diamond","index":59.02090334069436,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmluProArcee","benchmarkName":"MMLU-Pro first-party comparison snapshot","category":"knowledge","score":89.1,"unit":"percent","family":"mmlu-pro","index":56.73606343109718,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":88.38383838383838,"unit":"percent","family":"gpqa-diamond","index":58.27234220803255,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.282888177524936,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:medscribe:overall","benchmarkName":"MedScribe","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":86.738,"unit":"percent","family":"vals-ai:medscribe:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medscribe","measuredAt":"2026-10-29","stderr":1.942,"costPerTest":0.115121,"latency":54.318},{"benchmarkId":"tau2Bench","benchmarkName":"τ²-Bench Tool-Agent-User Evaluation","category":"agentic","score":84.8,"unit":"percent","family":"tau2-bench","index":59.06044003629105,"aboveFit":false,"run":null,"source":"Victor Barres, Honghua Dong, Soham Ray, Xujie Si, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2506.07982","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"reactNativeEvals","benchmarkName":"React Native Evals","category":"coding","score":84.1,"unit":"percent","family":"reactNativeEvals","index":61.210972508795216,"aboveFit":false,"run":null,"source":"Callstack","sourceUrl":"https://rn-evals.vercel.app/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":84,"unit":"percent","family":"gpqa-diamond","index":54.25160543111147,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"browseComp","benchmarkName":"BrowseComp","category":"agentic","score":83.7,"unit":"percent","family":"browseComp","index":66.03365181274069,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"screenSpotPro","benchmarkName":"ScreenSpot Pro","category":"multimodal","score":83.1,"unit":"percent","family":"screenSpotPro","index":63.20278556129974,"aboveFit":false,"run":null,"source":"Kaixin Li, Ziyang Meng, Hongzhan Lin, Ziyang Luo, Yuchen Tian, Jing Ma, Zhiyong Huang, Tat-Seng Chua","sourceUrl":"https://arxiv.org/abs/2504.07981","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmluPro","benchmarkName":"Massive Multitask Language Understanding Professional","category":"knowledge","score":82,"unit":"percent","family":"mmlu-pro","index":47.79091327925139,"aboveFit":false,"run":null,"source":"Yubo Wang, Xueguang Ma, Ge Zhang, Yuansheng Ni, Abhranil Chandra, Shiguang Guo, Weiming Ren, Aaran Arulraj, Xuan He, Ziyan Jiang, Tianle Li, Max Ku, Kai Wang, Alex Zhuang, Rongqi Fan, Xiang Yue, Wenhu Chen","sourceUrl":"https://arxiv.org/abs/2406.01574","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"sweVerified","benchmarkName":"Software Engineering Benchmark Verified","category":"coding","score":80.84,"unit":"percent","family":"swe-bench-verified","index":60.852082117505304,"aboveFit":false,"run":null,"source":"Carlos E. Jimenez, John Yang, Alexander Wettig, Shunyu Yao, Kexin Pei, Ofir Press, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2310.06770","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmmuPro","benchmarkName":"Massive Multi-discipline Multimodal Understanding Pro","category":"multimodal","score":77.3,"unit":"percent","family":"mmmu-pro","index":55.75198312154714,"aboveFit":false,"run":null,"source":"MMMU-Pro authors","sourceUrl":"https://arxiv.org/abs/2409.02813","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-swe-bench-verified:overall","benchmarkName":"SWE-Bench verified","category":"coding","domain":null,"eligible":true,"ranked":true,"score":77.1694214876033,"unit":"percent","family":"swe-bench-verified","index":58.14481912083713,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.9537213360116366,"costPerTest":null,"latency":null},{"benchmarkId":"sweVerifiedArcee","benchmarkName":"SWE-bench Verified (mini-swe-agent-v2)","category":"coding","score":75.6,"unit":"percent","family":"swe-bench-verified","index":56.98728040991875,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"swebench-experiments:swe-bench-verified:overall","benchmarkName":"SWE-bench Verified","category":"coding","domain":null,"eligible":true,"ranked":true,"score":75.6,"unit":"percent","family":"swe-bench-verified","index":56.98728040991875,"aboveFit":false,"run":"mini-SWE-agent","source":"SWE-bench team","sourceUrl":"https://github.com/SWE-bench/experiments/tree/main/evaluation/verified","measuredAt":"2026-09-01","stderr":null,"costPerTest":0.5515226445,"latency":null},{"benchmarkId":"deepSearchQa","benchmarkName":"DeepSearchQA","category":"agentic","score":73.7,"unit":"percent","family":"deepSearchQa","index":54.491378243430496,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"osWorldVerified","benchmarkName":"OSWorld-Verified","category":"agentic","score":72.7,"unit":"percent","family":"osWorldVerified","index":59.16859894775328,"aboveFit":false,"run":null,"source":"Tianbao Xie, Danyang Zhang, Jixuan Chen, Xiaochuan Li, Siheng Zhao, Ruisheng Cao, Toh Jing Hua, Zhoujun Cheng, Dongchan Shin, Fangyu Lei, Yitao Liu, Yiheng Xu, Shuyan Zhou, Silvio Savarese, Caiming Xiong, Victor Zhong, Tao Yu","sourceUrl":"https://os-world.github.io/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaMmmuPro","benchmarkName":"Artificial Analysis MMMU-Pro","category":"multimodal","score":72.5,"unit":"percent","family":"mmmu-pro","index":49.505204248975154,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmmu-pro","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"swebench-experiments:swe-bench-multilingual:overall","benchmarkName":"SWE-bench Multilingual","category":"coding","domain":null,"eligible":true,"ranked":true,"score":72,"unit":"percent","family":"swe-bench-multilingual","index":null,"aboveFit":false,"run":"mini-SWE-agent","source":"SWE-bench team","sourceUrl":"https://github.com/SWE-bench/experiments/tree/main/evaluation/multilingual","measuredAt":"2026-09-02","stderr":null,"costPerTest":0.6629735766666667,"latency":null},{"benchmarkId":"liveCodeBenchPro","benchmarkName":"LiveCodeBench Pro","category":"coding","score":70.7,"unit":"percent","family":"liveCodeBenchPro","index":null,"aboveFit":false,"run":null,"source":"LiveCodeBench Pro authors","sourceUrl":"https://arxiv.org/abs/2506.11928","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"clawEval","benchmarkName":"Claw-Eval","category":"agentic","score":70.4,"unit":"percent","family":"clawEval","index":66.92819123724637,"aboveFit":false,"run":null,"source":"Bowen Ye, Rang Li, Qibin Yang, Yuanxin Liu, Linli Yao, Hanglong Lv, Zhihui Xie, Chenxin An, Lei Li, Lingpeng Kong, Qi Liu, Zhifang Sui, Tong Yang","sourceUrl":"https://arxiv.org/abs/2604.06132","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":67,"unit":"percent","family":"lcr","index":53.26544467340994,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"cyberGym","benchmarkName":"CyberGym","category":"agentic","score":66.6,"unit":"percent","family":"cyberGym","index":58.42219740279491,"aboveFit":false,"run":null,"source":"Zhun Wang, Tianneng Shi, Jingxuan He, Matthew Cai, Jialin Zhang, Dawn Song","sourceUrl":"https://www.cybergym.io/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tiers-1-3-v2-private:overall","benchmarkName":"FrontierMath-Tiers-1-3-v2-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":65.96491228070175,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tiers-1-3-v2-private:overall","index":65.61544076994005,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.8116467881852296,"costPerTest":null,"latency":null},{"benchmarkId":"terminalBench2","benchmarkName":"Terminal-Bench 2.0","category":"agentic","score":65.4,"unit":"percent","family":"terminalBench2","index":60.41963400764559,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"sweRebench","benchmarkName":"SWE-Rebench","category":"coding","score":65.3,"unit":"percent","family":"sweRebench","index":null,"aboveFit":false,"run":null,"source":"Nebius","sourceUrl":"https://swe-rebench.com","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"medXpertQaMm","benchmarkName":"MedXpertQA Multimodal","category":"multimodal","score":64.8,"unit":"percent","family":"medXpertQaMm","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"androidWorld","benchmarkName":"AndroidWorld","category":"agentic","score":62,"unit":"percent","family":"androidWorld","index":45.71726723642914,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gertLabs","benchmarkName":"Gert Labs Composite Game Benchmark","category":"agentic","score":61.85,"unit":"percent","family":"gertLabs","index":65.62774013429453,"aboveFit":false,"run":null,"source":"Gert Labs","sourceUrl":"https://gertlabs.com/rankings","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:vibe-code:overall","benchmarkName":"Vibe Code Bench v1.1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":57.573,"unit":"percent","family":"vals-ai:vibe-code:overall","index":63.022765530631474,"aboveFit":false,"run":"OpenHands","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","measuredAt":"2026-11-09","stderr":4.371,"costPerTest":8.687436,"latency":1278.658},{"benchmarkId":"parseBenchLlamaIndex","benchmarkName":"ParseBench: document parsing for AI agents (LlamaIndex leaderboard)","category":"multimodal","score":53.8,"unit":"percent","family":"parseBenchLlamaIndex","index":48.7821339028539,"aboveFit":false,"run":null,"source":"Boyang Zhang, Sebastián G. Acosta, Preston Carlson, Sacha Bron, Pierre-Loïc Doulcet, Daniel B. Ospina and Simon Suo (LlamaIndex)","sourceUrl":"https://arxiv.org/abs/2604.08538","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"swePro","benchmarkName":"SWE-bench Pro","category":"coding","score":53.4,"unit":"percent","family":"swePro","index":54.315356669351814,"aboveFit":false,"run":null,"source":"Xiang Deng, Jeff Da, Edwin Pan, Yannis Yiming He, Charles Ide, Kanak Garg, Niklas Lauffer, Andrew Park, Nitin Pasari, Chetan Rane, Karmini Sampath, Maya Krishnan, Srivatsa Kundurthy, Sean Hendryx, Zifan Wang, Chen Bo Calvin Zhang, Noah Jacobson, Bing Liu, Brad Kenstler","sourceUrl":"https://arxiv.org/abs/2509.16941","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hle","benchmarkName":"Humanity's Last Exam","category":"knowledge","score":53,"unit":"percent","family":"hle","index":77.56115979989258,"aboveFit":true,"run":null,"source":"Center for AI Safety, Scale AI, and thousands of expert contributors","sourceUrl":"https://lastexam.ai/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"medXpertQaText","benchmarkName":"MedXpertQA Text","category":"knowledge","score":52.1,"unit":"percent","family":"medXpertQaText","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"erqa","benchmarkName":"ERQA","category":"multimodal","score":51.6,"unit":"percent","family":"erqa","index":45.95193316328448,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"terminalBenchHard","benchmarkName":"Terminal-Bench Hard","category":"agentic","score":48.5,"unit":"percent","family":"terminalBenchHard","index":58.891889563903774,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/terminalbench-hard","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:medcode:overall","benchmarkName":"MedCode","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":48.244,"unit":"percent","family":"vals-ai:medcode:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medcode","measuredAt":"2026-10-29","stderr":2.05,"costPerTest":0.00618,"latency":4.058},{"benchmarkId":"epoch-ai:epoch-simpleqa-verified:overall","benchmarkName":"SimpleQA Verified","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":47,"unit":"percent","family":"epoch-ai:epoch-simpleqa-verified:overall","index":63.0742013883153,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.5790799515836722,"costPerTest":null,"latency":null},{"benchmarkId":"omniscienceAccuracy","benchmarkName":"Artificial Analysis Omniscience Accuracy","category":"knowledge","score":45.8,"unit":"percent","family":"omniscienceAccuracy","index":68.40967453655135,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaIfBench","benchmarkName":"Artificial Analysis IFBench","category":"instruction","score":44.6,"unit":"percent","family":"ifbench-family","index":30.702802813460732,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-2025-02-28-private:overall","benchmarkName":"FrontierMath-2025-02-28-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":40.699999999999996,"unit":"percent","family":"epoch-ai:epoch-frontiermath-2025-02-28-private:overall","index":65.84322275055155,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.9000000000000004,"costPerTest":null,"latency":null},{"benchmarkId":"hleNoTools","benchmarkName":"Humanity's Last Exam without tools","category":"knowledge","score":40,"unit":"percent","family":"hle","index":64.51835411450526,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"jobBench","benchmarkName":"JobBench","category":"agentic","score":36.7,"unit":"percent","family":"jobBench","index":56.79233006474054,"aboveFit":false,"run":null,"source":"Yuetai Li, Yichen Feng, Zhangchen Xu, Zixian Ma, Kaiyuan Zheng, Fengqing Jiang, Xinghua Sun, Rulin Shao, Zichen Chen, Yue Huang, Xinyang Han, Brian Lee, Kayla Xu, Shenglai Zeng, Hang Hua, Xiangliang Zhang, Basel Alomair, Ranjay Krishna, Luke Zettlemoyer, Pang Wei Koh, Bhaskar Ramasubramanian, Luyao Niu, Xiang Yue, Radha Poovendran","sourceUrl":"https://arxiv.org/abs/2605.26329","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-mirrorcode:overall","benchmarkName":"MirrorCode","category":"coding","domain":null,"eligible":true,"ranked":true,"score":33.33333333333333,"unit":"percent","family":"epoch-ai:epoch-mirrorcode:overall","index":64.20468425399986,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":10.026420125303973,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-furniture-assembly:overall","benchmarkName":"Furniture Assembly","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":28.333333333333332,"unit":"percent","family":"epoch-ai:epoch-furniture-assembly:overall","index":59.87186229150464,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":5.866531842313124,"costPerTest":null,"latency":null},{"benchmarkId":"tau2-bench:banking-knowledge:overall","benchmarkName":"τ²-bench Banking","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":27.32,"unit":"percent","family":"tau2-bench","index":20.918257563037148,"aboveFit":false,"run":"max effort · Sierra","source":"Sierra Research","sourceUrl":"https://github.com/sierra-research/tau2-bench","measuredAt":"2026-10-20","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"frontierCode","benchmarkName":"FrontierCode 1.1 Main","category":"coding","score":26.9,"unit":"percent","family":"frontierCode","index":54.79492499908836,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tier-4-v2-private:overall","benchmarkName":"FrontierMath-Tier-4-v2-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":26.82926829268293,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tier-4-v2-private:overall","index":59.02818358680503,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":7.005564203095157,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-mystery-game-puzzles:overall","benchmarkName":"Mystery Game Puzzles","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":25,"unit":"percent","family":"epoch-ai:epoch-mystery-game-puzzles:overall","index":58.041370619699485,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":4.351941398892446,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-private:overall","benchmarkName":"FrontierMath-Tier-4-2025-07-01-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":22.900000000000002,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-private:overall","index":66.46149110400762,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":6.1,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-chess-puzzles:overall","benchmarkName":"Chess Puzzles","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":21,"unit":"percent","family":"epoch-ai:epoch-chess-puzzles:overall","index":49.923994823290954,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":4.093601807403323,"costPerTest":null,"latency":null},{"benchmarkId":"researchClawBench","benchmarkName":"ResearchClawBench","category":"agentic","score":19.9,"unit":"percent","family":"researchClawBench","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":19.1,"unit":"percent","family":"hle","index":43.54953574338255,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"healthBenchHard","benchmarkName":"HealthBench Hard","category":"knowledge","score":14.8,"unit":"percent","family":"healthBenchHard","index":51.0577028825577,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-ebr-bench:overall","benchmarkName":"EBR-bench","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":12.698400000000001,"unit":"percent","family":"epoch-ai:epoch-ebr-bench:overall","index":58.254422105417895,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"apprenticeBench","benchmarkName":"ApprenticeBench: end-to-end computer use, continual learning, and long-horizon agency on a real accounts-payable job","category":"agentic","score":5,"unit":"percent","family":"apprenticeBench","index":60.74332444422925,"aboveFit":false,"run":null,"source":"NeoCognition","sourceUrl":"https://neocognition.io/blog/apprentice-bench/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":2.8,"unit":"percent","family":"critpt","index":42.269589299529784,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"arc-prize:v3-semi-private:overall","benchmarkName":"ARC-AGI-3 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":0.51,"unit":"percent","family":"arc-prize:v3-semi-private:overall","index":65.52226737848463,"aboveFit":false,"run":"max effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-public:overall","benchmarkName":"FrontierMath-Tier-4-2025-07-01-Public","category":"math","domain":null,"eligible":false,"ranked":true,"score":0,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-public:overall","index":null,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":5,"outputPerMillion":25,"batch":{"inputPerMillion":2.5,"outputPerMillion":12.5},"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":36,"timeToFirstToken":1.12,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":{"elo":1496.747935344304,"eloUpper":1499.9138066481992,"eloLower":1493.5820640404095,"votes":89228,"rank":10,"rankUpper":null,"rankLower":null,"arenaName":"Claude Opus 4.6","organization":"anthropic","updatedAt":"2026-11-09"},"quality":{"coveragePercent":95,"benchmarkCount":42,"categoryCount":7,"confidence":"high","state":"available"},"hosts":[{"host":"amazon-bedrock","name":"Amazon Bedrock","kind":"cloud","url":"https://docs.aws.amazon.com/bedrock/latest/userguide/models-supported.html","inputPerMillion":5,"outputPerMillion":25,"cacheReadPerMillion":0.5,"contextTokens":1000000,"maxOutputTokens":128000,"deprecated":false},{"host":"databricks","name":"Databricks","kind":"cloud","url":"https://docs.databricks.com/aws/en/machine-learning/foundation-models/","inputPerMillion":5,"outputPerMillion":25,"cacheReadPerMillion":0.5,"contextTokens":1000000,"maxOutputTokens":128000,"deprecated":false},{"host":"digitalocean","name":"DigitalOcean","kind":"cloud","url":"https://docs.digitalocean.com/products/gradient-ai-platform/details/models/","inputPerMillion":5,"outputPerMillion":25,"cacheReadPerMillion":0.5,"contextTokens":200000,"maxOutputTokens":8192,"deprecated":false},{"host":"gmi-cloud","name":"GMI Cloud","kind":"host","url":"https://docs.gmicloud.ai/inference-engine/api-reference/llm-api-reference","inputPerMillion":5,"outputPerMillion":25,"cacheReadPerMillion":0.5,"contextTokens":409600,"maxOutputTokens":128000,"deprecated":false},{"host":"google-vertex","name":"Google Vertex AI","kind":"cloud","url":"https://cloud.google.com/vertex-ai/generative-ai/docs/models","inputPerMillion":5,"outputPerMillion":25,"cacheReadPerMillion":0.5,"contextTokens":1000000,"maxOutputTokens":128000,"deprecated":false},{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":5,"outputPerMillion":25,"cacheReadPerMillion":0.5,"contextTokens":1000000,"maxOutputTokens":128000,"deprecated":false},{"host":"sap-ai-core","name":"SAP AI Core","kind":"cloud","url":"https://help.sap.com/docs/sap-ai-core","inputPerMillion":5,"outputPerMillion":25,"cacheReadPerMillion":0.5,"contextTokens":1000000,"maxOutputTokens":128000,"deprecated":false},{"host":"snowflake","name":"Snowflake Cortex","kind":"cloud","url":"https://docs.snowflake.com/en/user-guide/snowflake-cortex/cortex-rest-api","inputPerMillion":5,"outputPerMillion":25,"cacheReadPerMillion":0.5,"contextTokens":1000000,"maxOutputTokens":128000,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:swe-bench-team","name":"SWE-bench team","url":"https://github.com","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:sierra-research","name":"Sierra Research","url":"https://github.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:arc-prize-foundation","name":"ARC Prize Foundation","url":"https://arcprize.org","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gemini-3-pro","providerId":"google","slug":"gemini-3-pro","name":"Gemini 3 Pro","provider":"Google","description":"Gemini 3 Pro is a standard model from Google. 35 benchmarks count toward its index, in 8 categories.","logoUrl":null,"releaseDate":"2025-11-18","contextWindowTokens":2000000,"modalities":["text"],"openWeights":false,"details":null,"reasoning":false,"overallScore":57.92227608373287,"overallScoreRange":{"low":55.18904722394085,"high":60.65550494352489},"badges":[],"capabilityScores":{"agentic":56.15178125804805,"coding":56.39865426509829,"reasoning":57.211396532823734,"multimodal":54.84515731078812,"knowledge":66.0801183485513,"multilingual":60.10791351971327,"instruction":54.81317667606192,"math":58.23920948236731},"capabilityCounts":{"multimodal":11,"math":5,"knowledge":4,"coding":4,"agentic":5,"reasoning":4,"instruction":1,"multilingual":1},"benchmarks":[{"benchmarkId":"countBench","benchmarkName":"CountBench","category":"multimodal","score":97.3,"unit":"percent","family":"countBench","index":53.37602742308809,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:aime:overall","benchmarkName":"AIME","category":"math","domain":null,"eligible":true,"ranked":true,"score":96.68,"unit":"percent","family":"vals-ai:aime:overall","index":57.75615010902649,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/aime","measuredAt":"2026-04-16","stderr":0.384,"costPerTest":0.169306,"latency":109.264},{"benchmarkId":"vals-ai:math500:overall","benchmarkName":"MATH 500","category":"math","domain":null,"eligible":true,"ranked":true,"score":96.4,"unit":"percent","family":"math-500","index":45.998274934617726,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/math500","measuredAt":"2026-01-09","stderr":0.833,"costPerTest":0.050514,"latency":32.146},{"benchmarkId":"vals-ai:medqa:overall","benchmarkName":"MedQA","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":96.033,"unit":"percent","family":"vals-ai:medqa:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medqa","measuredAt":"2026-04-16","stderr":0.178,"costPerTest":0.029109,"latency":21.603},{"benchmarkId":"aaAime2025","benchmarkName":"Artificial Analysis AIME 2025","category":"math","score":95.7,"unit":"percent","family":"aaAime2025","index":null,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/aime-2025","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"ai2dTest","benchmarkName":"AI2D test split","category":"multimodal","score":94.1,"unit":"percent","family":"ai2dTest","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:mgsm:overall","benchmarkName":"MGSM","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":93.927,"unit":"percent","family":"vals-ai:mgsm:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mgsm","measuredAt":"2026-01-09","stderr":1.413,"costPerTest":0.018166,"latency":12.9},{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":92.61363636363636,"unit":"percent","family":"gpqa-diamond","index":62.15179733553416,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.6526963574741,"costPerTest":null,"latency":null},{"benchmarkId":"aaGlobalMmluLite","benchmarkName":"Artificial Analysis Global-MMLU-Lite","category":"multilingual","score":92.2,"unit":"percent","family":"aaGlobalMmluLite","index":null,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/global-mmlu-lite","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaLiveCodeBench","benchmarkName":"Artificial Analysis LiveCodeBench","category":"coding","score":91.7,"unit":"percent","family":"livecodebench","index":65.43103720304997,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/livecodebench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:gpqa:overall","benchmarkName":"GPQA Diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":91.666,"unit":"percent","family":"gpqa-diamond","index":61.282651157046544,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","measuredAt":"2026-09-01","stderr":1.389,"costPerTest":0.089573,"latency":61.34},{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":91.38888888888889,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":62.08860227828288,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":3.658872186755414,"costPerTest":null,"latency":null},{"benchmarkId":"tau2-bench:telecom:overall","benchmarkName":"τ²-bench Telecom","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":91.01,"unit":"percent","family":"tau2-bench","index":63.181229061324224,"aboveFit":false,"run":"high effort · Sierra","source":"Sierra Research","sourceUrl":"https://github.com/sierra-research/tau2-bench","measuredAt":"2026-03-02","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":90.8,"unit":"percent","family":"gpqa-diamond","index":60.48837962056601,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:mmlu_pro:overall","benchmarkName":"MMLU Pro","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":90.102,"unit":"percent","family":"mmlu-pro","index":57.99846349478022,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","measuredAt":"2026-09-01","stderr":0.295,"costPerTest":0.036926,"latency":24.272},{"benchmarkId":"aaMmluPro","benchmarkName":"Artificial Analysis MMLU-Pro","category":"knowledge","score":89.8,"unit":"percent","family":"mmlu-pro","index":57.61797964325099,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmlu-pro","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"omniDocBench15","benchmarkName":"OmniDocBench 1.5","category":"multimodal","score":88.5,"unit":"percent","family":"omniDocBench15","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"videoMmeWithSub","benchmarkName":"Video-MME with subtitle","category":"multimodal","score":88.4,"unit":"percent","family":"videoMmeWithSub","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vStar","benchmarkName":"V*","category":"multimodal","score":88,"unit":"percent","family":"vStar","index":47.05312624157424,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"videoMmeNoSub","benchmarkName":"Video-MME without subtitle","category":"multimodal","score":87.7,"unit":"percent","family":"videoMmeNoSub","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"videoMmmu","benchmarkName":"VideoMMMU","category":"multimodal","score":87.6,"unit":"percent","family":"videoMmmu","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:mmmu:overall","benchmarkName":"MMMU Pro","category":"multimodal","domain":null,"eligible":true,"ranked":true,"score":87.514,"unit":"percent","family":"mmmu-pro","index":69.04460799747429,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmmu","measuredAt":"2026-09-01","stderr":0.795,"costPerTest":0.010159,"latency":48.102},{"benchmarkId":"mmmu","benchmarkName":"Massive Multi-discipline Multimodal Understanding","category":"multimodal","score":87.2,"unit":"percent","family":"mmmu","index":56.50057738306743,"aboveFit":false,"run":null,"source":"MMMU authors","sourceUrl":"https://arxiv.org/abs/2401.05508","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"tau2Bench","benchmarkName":"τ²-Bench Tool-Agent-User Evaluation","category":"agentic","score":87.1,"unit":"percent","family":"tau2-bench","index":60.58665819371074,"aboveFit":false,"run":null,"source":"Victor Barres, Honghua Dong, Soham Ray, Xujie Si, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2506.07982","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:legal_bench:overall","benchmarkName":"LegalBench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":87.025,"unit":"percent","family":"vals-ai:legal_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","measuredAt":"2026-10-13","stderr":0.368,"costPerTest":0.01046,"latency":8.327},{"benchmarkId":"weMath","benchmarkName":"We-Math","category":"multimodal","score":86.9,"unit":"percent","family":"weMath","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mathVision","benchmarkName":"MathVision","category":"multimodal","score":86.6,"unit":"percent","family":"mathVision","index":57.6504428268827,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:lcb:overall","benchmarkName":"LiveCodeBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":86.407,"unit":"percent","family":"livecodebench","index":61.03921687793846,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","measuredAt":"2026-09-01","stderr":0.982,"costPerTest":0.12729,"latency":87.226},{"benchmarkId":"dynaMath","benchmarkName":"DynaMath","category":"multimodal","score":85.1,"unit":"percent","family":"dynaMath","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"refcocoAvg","benchmarkName":"RefCOCO average","category":"multimodal","score":84.1,"unit":"percent","family":"refcocoAvg","index":34.15101100647326,"aboveFit":false,"run":null,"source":"RefCOCO dataset authors","sourceUrl":"https://github.com/lichengunc/refer","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"realWorldQa","benchmarkName":"RealWorldQA","category":"multimodal","score":83.3,"unit":"percent","family":"realWorldQa","index":56.1890694271189,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mlvuAvg","benchmarkName":"MLVU mean average","category":"multimodal","score":83,"unit":"percent","family":"mlvuAvg","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"charxiv","benchmarkName":"CharXiv Reasoning","category":"multimodal","score":81.4,"unit":"percent","family":"charxiv","index":56.85797287252592,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmmuPro","benchmarkName":"Massive Multi-discipline Multimodal Understanding Pro","category":"multimodal","score":81,"unit":"percent","family":"mmmu-pro","index":60.56720850248806,"aboveFit":false,"run":null,"source":"MMMU-Pro authors","sourceUrl":"https://arxiv.org/abs/2409.02813","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"tau2-bench:airline:overall","benchmarkName":"τ²-bench Airline","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":80.5,"unit":"percent","family":"tau2-bench","index":56.20707565502815,"aboveFit":false,"run":"high effort · Sierra","source":"Sierra Research","sourceUrl":"https://github.com/sierra-research/tau2-bench","measuredAt":"2026-03-02","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaMmmuPro","benchmarkName":"Artificial Analysis MMMU-Pro","category":"multimodal","score":80.2,"unit":"percent","family":"mmmu-pro","index":59.52607869039272,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmmu-pro","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-2025-02-28-public:overall","benchmarkName":"FrontierMath-2025-02-28-Public","category":"math","domain":null,"eligible":false,"ranked":true,"score":80,"unit":"percent","family":"epoch-ai:epoch-frontiermath-2025-02-28-public:overall","index":null,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":13.333333333333334,"costPerTest":null,"latency":null},{"benchmarkId":"ccOcr","benchmarkName":"CC-OCR","category":"multimodal","score":79,"unit":"percent","family":"ccOcr","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:swebench:overall","benchmarkName":"SWE-bench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":76.4,"unit":"percent","family":"swe-bench-verified","index":57.577326472145714,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/swebench","measuredAt":"2026-09-01","stderr":1.901,"costPerTest":0.771547,"latency":394.488},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":76,"unit":"percent","family":"lcr","index":59.44016940368088,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"tau2-bench:retail:overall","benchmarkName":"τ²-bench Retail","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":75.88,"unit":"percent","family":"tau2-bench","index":53.14136787795032,"aboveFit":false,"run":"high effort · Sierra","source":"Sierra Research","sourceUrl":"https://github.com/sierra-research/tau2-bench","measuredAt":"2026-04-30","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"arc-prize:v1-semi-private:overall","benchmarkName":"ARC-AGI-1 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":75,"unit":"percent","family":"arc-prize:v1-semi-private:overall","index":61.467566583270525,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.4926,"latency":null},{"benchmarkId":"swe-bench:swe-bench-verified:overall","benchmarkName":"SWE-bench Verified","category":"coding","domain":null,"eligible":true,"ranked":true,"score":74.2,"unit":"percent","family":"swe-bench-verified","index":55.95469980102159,"aboveFit":false,"run":"mini-SWE-agent","source":"SWE-bench team","sourceUrl":"https://www.swebench.com/","measuredAt":"2026-02-26","stderr":null,"costPerTest":0.45995751895000003,"latency":null},{"benchmarkId":"simpleVqa","benchmarkName":"SimpleVQA","category":"multimodal","score":73.2,"unit":"percent","family":"simpleVqa","index":63.37607399036242,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-swe-bench-verified:overall","benchmarkName":"SWE-Bench verified","category":"coding","domain":null,"eligible":true,"ranked":true,"score":72.93388429752066,"unit":"percent","family":"swe-bench-verified","index":55.020866570307035,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.021641171153417,"costPerTest":null,"latency":null},{"benchmarkId":"screenSpotPro","benchmarkName":"ScreenSpot Pro","category":"multimodal","score":72.7,"unit":"percent","family":"screenSpotPro","index":54.126525261617196,"aboveFit":false,"run":null,"source":"Kaixin Li, Ziyang Meng, Hongzhan Lin, Ziyang Luo, Yuchen Tian, Jing Ma, Zhiyong Huang, Tat-Seng Chua","sourceUrl":"https://arxiv.org/abs/2504.07981","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:tax_eval_v2:overall","benchmarkName":"TaxEval v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":72.568,"unit":"percent","family":"vals-ai:tax_eval_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","measuredAt":"2026-09-01","stderr":0.872,"costPerTest":0.034377,"latency":24.404},{"benchmarkId":"vals-ai:medscribe:overall","benchmarkName":"MedScribe","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":72.036,"unit":"percent","family":"vals-ai:medscribe:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medscribe","measuredAt":"2026-10-29","stderr":1.9,"costPerTest":0.061162,"latency":43.386},{"benchmarkId":"erqa","benchmarkName":"ERQA","category":"multimodal","score":70.5,"unit":"percent","family":"erqa","index":63.44869548347105,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaIfBench","benchmarkName":"Artificial Analysis IFBench","category":"instruction","score":70.4,"unit":"percent","family":"ifbench-family","index":54.81317667606192,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"swebench-experiments:swe-bench-verified:overall","benchmarkName":"SWE-bench Verified","category":"coding","domain":null,"eligible":true,"ranked":true,"score":69.6,"unit":"percent","family":"swe-bench-verified","index":52.5619349432166,"aboveFit":false,"run":"high effort · mini-SWE-agent","source":"SWE-bench team","sourceUrl":"https://github.com/SWE-bench/experiments/tree/main/evaluation/verified","measuredAt":"2026-09-01","stderr":null,"costPerTest":0.9600198564,"latency":null},{"benchmarkId":"vals-ai:mortgage_tax:overall","benchmarkName":"MortgageTax","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":69.078,"unit":"percent","family":"vals-ai:mortgage_tax:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mortgage_tax","measuredAt":"2026-09-01","stderr":0.911,"costPerTest":0.041198,"latency":25.602},{"benchmarkId":"swe-bench:swe-bench-multilingual:overall","benchmarkName":"SWE-bench Multilingual","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":68.7,"unit":"percent","family":"swe-bench-multilingual","index":null,"aboveFit":false,"run":"mini-SWE-agent","source":"SWE-bench team","sourceUrl":"https://www.swebench.com/","measuredAt":"2026-02-20","stderr":null,"costPerTest":1.0210855846666667,"latency":null},{"benchmarkId":"swebench-experiments:swe-bench-multilingual:overall","benchmarkName":"SWE-bench Multilingual","category":"coding","domain":null,"eligible":true,"ranked":true,"score":68.7,"unit":"percent","family":"swe-bench-multilingual","index":null,"aboveFit":false,"run":"mini-SWE-agent","source":"SWE-bench team","sourceUrl":"https://github.com/SWE-bench/experiments/tree/main/evaluation/multilingual","measuredAt":"2026-09-02","stderr":null,"costPerTest":1.0210855846666667,"latency":null},{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":64.77714285714286,"unit":"percent","family":"euroeval","index":60.10791351971327,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.4580771702208603,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:corp_fin_v2:overall","benchmarkName":"CorpFin v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":63.675,"unit":"percent","family":"vals-ai:corp_fin_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","measuredAt":"2026-08-12","stderr":0.946,"costPerTest":0.179069,"latency":20.059},{"benchmarkId":"gertLabs","benchmarkName":"Gert Labs Composite Game Benchmark","category":"agentic","score":63.23,"unit":"percent","family":"gertLabs","index":66.74514929959838,"aboveFit":false,"run":null,"source":"Gert Labs","sourceUrl":"https://gertlabs.com/rankings","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmLongBenchDoc","benchmarkName":"MMLongBench-Doc","category":"multimodal","score":60.5,"unit":"percent","family":"mmLongBenchDoc","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"omniscienceAccuracy","benchmarkName":"Artificial Analysis Omniscience Accuracy","category":"knowledge","score":55.8,"unit":"percent","family":"omniscienceAccuracy","index":81.01223437714671,"aboveFit":true,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:finance_agent:overall","benchmarkName":"Finance Agent v1.1","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":55.154,"unit":"percent","family":"vals-ai:finance_agent:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/finance_agent","measuredAt":"2026-06-04","stderr":2.798,"costPerTest":0.558567,"latency":183.624},{"benchmarkId":"vals-ai:terminal-bench-2:overall","benchmarkName":"Terminal-Bench 2.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":55.056,"unit":"percent","family":"vals-ai:terminal-bench-2:overall","index":59.70110180463439,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2","measuredAt":"2026-06-04","stderr":5.303,"costPerTest":0.455782,"latency":416.249},{"benchmarkId":"vals-ai:case_law_v2:overall","benchmarkName":"CaseLaw v2","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":53.055,"unit":"percent","family":"vals-ai:case_law_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/case_law_v2","measuredAt":"2026-05-04","stderr":0.186,"costPerTest":0.137729,"latency":32.345},{"benchmarkId":"vals-ai:medcode:overall","benchmarkName":"MedCode","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":52.198,"unit":"percent","family":"vals-ai:medcode:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medcode","measuredAt":"2026-10-29","stderr":2.073,"costPerTest":0.028248,"latency":55.836},{"benchmarkId":"vals-ai:terminal-bench:overall","benchmarkName":"Terminal-Bench 1.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":51.25,"unit":"percent","family":"vals-ai:terminal-bench:overall","index":55.30666100777599,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench","measuredAt":"2026-01-12","stderr":5.588,"costPerTest":0.318077,"latency":508.608},{"benchmarkId":"odinw13","benchmarkName":"ODINW13","category":"multimodal","score":46.3,"unit":"percent","family":"odinw13","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":39.7,"unit":"percent","family":"hle","index":64.21736629099631,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:ioi-v1:overall","benchmarkName":"IOI v1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":38.834,"unit":"percent","family":"vals-ai:ioi-v1:overall","index":59.46251957342304,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/ioi-v1","measuredAt":"2026-08-09","stderr":11.492,"costPerTest":0.331188,"latency":900.44},{"benchmarkId":"epoch-ai:epoch-frontiermath-2025-02-28-private:overall","benchmarkName":"FrontierMath-2025-02-28-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":37.6,"unit":"percent","family":"epoch-ai:epoch-frontiermath-2025-02-28-private:overall","index":63.106612108779316,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.8400000000000003,"costPerTest":null,"latency":null},{"benchmarkId":"arc-prize:v2-semi-private:overall","benchmarkName":"ARC-AGI-2 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":31.11,"unit":"percent","family":"arc-prize:v2-semi-private:overall","index":53.47219594961076,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.8109,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-private:overall","benchmarkName":"FrontierMath-Tier-4-2025-07-01-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":18.75,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-private:overall","index":62.24640798113014,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":5.693291125,"costPerTest":null,"latency":null},{"benchmarkId":"tau2-bench:banking-knowledge:overall","benchmarkName":"τ²-bench Banking","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":18.04,"unit":"percent","family":"tau2-bench","index":14.76029908440465,"aboveFit":false,"run":"high effort · Sierra","source":"Sierra Research","sourceUrl":"https://github.com/sierra-research/tau2-bench","measuredAt":"2026-10-20","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:vibe-code:overall","benchmarkName":"Vibe Code Bench v1.1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":14.3,"unit":"percent","family":"vals-ai:vibe-code:overall","index":47.4091872608116,"aboveFit":false,"run":"OpenHands","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","measuredAt":"2026-11-09","stderr":3.061,"costPerTest":6.417592,"latency":10398.545},{"benchmarkId":"jobBench","benchmarkName":"JobBench","category":"agentic","score":11.4,"unit":"percent","family":"jobBench","index":42.79891852320333,"aboveFit":false,"run":null,"source":"Yuetai Li, Yichen Feng, Zhangchen Xu, Zixian Ma, Kaiyuan Zheng, Fengqing Jiang, Xinghua Sun, Rulin Shao, Zichen Chen, Yue Huang, Xinyang Han, Brian Lee, Kayla Xu, Shenglai Zeng, Hang Hua, Xiangliang Zhang, Basel Alomair, Ranjay Krishna, Luke Zettlemoyer, Pang Wei Koh, Bhaskar Ramasubramanian, Luyao Niu, Xiang Yue, Radha Poovendran","sourceUrl":"https://arxiv.org/abs/2605.26329","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":9.1,"unit":"percent","family":"critpt","index":54.46565419473278,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-public:overall","benchmarkName":"FrontierMath-Tier-4-2025-07-01-Public","category":"math","domain":null,"eligible":false,"ranked":true,"score":0,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-public:overall","index":null,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":2,"outputPerMillion":12,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":109,"timeToFirstToken":32.65,"source":"BenchLM","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":{"elo":1479.5383062894753,"eloUpper":1483.3103703501579,"eloLower":1475.7662422287926,"votes":41829,"rank":21,"rankUpper":null,"rankLower":null,"arenaName":"Gemini 3 Pro","organization":"google","updatedAt":"2026-11-09"},"quality":{"coveragePercent":100,"benchmarkCount":35,"categoryCount":8,"confidence":"high","state":"available"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:sierra-research","name":"Sierra Research","url":"https://github.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:arc-prize-foundation","name":"ARC Prize Foundation","url":"https://arcprize.org","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:swe-bench-team","name":"SWE-bench team","url":"https://www.swebench.com","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"kimi-k2-7-code","providerId":"moonshot-ai","slug":"kimi-k2-7-code","name":"Kimi K2.7 Code","provider":"Moonshot AI","description":"Kimi K2.7 Code is a reasoning model from Moonshot AI. 30 benchmarks count toward its index, in 6 categories.","logoUrl":null,"releaseDate":"2026-06-12","contextWindowTokens":262144,"modalities":["text","image"],"openWeights":true,"details":{"apiId":"moonshotai/kimi-k2.7-code","maxOutputTokens":235929,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.16,"weightsUrl":"https://huggingface.co/moonshotai/Kimi-K2.7-Code"},"reasoning":true,"overallScore":57.703944433623164,"overallScoreRange":{"low":53.25171984867032,"high":62.15616901857601},"badges":[],"capabilityScores":{"agentic":58.40869230046968,"coding":60.40787836961704,"reasoning":54.590750302920235,"multimodal":null,"knowledge":58.770582926058296,"multilingual":null,"instruction":44.24026302168646,"math":56.9622158206797},"capabilityCounts":{"math":4,"agentic":6,"knowledge":4,"reasoning":4,"coding":10,"instruction":2},"benchmarks":[{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":95.55555555555556,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":64.51274413550408,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":3.1067790907534736,"costPerTest":null,"latency":null},{"benchmarkId":"tau2Bench","benchmarkName":"τ²-Bench Tool-Agent-User Evaluation","category":"agentic","score":90.1,"unit":"percent","family":"tau2-bench","index":62.577377529475555,"aboveFit":false,"run":null,"source":"Victor Barres, Honghua Dong, Soham Ray, Xujie Si, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2506.07982","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":89.6,"unit":"percent","family":"gpqa-diamond","index":59.387772410662265,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":87.87878787878788,"unit":"percent","family":"gpqa-diamond","index":57.80912368534579,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.325315795194205,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-reasoning:overall","benchmarkName":"LiveBench Reasoning","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":82.80775,"unit":"percent","family":"livebench","index":68.89169582982441,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Reasoning","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.05338663366336634,"latency":null},{"benchmarkId":"vals-ai:lcb:overall","benchmarkName":"LiveCodeBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":82.05,"unit":"percent","family":"livecodebench","index":57.424034343173524,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","measuredAt":"2026-09-01","stderr":1.071,"costPerTest":0.027349,"latency":214.178},{"benchmarkId":"mcpMarkVerified","benchmarkName":"MCPMark-Verified","category":"agentic","score":81.1,"unit":"percent","family":"mcpMarkVerified","index":null,"aboveFit":false,"run":null,"source":"MCPMark","sourceUrl":"https://mcpmark.ai/","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-math:overall","benchmarkName":"LiveBench Mathematics","category":"math","domain":null,"eligible":true,"ranked":true,"score":79.60425000000001,"unit":"percent","family":"livebench:livebench-math:overall","index":50.720803321796666,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Mathematics","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.04802163588390501,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":79.3,"unit":"percent","family":"lcr","index":61.70423513811355,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:swebench:overall","benchmarkName":"SWE-bench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":78.2,"unit":"percent","family":"swe-bench-verified","index":58.90493011215636,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/swebench","measuredAt":"2026-09-01","stderr":1.848,"costPerTest":0.265276,"latency":896.71},{"benchmarkId":"livebench:livebench-language:overall","benchmarkName":"LiveBench Language","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":77.90566666666666,"unit":"percent","family":"livebench:livebench-language:overall","index":62.65521880095224,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Language","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.034586,"latency":null},{"benchmarkId":"mcpAtlas","benchmarkName":"MCP Atlas","category":"agentic","score":76,"unit":"percent","family":"mcpAtlas","index":62.24034861306442,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-coding:overall","benchmarkName":"LiveBench Coding","category":"coding","domain":null,"eligible":true,"ranked":true,"score":73.959,"unit":"percent","family":"livebench:livebench-coding:overall","index":58.84089771783336,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Coding","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.014160683760683761,"latency":null},{"benchmarkId":"vals-ai:terminal-bench-2-1:overall","benchmarkName":"Terminal-Bench 2.1","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":67.041,"unit":"percent","family":"terminal-bench-2-1","index":60.35040969698791,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2-1","measuredAt":"2026-10-01","stderr":0.375,"costPerTest":0.256503,"latency":784.159},{"benchmarkId":"aaIfBench","benchmarkName":"Artificial Analysis IFBench","category":"instruction","score":63.1,"unit":"percent","family":"ifbench-family","index":47.991249187806545,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-data-analysis:overall","benchmarkName":"LiveBench Data Analysis","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":62.65733333333333,"unit":"percent","family":"livebench","index":42.78816104330623,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Data%20Analysis","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.088934,"latency":null},{"benchmarkId":"kimiCodeBenchV2","benchmarkName":"Kimi Code Bench v2","category":"coding","score":62,"unit":"percent","family":"kimiCodeBenchV2","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-instruction-following:overall","benchmarkName":"LiveBench Instruction Following","category":"instruction","domain":null,"eligible":true,"ranked":true,"score":56.29175,"unit":"percent","family":"livebench:livebench-instruction-following:overall","index":40.489276855566374,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=IF","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.047241,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tiers-1-3-v2-private:overall","benchmarkName":"FrontierMath-Tiers-1-3-v2-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":54.03508771929825,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tiers-1-3-v2-private:overall","index":59.33615719655181,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.957276813514758,"costPerTest":null,"latency":null},{"benchmarkId":"programBench","benchmarkName":"ProgramBench: Can Language Models Rebuild Programs From Scratch?","category":"coding","score":53.6,"unit":"percent","family":"programBench","index":74.10991794227418,"aboveFit":false,"run":null,"source":"John Yang, Kilian Lieret, Jeffrey Ma, Parth Thakkar, Dmitrii Pedchenko, Sten Sootla, Emily McMilin, Pengcheng Yin, Rui Hou, Gabriel Synnaeve, Diyi Yang, Ofir Press","sourceUrl":"https://programbench.com/static/paper.pdf","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"openHarmonyBench","benchmarkName":"OpenHarmony Bench v1.0","category":"coding","score":52.1,"unit":"percent","family":"openHarmonyBench","index":59.62333279405969,"aboveFit":false,"run":null,"source":"OpenHarmony Bench authors","sourceUrl":"https://arxiv.org/abs/2608.16022","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:skillsbench:overall","benchmarkName":"SkillsBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":50.036,"unit":"percent","family":"vals-ai:skillsbench:overall","index":62.385499428422754,"aboveFit":false,"run":"OpenHands","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/skillsbench","measuredAt":"2026-10-29","stderr":4.237,"costPerTest":0.443192,"latency":803.827},{"benchmarkId":"cursorBench32","benchmarkName":"cursorBench32","category":"coding","score":49.7,"unit":"percent","family":"cursorBench32","index":56.67155708181238,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaSciCode","benchmarkName":"Artificial Analysis SciCode","category":"coding","score":47.8,"unit":"percent","family":"scicode-family","index":58.156235241135505,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/scicode","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:vibe-code:overall","benchmarkName":"Vibe Code Bench v1.1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":47.211,"unit":"percent","family":"vals-ai:vibe-code:overall","index":59.283993131401616,"aboveFit":false,"run":"OpenHands","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","measuredAt":"2026-11-09","stderr":5.191,"costPerTest":3.835806,"latency":10318.246},{"benchmarkId":"kimiClaw247","benchmarkName":"Kimi Claw 24/7 Bench","category":"agentic","score":46.9,"unit":"percent","family":"kimiClaw247","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-agentic-coding:overall","benchmarkName":"LiveBench Agentic Coding","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":45.65633333333333,"unit":"percent","family":"livebench:livebench-agentic-coding:overall","index":56.24728932520884,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Agentic%20Coding","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.39455,"latency":null},{"benchmarkId":"terminalBenchHard","benchmarkName":"Terminal-Bench Hard","category":"agentic","score":44.7,"unit":"percent","family":"terminalBenchHard","index":55.770078831378314,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/terminalbench-hard","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-simpleqa-verified:overall","benchmarkName":"SimpleQA Verified","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":36.5,"unit":"percent","family":"epoch-ai:epoch-simpleqa-verified:overall","index":54.32677446592064,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.5231776226264846,"costPerTest":null,"latency":null},{"benchmarkId":"mlsBenchLite","benchmarkName":"MLS-Bench Lite","category":"coding","score":35.1,"unit":"percent","family":"mlsBenchLite","index":null,"aboveFit":false,"run":null,"source":"MLS-Bench","sourceUrl":"https://mls-bench.com/","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":35,"unit":"percent","family":"hle","index":59.50189038935628,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gdpvalAaNormalized","benchmarkName":"GDPval-AA normalized","category":"agentic","score":26.9,"unit":"percent","family":"gdpvalAaNormalized","index":53.266649806703064,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:code-migration:overall","benchmarkName":"Code Migration","category":"coding","domain":null,"eligible":true,"ranked":true,"score":25.392,"unit":"percent","family":"vals-ai:code-migration:overall","index":58.67838590390103,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/code-migration","measuredAt":"2026-10-29","stderr":4.161,"costPerTest":5.040913,"latency":5945.669},{"benchmarkId":"epoch-ai:epoch-chess-puzzles:overall","benchmarkName":"Chess Puzzles","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":14.000000000000002,"unit":"percent","family":"epoch-ai:epoch-chess-puzzles:overall","index":44.61088845724028,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":3.4873508801977695,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tier-4-v2-private:overall","benchmarkName":"FrontierMath-Tier-4-v2-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":12.195121951219512,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tier-4-v2-private:overall","index":53.27915862886624,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":5.173952057462543,"costPerTest":null,"latency":null},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":10,"unit":"percent","family":"critpt","index":56.20794917976178,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:programbench:overall","benchmarkName":"ProgramBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":0,"unit":"percent","family":"vals-ai:programbench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/programbench","measuredAt":"2026-11-09","stderr":0,"costPerTest":3.185154,"latency":3860.332}],"price":{"inputPerMillion":0.6,"outputPerMillion":3.5,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":35,"timeToFirstToken":0.947,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":85,"benchmarkCount":30,"categoryCount":6,"confidence":"medium","state":"available"},"hosts":[{"host":"coreweave","name":"CoreWeave","kind":"host","url":"https://docs.wandb.ai/inference","inputPerMillion":0.6,"outputPerMillion":3,"cacheReadPerMillion":0.12,"contextTokens":262144,"maxOutputTokens":262144,"deprecated":false},{"host":"deepinfra","name":"Deep Infra","kind":"host","url":"https://deepinfra.com/models","inputPerMillion":0.68,"outputPerMillion":3.4,"cacheReadPerMillion":0.136,"contextTokens":262144,"maxOutputTokens":262144,"deprecated":true},{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":0.6,"outputPerMillion":3.5,"cacheReadPerMillion":0.16,"contextTokens":262144,"maxOutputTokens":235929,"deprecated":false},{"host":"siliconflow","name":"SiliconFlow","kind":"host","url":"https://cloud.siliconflow.com/models","inputPerMillion":0.85916,"outputPerMillion":3.8,"cacheReadPerMillion":0.17993,"contextTokens":262144,"maxOutputTokens":262144,"deprecated":false},{"host":"cloudflare","name":"Cloudflare Workers AI","kind":"cloud","url":"https://developers.cloudflare.com/workers-ai/models/","inputPerMillion":0.95,"outputPerMillion":4,"cacheReadPerMillion":0.19,"contextTokens":262144,"maxOutputTokens":262144,"deprecated":true},{"host":"databricks","name":"Databricks","kind":"cloud","url":"https://docs.databricks.com/aws/en/machine-learning/foundation-models/","inputPerMillion":0.95,"outputPerMillion":4,"cacheReadPerMillion":0.19,"contextTokens":262144,"maxOutputTokens":262144,"deprecated":true},{"host":"azure","name":"Microsoft Azure","kind":"cloud","url":"https://learn.microsoft.com/en-us/azure/ai-services/openai/concepts/models","inputPerMillion":0.95,"outputPerMillion":4,"cacheReadPerMillion":0.19,"contextTokens":262144,"maxOutputTokens":262144,"deprecated":false},{"host":"moonshot-ai","name":"Moonshot AI","kind":"maker","url":"https://platform.moonshot.ai/docs/api/chat","inputPerMillion":0.95,"outputPerMillion":4,"cacheReadPerMillion":0.19,"contextTokens":262144,"maxOutputTokens":262144,"deprecated":false},{"host":"novita-ai","name":"Novita AI","kind":"host","url":"https://novita.ai/docs/guides/introduction","inputPerMillion":0.95,"outputPerMillion":4,"cacheReadPerMillion":0.19,"contextTokens":262144,"maxOutputTokens":262144,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:livebench","name":"LiveBench","url":"https://livebench.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"qwen3-7-plus","providerId":"alibaba","slug":"qwen3-7-plus","name":"Qwen3.7 Plus","provider":"Alibaba","description":"Qwen3.7 Plus is a reasoning model from Alibaba. 48 benchmarks count toward its index, in 8 categories.","logoUrl":null,"releaseDate":"2026-06-03","contextWindowTokens":1000000,"modalities":["text","image"],"openWeights":false,"details":{"apiId":"qwen/qwen3.7-plus","maxOutputTokens":131072,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.064,"weightsUrl":null},"reasoning":true,"overallScore":57.69838466691166,"overallScoreRange":{"low":55.364446203712596,"high":60.03232313011072},"badges":[],"capabilityScores":{"agentic":57.28990486929247,"coding":58.25807702425782,"reasoning":57.72992262589184,"multimodal":62.61533428499545,"knowledge":54.81093253081311,"multilingual":54.00219689597205,"instruction":58.06407568803873,"math":55.35043475362427},"capabilityCounts":{"instruction":2,"knowledge":6,"agentic":15,"math":4,"reasoning":4,"multimodal":7,"coding":8,"multilingual":2},"benchmarks":[{"benchmarkId":"ifeval","benchmarkName":"Instruction-Following Eval","category":"instruction","score":94.6,"unit":"percent","family":"ifeval","index":53.69871318915508,"aboveFit":false,"run":null,"source":"Jeffrey Zhou, Tianjian Lu, Swaroop Mishra, Siddhartha Brahma, Sujoy Basu, Yi Luan, Denny Zhou, Le Hou","sourceUrl":"https://arxiv.org/abs/2311.07911","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmluRedux","benchmarkName":"MMLU-Redux","category":"knowledge","score":94.5,"unit":"percent","family":"mmluRedux","index":50.46926829204164,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"tau2Bench","benchmarkName":"τ²-Bench Tool-Agent-User Evaluation","category":"agentic","score":93,"unit":"percent","family":"tau2-bench","index":64.50173955404821,"aboveFit":false,"run":null,"source":"Victor Barres, Honghua Dong, Soham Ray, Xujie Si, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2506.07982","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hmmtFeb2026","benchmarkName":"Harvard-MIT Mathematics Tournament February 2026","category":"math","score":92.9,"unit":"percent","family":"hmmtFeb2026","index":58.41387833475139,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mrcrv2","benchmarkName":"MRCRv2","category":"reasoning","score":91.7,"unit":"percent","family":"mrcrv2","index":67.22605703585256,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"omniDocBench15","benchmarkName":"OmniDocBench 1.5","category":"multimodal","score":91.4,"unit":"percent","family":"omniDocBench15","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mathVision","benchmarkName":"MathVision","category":"multimodal","score":90.3,"unit":"percent","family":"mathVision","index":61.17532423010602,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gpqa","benchmarkName":"Graduate-Level Google-Proof Q&A","category":"knowledge","score":90.3,"unit":"percent","family":"gpqa-diamond","index":60.029793283106116,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gpqaDiamond","benchmarkName":"GPQA Diamond","category":"knowledge","score":90.3,"unit":"percent","family":"gpqa-diamond","index":60.029793283106116,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":90,"unit":"percent","family":"gpqa-diamond","index":59.754641480630184,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"liveCodeBenchV6","benchmarkName":"LiveCodeBench v6","category":"coding","score":89.6,"unit":"percent","family":"liveCodeBenchV6","index":57.983791485794406,"aboveFit":false,"run":null,"source":"LiveCodeBench maintainers","sourceUrl":"https://github.com/LiveCodeBench/LiveCodeBench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:sage:overall","benchmarkName":"SAGE","category":null,"domain":"Education","eligible":false,"ranked":true,"score":89.147,"unit":"percent","family":"vals-ai:sage:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/sage","measuredAt":"2026-11-05","stderr":1.396,"costPerTest":0.020459,"latency":108.812},{"benchmarkId":"mmmlu","benchmarkName":"MMMLU","category":"knowledge","score":89,"unit":"percent","family":"mmmlu","index":47.225743040525806,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"maxife","benchmarkName":"MAXIFE","category":"multilingual","score":88.8,"unit":"percent","family":"maxife","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmluPro","benchmarkName":"Massive Multitask Language Understanding Professional","category":"knowledge","score":88.5,"unit":"percent","family":"mmlu-pro","index":55.98013524925106,"aboveFit":false,"run":null,"source":"Yubo Wang, Xueguang Ma, Ge Zhang, Yuansheng Ni, Abhranil Chandra, Shiguang Guo, Weiming Ren, Aaran Arulraj, Xuan He, Ziyan Jiang, Tianle Li, Max Ku, Kai Wang, Alex Zhuang, Rongqi Fan, Xiang Yue, Wenhu Chen","sourceUrl":"https://arxiv.org/abs/2406.01574","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"videoMmeWithSub","benchmarkName":"Video-MME with subtitle","category":"multimodal","score":88,"unit":"percent","family":"videoMmeWithSub","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mlvuAvg","benchmarkName":"MLVU mean average","category":"multimodal","score":87.4,"unit":"percent","family":"mlvuAvg","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"realWorldQa","benchmarkName":"RealWorldQA","category":"multimodal","score":86.9,"unit":"percent","family":"realWorldQa","index":62.43883176408633,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"imoAnswerBench","benchmarkName":"IMOAnswerBench","category":"math","score":86,"unit":"percent","family":"imoAnswerBench","index":58.53143836297463,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"charxiv","benchmarkName":"CharXiv Reasoning","category":"multimodal","score":85.9,"unit":"percent","family":"charxiv","index":61.98121578757658,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"videoMmmu","benchmarkName":"VideoMMMU","category":"multimodal","score":85.4,"unit":"percent","family":"videoMmmu","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmluProX","benchmarkName":"MMLU-ProX","category":"multilingual","score":85.4,"unit":"percent","family":"mmluProX","index":54.7422419456715,"aboveFit":false,"run":null,"source":"MMLU-ProX authors","sourceUrl":"https://arxiv.org/abs/2503.10497","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"polyMath","benchmarkName":"PolyMath","category":"multilingual","score":84,"unit":"percent","family":"polyMath","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"include","benchmarkName":"INCLUDE","category":"multilingual","score":83,"unit":"percent","family":"include","index":53.26215184627259,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":81.81818181818183,"unit":"percent","family":"gpqa-diamond","index":52.25050141310468,"aboveFit":false,"run":"none effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.7479603010538827,"costPerTest":null,"latency":null},{"benchmarkId":"simpleVqa","benchmarkName":"SimpleVQA","category":"multimodal","score":81.7,"unit":"percent","family":"simpleVqa","index":71.34621331110284,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"androidWorld","benchmarkName":"AndroidWorld","category":"agentic","score":81,"unit":"percent","family":"androidWorld","index":61.00462699462438,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaMmmuPro","benchmarkName":"Artificial Analysis MMMU-Pro","category":"multimodal","score":80.5,"unit":"percent","family":"mmmu-pro","index":59.91650236992847,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmmu-pro","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":80,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":55.46261453521162,"aboveFit":false,"run":"none effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":6.030226891555273,"costPerTest":null,"latency":null},{"benchmarkId":"ifBench","benchmarkName":"Instruction Following Benchmark","category":"instruction","score":79.1,"unit":"percent","family":"ifbench-family","index":62.94341902507859,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmmuPro","benchmarkName":"Massive Multi-discipline Multimodal Understanding Pro","category":"multimodal","score":79,"unit":"percent","family":"mmmu-pro","index":57.96438397224972,"aboveFit":false,"run":null,"source":"MMMU-Pro authors","sourceUrl":"https://arxiv.org/abs/2409.02813","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"screenSpotPro","benchmarkName":"ScreenSpot Pro","category":"multimodal","score":79,"unit":"percent","family":"screenSpotPro","index":59.6246444816172,"aboveFit":false,"run":null,"source":"Kaixin Li, Ziyang Meng, Hongzhan Lin, Ziyang Luo, Yuchen Tian, Jing Ma, Zhiyong Huang, Tat-Seng Chua","sourceUrl":"https://arxiv.org/abs/2504.07981","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaIfBench","benchmarkName":"Artificial Analysis IFBench","category":"instruction","score":78,"unit":"percent","family":"ifbench-family","index":61.91545734876614,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"sweVerified","benchmarkName":"Software Engineering Benchmark Verified","category":"coding","score":77.7,"unit":"percent","family":"swe-bench-verified","index":58.53615132326451,"aboveFit":false,"run":null,"source":"Carlos E. Jimenez, John Yang, Alexander Wettig, Shunyu Yao, Kexin Pei, Ofir Press, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2310.06770","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"osWorldVerified","benchmarkName":"OSWorld-Verified","category":"agentic","score":73.3,"unit":"percent","family":"osWorldVerified","index":59.65295318355919,"aboveFit":false,"run":null,"source":"Tianbao Xie, Danyang Zhang, Jixuan Chen, Xiaochuan Li, Siheng Zhao, Ruisheng Cao, Toh Jing Hua, Zhoujun Cheng, Dongchan Shin, Fangyu Lei, Yitao Liu, Yiheng Xu, Shuyan Zhou, Silvio Savarese, Caiming Xiong, Victor Zhong, Tao Yu","sourceUrl":"https://os-world.github.io/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mcpAtlas","benchmarkName":"MCP Atlas","category":"agentic","score":73.2,"unit":"percent","family":"mcpAtlas","index":60.354840944063426,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":73,"unit":"percent","family":"lcr","index":57.3819278269239,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"bfclV4","benchmarkName":"Berkeley Function Calling Leaderboard v4","category":"agentic","score":72.9,"unit":"percent","family":"bfclV4","index":52.851866574611336,"aboveFit":false,"run":null,"source":"Shishir G. Patil et al.","sourceUrl":"https://gorilla.cs.berkeley.edu/leaderboard.html","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"superGpqa","benchmarkName":"SuperGPQA: Scaling LLM Evaluation Across 285 Graduate Disciplines","category":"knowledge","score":71.4,"unit":"percent","family":"superGpqa","index":55.64584692008124,"aboveFit":false,"run":null,"source":"Xiaoxuan Du, Yao Yao, Kexin Ma, Bowen Wang, Tianyu Zheng, Kaiyan Zhu, Yiming Zhang, Yutao Zhu, Jiawei Zhou, Jingren Zhou","sourceUrl":"https://arxiv.org/abs/2502.14739","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"medXpertQaMm","benchmarkName":"MedXpertQA Multimodal","category":"multimodal","score":71,"unit":"percent","family":"medXpertQaMm","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"ocrBenchV2","benchmarkName":"OCRBench V2","category":"multimodal","score":70.7,"unit":"percent","family":"ocrBenchV2","index":null,"aboveFit":false,"run":null,"source":"OCRBench authors","sourceUrl":"https://arxiv.org/abs/2501.00321","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"terminalBench2","benchmarkName":"Terminal-Bench 2.0","category":"agentic","score":70.3,"unit":"percent","family":"terminalBench2","index":63.905226959462624,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"erqa","benchmarkName":"ERQA","category":"multimodal","score":69.8,"unit":"percent","family":"erqa","index":62.80066724939006,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:mortgage_tax:overall","benchmarkName":"MortgageTax","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":66.176,"unit":"percent","family":"vals-ai:mortgage_tax:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mortgage_tax","measuredAt":"2026-09-01","stderr":0.928,"costPerTest":0.007588,"latency":60.548},{"benchmarkId":"clawEval","benchmarkName":"Claw-Eval","category":"agentic","score":62.7,"unit":"percent","family":"clawEval","index":55.785388142935474,"aboveFit":false,"run":null,"source":"Bowen Ye, Rang Li, Qibin Yang, Yuanxin Liu, Linli Yao, Hanglong Lv, Zhihui Xie, Chenxin An, Lei Li, Lingpeng Kong, Qi Liu, Zhifang Sui, Tong Yang","sourceUrl":"https://arxiv.org/abs/2604.06132","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"deepPlanning","benchmarkName":"DeepPlanning","category":"agentic","score":62.3,"unit":"percent","family":"deepPlanning","index":null,"aboveFit":false,"run":null,"source":"DeepPlanning authors","sourceUrl":"https://arxiv.org/abs/2601.18137","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"qwenClawBench","benchmarkName":"QwenClawBench","category":"agentic","score":61.8,"unit":"percent","family":"qwenClawBench","index":58.73978476695057,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"nova63","benchmarkName":"NOVA-63","category":"multilingual","score":58.8,"unit":"percent","family":"nova63","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"swePro","benchmarkName":"SWE-bench Pro","category":"coding","score":57.6,"unit":"percent","family":"swePro","index":57.89211859034134,"aboveFit":false,"run":null,"source":"Xiang Deng, Jeff Da, Edwin Pan, Yannis Yiming He, Charles Ide, Kanak Garg, Niklas Lauffer, Andrew Park, Nitin Pasari, Chetan Rane, Karmini Sampath, Maya Krishnan, Srivatsa Kundurthy, Sean Hendryx, Zifan Wang, Chen Bo Calvin Zhang, Noah Jacobson, Bing Liu, Brad Kenstler","sourceUrl":"https://arxiv.org/abs/2509.16941","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:skillsbench:overall","benchmarkName":"SkillsBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":54.296,"unit":"percent","family":"vals-ai:skillsbench:overall","index":65.5560988625412,"aboveFit":false,"run":"OpenHands","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/skillsbench","measuredAt":"2026-10-29","stderr":4.333,"costPerTest":0.137331,"latency":724.46},{"benchmarkId":"vals-ai:terminal-bench-2-1:overall","benchmarkName":"Terminal-Bench 2.1","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":52.809,"unit":"percent","family":"terminal-bench-2-1","index":52.02674814980849,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2-1","measuredAt":"2026-10-01","stderr":0.649,"costPerTest":0.095849,"latency":654.983},{"benchmarkId":"sciCode","benchmarkName":"Scientific Code Benchmark","category":"coding","score":51.3,"unit":"percent","family":"scicode-family","index":62.71915425358707,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"odinw13","benchmarkName":"ODINW13","category":"multimodal","score":51.1,"unit":"percent","family":"odinw13","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:emb:overall","benchmarkName":"EMB","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":49.345,"unit":"percent","family":"vals-ai:emb:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/emb","measuredAt":"2026-10-20","stderr":3.181,"costPerTest":0.597724,"latency":2209.243},{"benchmarkId":"terminalBenchHard","benchmarkName":"Terminal-Bench Hard","category":"agentic","score":47,"unit":"percent","family":"terminalBenchHard","index":57.65959585369635,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/terminalbench-hard","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:vibe-code:overall","benchmarkName":"Vibe Code Bench v1.1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":46.391,"unit":"percent","family":"vals-ai:vibe-code:overall","index":58.98812424823538,"aboveFit":false,"run":"OpenHands","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","measuredAt":"2026-11-09","stderr":4.614,"costPerTest":1.082348,"latency":2231.75},{"benchmarkId":"aaSciCode","benchmarkName":"Artificial Analysis SciCode","category":"coding","score":46.1,"unit":"percent","family":"scicode-family","index":55.93996029223047,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/scicode","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vitaBench","benchmarkName":"VITA-Bench","category":"agentic","score":45.6,"unit":"percent","family":"vitaBench","index":59.00001970966075,"aboveFit":false,"run":null,"source":"Meituan LongCat Team","sourceUrl":"https://vitabench.github.io/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmSearchPlus","benchmarkName":"MMSearch-Plus","category":"multimodal","score":41.4,"unit":"percent","family":"mmSearchPlus","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"nl2Repo","benchmarkName":"NL2Repo","category":"coding","score":41.1,"unit":"percent","family":"nl2Repo","index":55.76001432171678,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaEnterpriseOpsGym","benchmarkName":"Artificial Analysis EnterpriseOps-Gym","category":"agentic","score":40.6,"unit":"percent","family":"aaEnterpriseOpsGym","index":null,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/enterprise-ops-gym-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:fabv2:overall","benchmarkName":"Finance Agent (v2)","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":38.22,"unit":"percent","family":"vals-ai:fabv2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/fabv2","measuredAt":"2026-10-29","stderr":1.04,"costPerTest":0.357661,"latency":542.045},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":35.6,"unit":"percent","family":"hle","index":60.10386603637416,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hle","benchmarkName":"Humanity's Last Exam","category":"knowledge","score":34.7,"unit":"percent","family":"hle","index":59.20090256584734,"aboveFit":false,"run":null,"source":"Center for AI Safety, Scale AI, and thousands of expert contributors","sourceUrl":"https://lastexam.ai/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tiers-1-3-v2-private:overall","benchmarkName":"FrontierMath-Tiers-1-3-v2-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":34.385964912280706,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tiers-1-3-v2-private:overall","index":48.993807781559426,"aboveFit":false,"run":"none effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.8185763989072585,"costPerTest":null,"latency":null},{"benchmarkId":"apex","benchmarkName":"Apex","category":"math","score":22.7,"unit":"percent","family":"apex","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"apexAgentsAa","benchmarkName":"APEX-Agents-AA","category":"agentic","score":22.4,"unit":"percent","family":"apexAgentsAa","index":57.469318666644604,"aboveFit":false,"run":null,"source":"Artificial Analysis / Mercor","sourceUrl":"https://artificialanalysis.ai/evaluations/apex-agents-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-mystery-game-puzzles:overall","benchmarkName":"Mystery Game Puzzles","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":17,"unit":"percent","family":"epoch-ai:epoch-mystery-game-puzzles:overall","index":51.84605144605811,"aboveFit":false,"run":"none effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":3.7752516806863685,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:legal_research:overall","benchmarkName":"Legal Research Bench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":16.346,"unit":"percent","family":"vals-ai:legal_research:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_research","measuredAt":"2026-11-09","stderr":2.57,"costPerTest":0.295949,"latency":831.995},{"benchmarkId":"gdpvalAaNormalized","benchmarkName":"GDPval-AA normalized","category":"agentic","score":13.3,"unit":"percent","family":"gdpvalAaNormalized","index":43.542965847694155,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:code-migration:overall","benchmarkName":"Code Migration","category":"coding","domain":null,"eligible":true,"ranked":true,"score":12.856,"unit":"percent","family":"vals-ai:code-migration:overall","index":52.018760089260184,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/code-migration","measuredAt":"2026-10-29","stderr":2.931,"costPerTest":0.41923,"latency":2146.993},{"benchmarkId":"vals-ai:tax_agent_bench:overall","benchmarkName":"Tax Agent Bench","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":10.466,"unit":"percent","family":"vals-ai:tax_agent_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench","measuredAt":"2026-11-09","stderr":1.993,"costPerTest":0.093885,"latency":310.263},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":9.1,"unit":"percent","family":"critpt","index":54.46565419473278,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"osWorld2","benchmarkName":"OSWorld 2.0","category":"agentic","score":2.8,"unit":"percent","family":"osWorld2","index":55.203270389781075,"aboveFit":false,"run":null,"source":"Mengqi Yuan, Zilong Zhou, Xinzhuang Xiong, Weiming Wu, Jiayang Sun, Jiamin Song, Kaiqian Cui, Bowen Wang, Haoyuan Wu, Yitong Li, Dunjie Lu, Haikong Lu, Qi Zhen, Xinyuan Wang, Jiaqi Deng, Yuhao Yang, Cheng Chen, Boyuan Zheng, Alex Su, Xiao Yu, Hao Zou, Saaket Agashe, Xing Han Lu, Manpreet Kaur, Zhengyang Qi, Vincent Sunn Chen, Frederic Sala, Dayiheng Liu, Junyang Lin, Zhou Yu, Yu Su, Siva Reddy, Xin Eric Wang, Peng Qi, Tianbao Xie, Tao Yu","sourceUrl":"https://arxiv.org/abs/2606.29537","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lmarena:agent-bash-recovery:overall","benchmarkName":"Agent Arena command recovery","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":-6.30996816869549,"unit":"index","family":"agent-arena","index":61.34301825388173,"aboveFit":false,"run":null,"source":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset","measuredAt":"2026-10-08","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lmarena:agent-task-outcome:overall","benchmarkName":"Agent Arena task outcome","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":-10.80155794916019,"unit":"index","family":"agent-arena","index":57.65022730184636,"aboveFit":false,"run":null,"source":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset","measuredAt":"2026-11-11","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lmarena:agent-steerability:overall","benchmarkName":"Agent Arena steerability","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":-11.599041861401993,"unit":"index","family":"agent-arena","index":56.994570503371854,"aboveFit":false,"run":null,"source":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset","measuredAt":"2026-11-11","stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":0.32,"outputPerMillion":1.28,"batch":{"inputPerMillion":0.22999999999999998,"outputPerMillion":0.9199999999999999},"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":51,"timeToFirstToken":3.5255,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":{"elo":1454.2317075855708,"eloUpper":1458.5701660746581,"eloLower":1449.893249096483,"votes":41318,"rank":47,"rankUpper":null,"rankLower":null,"arenaName":"Qwen3.7-Plus","organization":"alibaba","updatedAt":"2026-11-09"},"quality":{"coveragePercent":100,"benchmarkCount":48,"categoryCount":8,"confidence":"high","state":"available"},"hosts":[{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":0.32,"outputPerMillion":1.28,"cacheReadPerMillion":0.064,"contextTokens":1000000,"maxOutputTokens":131072,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:lmarena","name":"LMArena","url":"https://huggingface.co","license":"CC BY 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"deepseekv4pro","providerId":"deepseek","slug":"deepseek-deepseek-v4-pro-0423","name":"DeepSeek V4 Pro 0423","provider":"DeepSeek","description":"DeepSeek V4 Pro 0423 is a model from DeepSeek. 24 benchmarks count toward its index, in 6 categories.","logoUrl":null,"releaseDate":"2026-04-24T03:17:59.000Z","contextWindowTokens":1048576,"modalities":["text"],"openWeights":true,"details":{"apiId":"deepseek/deepseek-v4-pro","maxOutputTokens":384000,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.07435599999999999,"weightsUrl":"https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro"},"reasoning":null,"overallScore":57.56351148825038,"overallScoreRange":{"low":52.76282405985005,"high":62.36419891665071},"badges":[],"capabilityScores":{"agentic":55.94630897746539,"coding":59.17864978034201,"reasoning":54.57074228170444,"multimodal":null,"knowledge":59.92427588542434,"multilingual":null,"instruction":52.74790261868855,"math":57.631249613220305},"capabilityCounts":{"math":5,"knowledge":4,"coding":7,"reasoning":3,"instruction":1,"agentic":4},"benchmarks":[{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":96.66666666666667,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":65.15918196409639,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.002208376727653,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-math:overall","benchmarkName":"LiveBench Mathematics","category":"math","domain":null,"eligible":true,"ranked":true,"score":90.6755,"unit":"percent","family":"livebench:livebench-math:overall","index":66.15101804988763,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Mathematics","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.03530343007915567,"latency":null},{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":89.64646464646465,"unit":"percent","family":"gpqa-diamond","index":59.43038851474945,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.7473014662966007,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:gpqa:overall","benchmarkName":"GPQA Diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":89.394,"unit":"percent","family":"gpqa-diamond","index":59.1988348396288,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","measuredAt":"2026-09-01","stderr":1.654,"costPerTest":0.125847,"latency":900.378},{"benchmarkId":"vals-ai:lcb:overall","benchmarkName":"LiveCodeBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":87.484,"unit":"percent","family":"livecodebench","index":61.93284818157441,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","measuredAt":"2026-09-01","stderr":0.953,"costPerTest":0.109669,"latency":715.504},{"benchmarkId":"vals-ai:mmlu_pro:overall","benchmarkName":"MMLU Pro","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":87.249,"unit":"percent","family":"mmlu-pro","index":54.40402499010189,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","measuredAt":"2026-09-01","stderr":0.335,"costPerTest":0.027032,"latency":577.617},{"benchmarkId":"livebench:livebench-reasoning:overall","benchmarkName":"LiveBench Reasoning","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":82.69225,"unit":"percent","family":"livebench","index":68.7420732032695,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Reasoning","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.04137623762376238,"latency":null},{"benchmarkId":"vals-ai:legal_bench:overall","benchmarkName":"LegalBench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":80.323,"unit":"percent","family":"vals-ai:legal_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","measuredAt":"2026-10-13","stderr":0.468,"costPerTest":0.004788,"latency":184.46},{"benchmarkId":"livebench:livebench-language:overall","benchmarkName":"LiveBench Language","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":78.13033333333333,"unit":"percent","family":"livebench:livebench-language:overall","index":62.909274032854526,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Language","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.014205333333333332,"latency":null},{"benchmarkId":"epoch-ai:epoch-swe-bench-verified:overall","benchmarkName":"SWE-Bench verified","category":"coding","domain":null,"eligible":true,"ranked":true,"score":77.63975155279503,"unit":"percent","family":"swe-bench-verified","index":58.49171462447879,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.897828052158487,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:swebench:overall","benchmarkName":"SWE-bench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":77.4,"unit":"percent","family":"swe-bench-verified","index":58.314884049929404,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/swebench","measuredAt":"2026-09-01","stderr":1.872,"costPerTest":0.436763,"latency":634.611},{"benchmarkId":"vals-ai:medscribe:overall","benchmarkName":"MedScribe","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":75.144,"unit":"percent","family":"vals-ai:medscribe:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medscribe","measuredAt":"2026-10-29","stderr":2.002,"costPerTest":0.053954,"latency":345.723},{"benchmarkId":"livebench:livebench-data-analysis:overall","benchmarkName":"LiveBench Data Analysis","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":74.53766666666667,"unit":"percent","family":"livebench","index":58.17834872885938,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Data%20Analysis","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.042776,"latency":null},{"benchmarkId":"vals-ai:tax_eval_v2:overall","benchmarkName":"TaxEval v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":72.077,"unit":"percent","family":"vals-ai:tax_eval_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","measuredAt":"2026-09-01","stderr":0.877,"costPerTest":0.033187,"latency":189.695},{"benchmarkId":"livebench:livebench-coding:overall","benchmarkName":"LiveBench Coding","category":"coding","domain":null,"eligible":true,"ranked":true,"score":69.994,"unit":"percent","family":"livebench:livebench-coding:overall","index":53.2645708940229,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Coding","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.006206837606837607,"latency":null},{"benchmarkId":"vals-ai:public-benefits-bench:overall","benchmarkName":"Public Benefits Bench v1.1","category":null,"domain":"Public benefits","eligible":false,"ranked":true,"score":62.923,"unit":"percent","family":"vals-ai:public-benefits-bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/public-benefits-bench","measuredAt":"2026-10-12","stderr":1.256,"costPerTest":0.460453,"latency":922.395},{"benchmarkId":"livebench:livebench-instruction-following:overall","benchmarkName":"LiveBench Instruction Following","category":"instruction","domain":null,"eligible":true,"ranked":true,"score":62.35,"unit":"percent","family":"livebench:livebench-instruction-following:overall","index":52.74790261868855,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=IF","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.020139499999999998,"latency":null},{"benchmarkId":"vals-ai:corp_fin_v2:overall","benchmarkName":"CorpFin v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":61.383,"unit":"percent","family":"vals-ai:corp_fin_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","measuredAt":"2026-08-12","stderr":0.959,"costPerTest":0.148855,"latency":79.331},{"benchmarkId":"vals-ai:finance_agent:overall","benchmarkName":"Finance Agent v1.1","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":60.389,"unit":"percent","family":"vals-ai:finance_agent:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/finance_agent","measuredAt":"2026-06-04","stderr":2.796,"costPerTest":0.585364,"latency":588.439},{"benchmarkId":"vals-ai:case_law_v2:overall","benchmarkName":"CaseLaw v2","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":59.378,"unit":"percent","family":"vals-ai:case_law_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/case_law_v2","measuredAt":"2026-05-04","stderr":1.5,"costPerTest":0.013152,"latency":48.444},{"benchmarkId":"vals-ai:public-benefits-bench-v1:overall","benchmarkName":"Public Benefits Bench v1","category":null,"domain":"Public benefits","eligible":false,"ranked":true,"score":57.578,"unit":"percent","family":"vals-ai:public-benefits-bench-v1:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/public-benefits-bench-v1","measuredAt":"2026-06-09","stderr":1.286,"costPerTest":0.359673,"latency":977.871},{"benchmarkId":"vals-ai:terminal-bench-2:overall","benchmarkName":"Terminal-Bench 2.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":56.18,"unit":"percent","family":"vals-ai:terminal-bench-2:overall","index":60.37787485593908,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2","measuredAt":"2026-06-04","stderr":5.289,"costPerTest":0.179637,"latency":828.342},{"benchmarkId":"vals-ai:emb:overall","benchmarkName":"EMB","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":51.623,"unit":"percent","family":"vals-ai:emb:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/emb","measuredAt":"2026-10-20","stderr":3.053,"costPerTest":0.796757,"latency":822.144},{"benchmarkId":"vals-ai:skillsbench:overall","benchmarkName":"SkillsBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":51.274,"unit":"percent","family":"vals-ai:skillsbench:overall","index":63.30690837195295,"aboveFit":false,"run":"OpenHands","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/skillsbench","measuredAt":"2026-10-29","stderr":4.6,"costPerTest":0.397123,"latency":495.558},{"benchmarkId":"vals-ai:terminal-bench-2-1:overall","benchmarkName":"Terminal-Bench 2.1","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":50.187,"unit":"percent","family":"terminal-bench-2-1","index":50.493257384160344,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2-1","measuredAt":"2026-10-01","stderr":1.498,"costPerTest":0.239191,"latency":1078.158},{"benchmarkId":"vals-ai:vibe-code:overall","benchmarkName":"Vibe Code Bench v1.1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":49.931,"unit":"percent","family":"vals-ai:vibe-code:overall","index":60.26541186580667,"aboveFit":false,"run":"OpenHands","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","measuredAt":"2026-11-09","stderr":4.767,"costPerTest":2.205991,"latency":3418.92},{"benchmarkId":"epoch-ai:epoch-simpleqa-verified:overall","benchmarkName":"SimpleQA Verified","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":46.9939879759519,"unit":"percent","family":"epoch-ai:epoch-simpleqa-verified:overall","index":63.06919284155183,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.5806515332355318,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tiers-1-3-v2-private:overall","benchmarkName":"FrontierMath-Tiers-1-3-v2-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":45.26315789473684,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tiers-1-3-v2-private:overall","index":54.719036922001635,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.953609826992209,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:fabv2:overall","benchmarkName":"Finance Agent (v2)","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":44.083,"unit":"percent","family":"vals-ai:fabv2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/fabv2","measuredAt":"2026-10-29","stderr":0.653,"costPerTest":0.879086,"latency":1526.183},{"benchmarkId":"livebench:livebench-agentic-coding:overall","benchmarkName":"LiveBench Agentic Coding","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":42.626,"unit":"percent","family":"livebench:livebench-agentic-coding:overall","index":53.12215817021151,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Agentic%20Coding","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.14245833333333333,"latency":null},{"benchmarkId":"vals-ai:medcode:overall","benchmarkName":"MedCode","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":40.455,"unit":"percent","family":"vals-ai:medcode:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medcode","measuredAt":"2026-10-29","stderr":2.122,"costPerTest":0.06071,"latency":382.667},{"benchmarkId":"vals-ai:ioi-v1:overall","benchmarkName":"IOI v1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":35.833,"unit":"percent","family":"vals-ai:ioi-v1:overall","index":57.969350672127625,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/ioi-v1","measuredAt":"2026-08-09","stderr":7.139,"costPerTest":0.568564,"latency":5373.876},{"benchmarkId":"vals-ai:code-migration:overall","benchmarkName":"Code Migration","category":"coding","domain":null,"eligible":true,"ranked":true,"score":26.201,"unit":"percent","family":"vals-ai:code-migration:overall","index":59.10815913970547,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/code-migration","measuredAt":"2026-10-29","stderr":4.037,"costPerTest":1.071016,"latency":1737.978},{"benchmarkId":"vals-ai:tax_agent_bench:overall","benchmarkName":"Tax Agent Bench","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":25.17,"unit":"percent","family":"vals-ai:tax_agent_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench","measuredAt":"2026-11-09","stderr":3.058,"costPerTest":0.719152,"latency":1244.635},{"benchmarkId":"vals-ai:legal_research:overall","benchmarkName":"Legal Research Bench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":23.077,"unit":"percent","family":"vals-ai:legal_research:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_research","measuredAt":"2026-11-09","stderr":2.928,"costPerTest":0.682218,"latency":1024.619},{"benchmarkId":"epoch-ai:epoch-chess-puzzles:overall","benchmarkName":"Chess Puzzles","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":19,"unit":"percent","family":"epoch-ai:epoch-chess-puzzles:overall","index":48.40596443299076,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":3.9427724440366254,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-mystery-game-puzzles:overall","benchmarkName":"Mystery Game Puzzles","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":17,"unit":"percent","family":"epoch-ai:epoch-mystery-game-puzzles:overall","index":51.84605144605811,"aboveFit":false,"run":"none effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":3.7752516806863685,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:proof_bench:overall","benchmarkName":"ProofBench v1.1","category":"math","domain":null,"eligible":true,"ranked":true,"score":16,"unit":"percent","family":"vals-ai:proof_bench:overall","index":52.68053580654214,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/proof_bench","measuredAt":"2026-11-09","stderr":3.685,"costPerTest":0.142045,"latency":423.45},{"benchmarkId":"vals-ai:terminal-bench-4:overall","benchmarkName":"Terminal-Bench 4.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":11.111,"unit":"percent","family":"terminal-bench-4","index":59.791945499550636,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-4","measuredAt":"2026-11-09","stderr":1.336,"costPerTest":3.452522,"latency":5146.973},{"benchmarkId":"epoch-ai:epoch-frontiermath-tier-4-v2-private:overall","benchmarkName":"FrontierMath-Tier-4-v2-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":2.4390243902439024,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tier-4-v2-private:overall","index":49.446475323573715,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.4390243902439024,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:programbench:overall","benchmarkName":"ProgramBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":0,"unit":"percent","family":"vals-ai:programbench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/programbench","measuredAt":"2026-11-09","stderr":0,"costPerTest":null,"latency":3778.098}],"price":{"inputPerMillion":0.892272,"outputPerMillion":1.784544,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":89,"timeToFirstToken":0.715,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":{"elo":1451.4484068250058,"eloUpper":1455.3414260384097,"eloLower":1447.5553876116014,"votes":56906,"rank":50,"rankUpper":null,"rankLower":null,"arenaName":"DeepSeek-V4-Pro","organization":"deepseek","updatedAt":"2026-11-09"},"quality":{"coveragePercent":85,"benchmarkCount":24,"categoryCount":6,"confidence":"medium","state":"available"},"hosts":[{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":0.95526,"outputPerMillion":1.91052,"cacheReadPerMillion":0.079605,"contextTokens":1048576,"maxOutputTokens":384000,"deprecated":false},{"host":"deepinfra","name":"Deep Infra","kind":"host","url":"https://deepinfra.com/models","inputPerMillion":1.3,"outputPerMillion":2.6,"cacheReadPerMillion":0.1,"contextTokens":1048576,"maxOutputTokens":16384,"deprecated":false},{"host":"gmi-cloud","name":"GMI Cloud","kind":"host","url":"https://docs.gmicloud.ai/inference-engine/api-reference/llm-api-reference","inputPerMillion":1.392,"outputPerMillion":2.784,"cacheReadPerMillion":0.116,"contextTokens":1048576,"maxOutputTokens":384000,"deprecated":false},{"host":"siliconflow","name":"SiliconFlow","kind":"host","url":"https://cloud.siliconflow.com/models","inputPerMillion":1.50162,"outputPerMillion":3.135,"cacheReadPerMillion":0.135,"contextTokens":1000000,"maxOutputTokens":384000,"deprecated":false},{"host":"novita-ai","name":"Novita AI","kind":"host","url":"https://novita.ai/docs/guides/introduction","inputPerMillion":1.6,"outputPerMillion":3.2,"cacheReadPerMillion":0.135,"contextTokens":1048576,"maxOutputTokens":393216,"deprecated":false},{"host":"arcee-ai","name":"Arcee AI","kind":"host","url":"https://docs.arcee.ai","inputPerMillion":1.74,"outputPerMillion":3.48,"cacheReadPerMillion":0.2,"contextTokens":512000,"maxOutputTokens":384000,"deprecated":true},{"host":"digitalocean","name":"DigitalOcean","kind":"cloud","url":"https://docs.digitalocean.com/products/gradient-ai-platform/details/models/","inputPerMillion":1.74,"outputPerMillion":3.48,"cacheReadPerMillion":0.348,"contextTokens":1048576,"maxOutputTokens":384000,"deprecated":false},{"host":"azure","name":"Microsoft Azure","kind":"cloud","url":"https://learn.microsoft.com/en-us/azure/ai-services/openai/concepts/models","inputPerMillion":1.74,"outputPerMillion":3.48,"cacheReadPerMillion":null,"contextTokens":1000000,"maxOutputTokens":384000,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:livebench","name":"LiveBench","url":"https://livebench.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gemini3flashpreview","providerId":"google","slug":"google-gemini-3-flash-preview","name":"Gemini 3 Flash Preview","provider":"Google","description":"Gemini 3 Flash Preview is a model from Google. 14 benchmarks count toward its index, in 6 categories.","logoUrl":null,"releaseDate":"2025-12-17T15:57:58.000Z","contextWindowTokens":1048576,"modalities":["text","image","file","audio","video"],"openWeights":null,"details":{"apiId":"google/gemini-3-flash-preview","maxOutputTokens":65536,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.049999999999999996,"weightsUrl":null},"reasoning":null,"overallScore":57.52273357645463,"overallScoreRange":{"low":52.201119294730724,"high":62.84434785817853},"badges":[],"capabilityScores":{"agentic":57.331466625508504,"coding":54.98251237692561,"reasoning":60.26662370054204,"multimodal":69.19557182022811,"knowledge":56.952681237129724,"multilingual":null,"instruction":null,"math":57.29506683176784},"capabilityCounts":{"math":1,"knowledge":2,"multimodal":1,"coding":5,"reasoning":2,"agentic":3},"benchmarks":[{"benchmarkId":"vals-ai:medqa:overall","benchmarkName":"MedQA","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":95.808,"unit":"percent","family":"vals-ai:medqa:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medqa","measuredAt":"2026-04-16","stderr":0.183,"costPerTest":0.012699,"latency":26.324},{"benchmarkId":"vals-ai:aime:overall","benchmarkName":"AIME","category":"math","domain":null,"eligible":true,"ranked":true,"score":95.625,"unit":"percent","family":"vals-ai:aime:overall","index":57.29506683176784,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/aime","measuredAt":"2026-04-16","stderr":0.516,"costPerTest":0.046328,"latency":84.007},{"benchmarkId":"vals-ai:mgsm:overall","benchmarkName":"MGSM","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":93.309,"unit":"percent","family":"vals-ai:mgsm:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mgsm","measuredAt":"2026-01-09","stderr":1.456,"costPerTest":0.007116,"latency":14.02},{"benchmarkId":"vals-ai:mmlu_pro:overall","benchmarkName":"MMLU Pro","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":88.592,"unit":"percent","family":"mmlu-pro","index":56.09604423713413,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","measuredAt":"2026-09-01","stderr":0.315,"costPerTest":0.019364,"latency":36.249},{"benchmarkId":"arc-prize:v1-public-eval:overall","benchmarkName":"ARC-AGI-1 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":88.33,"unit":"percent","family":"arc-prize:v1-public-eval:overall","index":null,"aboveFit":false,"run":"high effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.1353,"latency":null},{"benchmarkId":"vals-ai:gpqa:overall","benchmarkName":"GPQA Diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":87.879,"unit":"percent","family":"gpqa-diamond","index":57.809318237125325,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","measuredAt":"2026-09-01","stderr":1.638,"costPerTest":0.044346,"latency":83.84},{"benchmarkId":"vals-ai:mmmu:overall","benchmarkName":"MMMU Pro","category":"multimodal","domain":null,"eligible":true,"ranked":true,"score":87.63,"unit":"percent","family":"mmmu-pro","index":69.19557182022811,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmmu","measuredAt":"2026-09-01","stderr":0.792,"costPerTest":0.014422,"latency":27.856},{"benchmarkId":"vals-ai:legal_bench:overall","benchmarkName":"LegalBench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":86.858,"unit":"percent","family":"vals-ai:legal_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","measuredAt":"2026-10-13","stderr":0.356,"costPerTest":0.002067,"latency":4.673},{"benchmarkId":"vals-ai:lcb:overall","benchmarkName":"LiveCodeBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":85.591,"unit":"percent","family":"livecodebench","index":60.362148035072224,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","measuredAt":"2026-09-01","stderr":1.004,"costPerTest":0.050525,"latency":91.931},{"benchmarkId":"arc-prize:v1-semi-private:overall","benchmarkName":"ARC-AGI-1 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":84.67,"unit":"percent","family":"arc-prize:v1-semi-private:overall","index":65.91321469903293,"aboveFit":false,"run":"high effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.1743,"latency":null},{"benchmarkId":"swebench-experiments:swe-bench-verified:overall","benchmarkName":"SWE-bench Verified","category":"coding","domain":null,"eligible":true,"ranked":true,"score":75.8,"unit":"percent","family":"swe-bench-verified","index":57.1347919254755,"aboveFit":false,"run":"high effort · mini-SWE-agent","source":"SWE-bench team","sourceUrl":"https://github.com/SWE-bench/experiments/tree/main/evaluation/verified","measuredAt":"2026-09-01","stderr":null,"costPerTest":0.35596413079999994,"latency":null},{"benchmarkId":"vals-ai:swebench:overall","benchmarkName":"SWE-bench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":75,"unit":"percent","family":"swe-bench-verified","index":56.544745863248544,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/swebench","measuredAt":"2026-09-01","stderr":1.938,"costPerTest":0.280231,"latency":285.223},{"benchmarkId":"vals-ai:tax_eval_v2:overall","benchmarkName":"TaxEval v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":73.876,"unit":"percent","family":"vals-ai:tax_eval_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","measuredAt":"2026-09-01","stderr":0.857,"costPerTest":0.005764,"latency":12.001},{"benchmarkId":"vals-ai:medscribe:overall","benchmarkName":"MedScribe","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":69.917,"unit":"percent","family":"vals-ai:medscribe:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medscribe","measuredAt":"2026-10-29","stderr":1.899,"costPerTest":0.014379,"latency":23.491},{"benchmarkId":"vals-ai:mortgage_tax:overall","benchmarkName":"MortgageTax","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":68.72,"unit":"percent","family":"vals-ai:mortgage_tax:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mortgage_tax","measuredAt":"2026-09-01","stderr":0.914,"costPerTest":0.007477,"latency":12.838},{"benchmarkId":"vals-ai:corp_fin_v2:overall","benchmarkName":"CorpFin v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":66.434,"unit":"percent","family":"vals-ai:corp_fin_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","measuredAt":"2026-08-12","stderr":0.93,"costPerTest":0.047154,"latency":11.279},{"benchmarkId":"vals-ai:terminal-bench:overall","benchmarkName":"Terminal-Bench 1.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":60,"unit":"percent","family":"vals-ai:terminal-bench:overall","index":61.638889535073446,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench","measuredAt":"2026-01-12","stderr":5.477,"costPerTest":0.125536,"latency":118.432},{"benchmarkId":"vals-ai:medcode:overall","benchmarkName":"MedCode","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":55.92,"unit":"percent","family":"vals-ai:medcode:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medcode","measuredAt":"2026-10-29","stderr":2.112,"costPerTest":0.006187,"latency":44.149},{"benchmarkId":"vals-ai:case_law_v2:overall","benchmarkName":"CaseLaw v2","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":55.842,"unit":"percent","family":"vals-ai:case_law_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/case_law_v2","measuredAt":"2026-05-04","stderr":0.107,"costPerTest":0.025399,"latency":260.502},{"benchmarkId":"vals-ai:terminal-bench-2-1:overall","benchmarkName":"Terminal-Bench 2.1","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":53.933,"unit":"percent","family":"terminal-bench-2-1","index":52.684125579476124,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2-1","measuredAt":"2026-10-01","stderr":1.297,"costPerTest":0.154312,"latency":376.158},{"benchmarkId":"vals-ai:terminal-bench-2:overall","benchmarkName":"Terminal-Bench 2.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":51.685,"unit":"percent","family":"vals-ai:terminal-bench-2:overall","index":57.67138476197594,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2","measuredAt":"2026-06-04","stderr":5.327,"costPerTest":0.168702,"latency":456.071},{"benchmarkId":"vals-ai:finance_agent:overall","benchmarkName":"Finance Agent v1.1","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":47.598,"unit":"percent","family":"vals-ai:finance_agent:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/finance_agent","measuredAt":"2026-06-04","stderr":2.784,"costPerTest":0.369606,"latency":876.089},{"benchmarkId":"vals-ai:fabv2:overall","benchmarkName":"Finance Agent (v2)","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":42.551,"unit":"percent","family":"vals-ai:fabv2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/fabv2","measuredAt":"2026-10-29","stderr":0.434,"costPerTest":0.570657,"latency":246.543},{"benchmarkId":"vals-ai:ioi-v1:overall","benchmarkName":"IOI v1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":39.084,"unit":"percent","family":"vals-ai:ioi-v1:overall","index":59.586908852104756,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/ioi-v1","measuredAt":"2026-08-09","stderr":10.545,"costPerTest":0.375052,"latency":1846.526},{"benchmarkId":"arc-prize:v2-public-eval:overall","benchmarkName":"ARC-AGI-2 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":34.03,"unit":"percent","family":"arc-prize:v2-public-eval:overall","index":null,"aboveFit":false,"run":"high effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.2399,"latency":null},{"benchmarkId":"arc-prize:v2-semi-private:overall","benchmarkName":"ARC-AGI-2 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":33.61,"unit":"percent","family":"arc-prize:v2-semi-private:overall","index":54.62003270205115,"aboveFit":false,"run":"high effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.2314,"latency":null},{"benchmarkId":"vals-ai:emb:overall","benchmarkName":"EMB","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":26.017,"unit":"percent","family":"vals-ai:emb:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/emb","measuredAt":"2026-10-20","stderr":2.912,"costPerTest":0.453183,"latency":328.69},{"benchmarkId":"vals-ai:vibe-code:overall","benchmarkName":"Vibe Code Bench v1.1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":20.204,"unit":"percent","family":"vals-ai:vibe-code:overall","index":49.53944321960846,"aboveFit":false,"run":"OpenHands","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","measuredAt":"2026-11-09","stderr":3.946,"costPerTest":0.942067,"latency":806.66},{"benchmarkId":"vals-ai:legal_research:overall","benchmarkName":"Legal Research Bench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":18.269,"unit":"percent","family":"vals-ai:legal_research:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_research","measuredAt":"2026-11-09","stderr":2.686,"costPerTest":0.135395,"latency":187.7},{"benchmarkId":"vals-ai:tax_agent_bench:overall","benchmarkName":"Tax Agent Bench","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":11.843,"unit":"percent","family":"vals-ai:tax_agent_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench","measuredAt":"2026-11-09","stderr":2.324,"costPerTest":0.100699,"latency":100.623},{"benchmarkId":"vals-ai:code-migration:overall","benchmarkName":"Code Migration","category":"coding","domain":null,"eligible":true,"ranked":true,"score":6.391,"unit":"percent","family":"vals-ai:code-migration:overall","index":48.584292883480586,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/code-migration","measuredAt":"2026-10-29","stderr":1.093,"costPerTest":0.340087,"latency":284.234},{"benchmarkId":"vals-ai:programbench:overall","benchmarkName":"ProgramBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":0,"unit":"percent","family":"vals-ai:programbench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/programbench","measuredAt":"2026-11-09","stderr":0,"costPerTest":0.439965,"latency":418.92}],"price":{"inputPerMillion":0.5,"outputPerMillion":3,"batch":{"inputPerMillion":0.25,"outputPerMillion":1.5},"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":78,"timeToFirstToken":1.291,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":90,"benchmarkCount":14,"categoryCount":6,"confidence":"medium","state":"available"},"hosts":[{"host":"databricks","name":"Databricks","kind":"cloud","url":"https://docs.databricks.com/aws/en/machine-learning/foundation-models/","inputPerMillion":0.5,"outputPerMillion":3,"cacheReadPerMillion":0.05,"contextTokens":1048576,"maxOutputTokens":65536,"deprecated":false},{"host":"google-vertex","name":"Google Vertex AI","kind":"maker","url":"https://cloud.google.com/vertex-ai/generative-ai/docs/models","inputPerMillion":0.5,"outputPerMillion":3,"cacheReadPerMillion":0.05,"contextTokens":1048576,"maxOutputTokens":65536,"deprecated":false},{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":0.5,"outputPerMillion":3,"cacheReadPerMillion":0.05,"contextTokens":1048576,"maxOutputTokens":65536,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:arc-prize-foundation","name":"ARC Prize Foundation","url":"https://arcprize.org","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:swe-bench-team","name":"SWE-bench team","url":"https://github.com","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"seed-2-1-turbo","providerId":"bytedance","slug":"seed-2-1-turbo","name":"Seed 2.1 Turbo","provider":"ByteDance","description":"Seed 2.1 Turbo is a reasoning model from ByteDance in the Seed 2.1 family. 9 benchmarks count toward its index, in 4 categories.","logoUrl":null,"releaseDate":"2026-06-24","contextWindowTokens":262144,"modalities":["text","image","video"],"openWeights":false,"details":{"apiId":"bytedance-seed/seed-2-1-turbo","maxOutputTokens":235929,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":null,"weightsUrl":null},"reasoning":true,"overallScore":57.38535636931209,"overallScoreRange":{"low":48.49535636931209,"high":66.27535636931209},"badges":[],"capabilityScores":{"agentic":60.6773438457283,"coding":57.59310172280772,"reasoning":null,"multimodal":59.27548646279114,"knowledge":52.425952328475255,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"multimodal":6,"agentic":1,"knowledge":1,"coding":1},"benchmarks":[{"benchmarkId":"mathVision","benchmarkName":"MathVision","category":"multimodal","score":90.1,"unit":"percent","family":"mathVision","index":60.984790100202055,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"videoMme","benchmarkName":"Video-MME","category":"multimodal","score":89,"unit":"percent","family":"videoMme","index":null,"aboveFit":false,"run":null,"source":"Video-MME benchmark team","sourceUrl":"https://mme-benchmark.github.io/","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"charxiv","benchmarkName":"CharXiv Reasoning","category":"multimodal","score":82.5,"unit":"percent","family":"charxiv","index":58.110321140649404,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmmuPro","benchmarkName":"Massive Multi-discipline Multimodal Understanding Pro","category":"multimodal","score":80.1,"unit":"percent","family":"mmmu-pro","index":59.3959374638808,"aboveFit":false,"run":null,"source":"MMMU-Pro authors","sourceUrl":"https://arxiv.org/abs/2409.02813","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"erqa","benchmarkName":"ERQA","category":"multimodal","score":71.3,"unit":"percent","family":"erqa","index":64.1892991795636,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"terminalBench21","benchmarkName":"Terminal-Bench 2.1","category":"agentic","score":67.6,"unit":"percent","family":"terminal-bench-2-1","index":60.6773438457283,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"superGpqa","benchmarkName":"SuperGPQA: Scaling LLM Evaluation Across 285 Graduate Disciplines","category":"knowledge","score":67.4,"unit":"percent","family":"superGpqa","index":52.425952328475255,"aboveFit":false,"run":null,"source":"Xiaoxuan Du, Yao Yao, Kexin Ma, Bowen Wang, Tianyu Zheng, Kaiyan Zhu, Yiming Zhang, Yutao Zhu, Jiawei Zhou, Jingren Zhou","sourceUrl":"https://arxiv.org/abs/2502.14739","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"babyVision","benchmarkName":"BabyVision","category":"multimodal","score":62.9,"unit":"percent","family":"babyVision","index":58.93628276503171,"aboveFit":false,"run":null,"source":"Meta AI","sourceUrl":"https://ai.meta.com/static-resource/muse-spark-1-1-evaluation-report","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"nl2Repo","benchmarkName":"NL2Repo","category":"coding","score":43.7,"unit":"percent","family":"nl2Repo","index":57.59310172280772,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"zeroBench","benchmarkName":"ZeroBench","category":"multimodal","score":11,"unit":"percent","family":"zeroBench","index":54.036288127419276,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":0.5,"outputPerMillion":2.5,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":37,"timeToFirstToken":2.639,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":65,"benchmarkCount":9,"categoryCount":4,"confidence":"low","state":"available"},"hosts":[{"host":"volcengine","name":"Volcengine Ark","kind":"maker","url":"https://www.volcengine.com/docs/82379/1330310","inputPerMillion":0.4453,"outputPerMillion":2.22651,"cacheReadPerMillion":0.08906,"contextTokens":256000,"maxOutputTokens":256000,"deprecated":false},{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":0.5,"outputPerMillion":2.5,"cacheReadPerMillion":null,"contextTokens":262144,"maxOutputTokens":235929,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"qwen3-6-max-preview","providerId":"alibaba","slug":"qwen3-6-max-preview","name":"Qwen 3.6 Max (preview)","provider":"Alibaba","description":"Qwen 3.6 Max (preview) is a reasoning model from Alibaba in the Qwen 3.6 Max family. 21 benchmarks count toward its index, in 6 categories.","logoUrl":null,"releaseDate":"2026-04-20","contextWindowTokens":262144,"modalities":["text"],"openWeights":false,"details":{"apiId":"qwen/qwen3.6-max-preview","maxOutputTokens":65536,"knowledgeCutoff":null,"retiresAt":"2026-10-09","tools":true,"structuredOutput":true,"cacheReadPerMillion":null,"weightsUrl":null},"reasoning":true,"overallScore":57.1584398181424,"overallScoreRange":{"low":52.1298565330264,"high":62.1870231032584},"badges":[],"capabilityScores":{"agentic":59.21454313243861,"coding":57.030494248754025,"reasoning":53.357171132653576,"multimodal":null,"knowledge":59.32858913806946,"multilingual":null,"instruction":60.60714248800483,"math":53.22358076764165},"capabilityCounts":{"agentic":5,"math":3,"knowledge":5,"reasoning":3,"coding":4,"instruction":1},"benchmarks":[{"benchmarkId":"tau2Bench","benchmarkName":"τ²-Bench Tool-Agent-User Evaluation","category":"agentic","score":95.9,"unit":"percent","family":"tau2-bench","index":66.42610157862087,"aboveFit":false,"run":null,"source":"Victor Barres, Honghua Dong, Soham Ray, Xujie Si, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2506.07982","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":91.11111111111111,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":61.9269928211348,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":4.290254662948546,"costPerTest":null,"latency":null},{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":88.8,"unit":"percent","family":"gpqa-diamond","index":58.65403427072644,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":87.37373737373737,"unit":"percent","family":"gpqa-diamond","index":57.34590516265903,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.3664359402880177,"costPerTest":null,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":80.7,"unit":"percent","family":"lcr","index":62.664747873933486,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-swe-bench-verified:overall","benchmarkName":"SWE-Bench verified","category":"coding","domain":null,"eligible":true,"ranked":true,"score":76.65289256198346,"unit":"percent","family":"swe-bench-verified","index":57.763849297601745,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.924895909142592,"costPerTest":null,"latency":null},{"benchmarkId":"aaIfBench","benchmarkName":"Artificial Analysis IFBench","category":"instruction","score":76.6,"unit":"percent","family":"ifbench-family","index":60.60714248800483,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"superGpqa","benchmarkName":"SuperGPQA: Scaling LLM Evaluation Across 285 Graduate Disciplines","category":"knowledge","score":73.9,"unit":"percent","family":"superGpqa","index":57.65828103983498,"aboveFit":false,"run":null,"source":"Xiaoxuan Du, Yao Yao, Kexin Ma, Bowen Wang, Tianyu Zheng, Kaiyan Zhu, Yiming Zhang, Yutao Zhu, Jiawei Zhou, Jingren Zhou","sourceUrl":"https://arxiv.org/abs/2502.14739","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:swebench:overall","benchmarkName":"SWE-bench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":72.8,"unit":"percent","family":"swe-bench-verified","index":54.92211919212442,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/swebench","measuredAt":"2026-09-01","stderr":1.992,"costPerTest":1.565093,"latency":1079.228},{"benchmarkId":"vals-ai:corp_fin_v2:overall","benchmarkName":"CorpFin v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":66.473,"unit":"percent","family":"vals-ai:corp_fin_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","measuredAt":"2026-08-12","stderr":0.929,"costPerTest":0.156029,"latency":47.068},{"benchmarkId":"terminalBench2","benchmarkName":"Terminal-Bench 2.0","category":"agentic","score":65.4,"unit":"percent","family":"terminalBench2","index":60.41963400764559,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"qwenClawBench","benchmarkName":"QwenClawBench","category":"agentic","score":59,"unit":"percent","family":"qwenClawBench","index":56.44273979468295,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"swePro","benchmarkName":"SWE-bench Pro","category":"coding","score":57.3,"unit":"percent","family":"swePro","index":57.63663559598494,"aboveFit":false,"run":null,"source":"Xiang Deng, Jeff Da, Edwin Pan, Yannis Yiming He, Charles Ide, Kanak Garg, Niklas Lauffer, Andrew Park, Nitin Pasari, Chetan Rane, Karmini Sampath, Maya Krishnan, Srivatsa Kundurthy, Sean Hendryx, Zifan Wang, Chen Bo Calvin Zhang, Noah Jacobson, Bing Liu, Brad Kenstler","sourceUrl":"https://arxiv.org/abs/2509.16941","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:finance_agent:overall","benchmarkName":"Finance Agent v1.1","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":52.785,"unit":"percent","family":"vals-ai:finance_agent:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/finance_agent","measuredAt":"2026-06-04","stderr":2.87,"costPerTest":0.588068,"latency":1838.57},{"benchmarkId":"epoch-ai:epoch-simpleqa-verified:overall","benchmarkName":"SimpleQA Verified","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":52.00400801603207,"unit":"percent","family":"epoch-ai:epoch-simpleqa-verified:overall","index":67.24298181110746,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.5822434816352415,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:terminal-bench-2:overall","benchmarkName":"Terminal-Bench 2.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":51.685,"unit":"percent","family":"vals-ai:terminal-bench-2:overall","index":57.67138476197594,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2","measuredAt":"2026-06-04","stderr":5.327,"costPerTest":0.320393,"latency":897.845},{"benchmarkId":"epoch-ai:epoch-frontiermath-2025-02-28-public:overall","benchmarkName":"FrontierMath-2025-02-28-Public","category":"math","domain":null,"eligible":false,"ranked":true,"score":50,"unit":"percent","family":"epoch-ai:epoch-frontiermath-2025-02-28-public:overall","index":null,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":16.666666666666664,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:case_law_v2:overall","benchmarkName":"CaseLaw v2","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":47.91,"unit":"percent","family":"vals-ai:case_law_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/case_law_v2","measuredAt":"2026-05-04","stderr":1.114,"costPerTest":0.05235,"latency":42.93},{"benchmarkId":"sciCode","benchmarkName":"Scientific Code Benchmark","category":"coding","score":47,"unit":"percent","family":"scicode-family","index":57.11328232400372,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"terminalBenchHard","benchmarkName":"Terminal-Bench Hard","category":"agentic","score":43.9,"unit":"percent","family":"terminalBenchHard","index":55.11285551926768,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/terminalbench-hard","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"nl2Repo","benchmarkName":"NL2Repo","category":"coding","score":42.9,"unit":"percent","family":"nl2Repo","index":57.02907483016435,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"omniscienceAccuracy","benchmarkName":"Artificial Analysis Omniscience Accuracy","category":"knowledge","score":37.9,"unit":"percent","family":"omniscienceAccuracy","index":58.45365226248101,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":30.8,"unit":"percent","family":"hle","index":55.28806086023114,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-2025-02-28-private:overall","benchmarkName":"FrontierMath-2025-02-28-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":23.103448275862068,"unit":"percent","family":"epoch-ai:epoch-frontiermath-2025-02-28-private:overall","index":50.30938058707686,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.479378972186167,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-mystery-game-puzzles:overall","benchmarkName":"Mystery Game Puzzles","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":19,"unit":"percent","family":"epoch-ai:epoch-mystery-game-puzzles:overall","index":53.39488123946846,"aboveFit":false,"run":"none effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":3.9427724440366254,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-private:overall","benchmarkName":"FrontierMath-Tier-4-2025-07-01-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":4.166666666666666,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-private:overall","index":47.43436889471329,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.914766351555603,"costPerTest":null,"latency":null},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":3.7,"unit":"percent","family":"critpt","index":44.01188428455879,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-public:overall","benchmarkName":"FrontierMath-Tier-4-2025-07-01-Public","category":"math","domain":null,"eligible":false,"ranked":true,"score":0,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-public:overall","index":null,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":1.0270000000000001,"outputPerMillion":6.162,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":75,"timeToFirstToken":0.93,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":{"elo":1446.2114178778434,"eloUpper":1454.401996068953,"eloLower":1438.0208396867338,"votes":5386,"rank":60,"rankUpper":null,"rankLower":null,"arenaName":"Qwen3.6-Max-Preview","organization":"alibaba","updatedAt":"2026-11-09"},"quality":{"coveragePercent":85,"benchmarkCount":21,"categoryCount":6,"confidence":"medium","state":"available"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"claude-sonnet-4-6","providerId":"anthropic","slug":"claude-sonnet-4-6","name":"Claude Sonnet 4.6","provider":"Anthropic","description":"Claude Sonnet 4.6 is a standard model from Anthropic. 42 benchmarks count toward its index, in 8 categories.","logoUrl":null,"releaseDate":"2026-02-01","contextWindowTokens":1000000,"modalities":["text","image","file"],"openWeights":false,"details":{"apiId":"anthropic/claude-sonnet-4.6","maxOutputTokens":128000,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.3,"weightsUrl":null},"reasoning":false,"overallScore":57.02208976503762,"overallScoreRange":{"low":54.52700459615059,"high":59.51717493392465},"badges":[],"capabilityScores":{"agentic":59.74518064874162,"coding":58.891787520274036,"reasoning":56.37580883798786,"multimodal":53.89263427033051,"knowledge":57.22701665479872,"multilingual":56.60199585314037,"instruction":27.525466723040424,"math":54.069549410811376},"capabilityCounts":{"knowledge":6,"math":4,"reasoning":7,"multimodal":2,"coding":8,"agentic":13,"multilingual":1,"instruction":1},"benchmarks":[{"benchmarkId":"arc-prize:v1-public-eval:overall","benchmarkName":"ARC-AGI-1 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":95.75,"unit":"percent","family":"arc-prize:v1-public-eval:overall","index":null,"aboveFit":false,"run":"max effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":1.0867,"latency":null},{"benchmarkId":"superGpqa","benchmarkName":"SuperGPQA: Scaling LLM Evaluation Across 285 Graduate Disciplines","category":"knowledge","score":95,"unit":"percent","family":"superGpqa","index":74.64322501055656,"aboveFit":true,"run":null,"source":"Xiaoxuan Du, Yao Yao, Kexin Ma, Bowen Wang, Tianyu Zheng, Kaiyan Zhu, Yiming Zhang, Yutao Zhu, Jiawei Zhou, Jingren Zhou","sourceUrl":"https://arxiv.org/abs/2502.14739","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:aime:overall","benchmarkName":"AIME","category":"math","domain":null,"eligible":true,"ranked":true,"score":92.292,"unit":"percent","family":"vals-ai:aime:overall","index":55.83839331223886,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/aime","measuredAt":"2026-04-16","stderr":0.438,"costPerTest":0.436012,"latency":328.034},{"benchmarkId":"vals-ai:medqa:overall","benchmarkName":"MedQA","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":92.058,"unit":"percent","family":"vals-ai:medqa:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medqa","measuredAt":"2026-04-16","stderr":0.247,"costPerTest":0.035415,"latency":35.119},{"benchmarkId":"gpqa","benchmarkName":"Graduate-Level Google-Proof Q&A","category":"knowledge","score":89.9,"unit":"percent","family":"gpqa-diamond","index":59.66292421313821,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:mmlu_pro:overall","benchmarkName":"MMLU Pro","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":87.341,"unit":"percent","family":"mmlu-pro","index":54.51993397798496,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","measuredAt":"2026-09-01","stderr":0.434,"costPerTest":0.066412,"latency":55.406},{"benchmarkId":"livebench:livebench-reasoning:overall","benchmarkName":"LiveBench Reasoning","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":86.4,"unit":"percent","family":"livebench","index":73.5452186024819,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Reasoning","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.31303091584158416,"latency":null},{"benchmarkId":"arc-prize:v1-semi-private:overall","benchmarkName":"ARC-AGI-1 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":86,"unit":"percent","family":"arc-prize:v1-semi-private:overall","index":66.52466371598888,"aboveFit":false,"run":"max effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":1.452,"latency":null},{"benchmarkId":"vals-ai:gpqa:overall","benchmarkName":"GPQA Diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":85.606,"unit":"percent","family":"gpqa-diamond","index":55.724584747032644,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","measuredAt":"2026-09-01","stderr":2.406,"costPerTest":0.408666,"latency":377.864},{"benchmarkId":"vals-ai:mmmu:overall","benchmarkName":"MMMU Pro","category":"multimodal","domain":null,"eligible":true,"ranked":true,"score":83.584,"unit":"percent","family":"mmmu-pro","index":63.93005779555598,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmmu","measuredAt":"2026-09-01","stderr":0.891,"costPerTest":0.144122,"latency":131.728},{"benchmarkId":"vals-ai:legal_bench:overall","benchmarkName":"LegalBench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":82.12,"unit":"percent","family":"vals-ai:legal_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","measuredAt":"2026-10-13","stderr":0.476,"costPerTest":0.004202,"latency":21.628},{"benchmarkId":"vals-ai:lcb:overall","benchmarkName":"LiveCodeBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":82.091,"unit":"percent","family":"livecodebench","index":57.458053733562636,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","measuredAt":"2026-09-01","stderr":1.063,"costPerTest":0.370575,"latency":295.009},{"benchmarkId":"reactNativeEvals","benchmarkName":"React Native Evals","category":"coding","score":80.6,"unit":"percent","family":"reactNativeEvals","index":57.427094413842696,"aboveFit":false,"run":null,"source":"Callstack","sourceUrl":"https://rn-evals.vercel.app/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-2025-02-28-public:overall","benchmarkName":"FrontierMath-2025-02-28-Public","category":"math","domain":null,"eligible":false,"ranked":true,"score":80,"unit":"percent","family":"epoch-ai:epoch-frontiermath-2025-02-28-public:overall","index":null,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":13.3,"costPerTest":null,"latency":null},{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":79.9,"unit":"percent","family":"gpqa-diamond","index":50.49119746394036,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"sweVerified","benchmarkName":"Software Engineering Benchmark Verified","category":"coding","score":79.6,"unit":"percent","family":"swe-bench-verified","index":59.93751072105352,"aboveFit":false,"run":null,"source":"Carlos E. Jimenez, John Yang, Alexander Wettig, Shunyu Yao, Kexin Pei, Ofir Press, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2310.06770","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"tau2Bench","benchmarkName":"τ²-Bench Tool-Agent-User Evaluation","category":"agentic","score":79.5,"unit":"percent","family":"tau2-bench","index":55.54350254310654,"aboveFit":false,"run":null,"source":"Victor Barres, Honghua Dong, Soham Ray, Xujie Si, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2506.07982","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmluPro","benchmarkName":"Massive Multitask Language Understanding Professional","category":"knowledge","score":79.2,"unit":"percent","family":"mmlu-pro","index":44.26324843063615,"aboveFit":false,"run":null,"source":"Yubo Wang, Xueguang Ma, Ge Zhang, Yuansheng Ni, Abhranil Chandra, Shiguang Guo, Weiming Ren, Aaran Arulraj, Xuan He, Ziyan Jiang, Tianle Li, Max Ku, Kai Wang, Alex Zhuang, Rongqi Fan, Xiang Yue, Wenhu Chen","sourceUrl":"https://arxiv.org/abs/2406.01574","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":78.78787878787878,"unit":"percent","family":"gpqa-diamond","index":49.4711902769841,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.9126522834586774,"costPerTest":null,"latency":null},{"benchmarkId":"charxiv","benchmarkName":"CharXiv Reasoning","category":"multimodal","score":77.4,"unit":"percent","family":"charxiv","index":52.303979170258664,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:swebench:overall","benchmarkName":"SWE-bench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":77.4,"unit":"percent","family":"swe-bench-verified","index":58.314884049929404,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/swebench","measuredAt":"2026-09-01","stderr":1.872,"costPerTest":1.303853,"latency":511.785},{"benchmarkId":"vals-ai:tax_eval_v2:overall","benchmarkName":"TaxEval v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":77.106,"unit":"percent","family":"vals-ai:tax_eval_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","measuredAt":"2026-09-01","stderr":0.818,"costPerTest":0.127973,"latency":127.879},{"benchmarkId":"livebench:livebench-data-analysis:overall","benchmarkName":"LiveBench Data Analysis","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":76.1,"unit":"percent","family":"livebench","index":60.20224844217235,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Data%20Analysis","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.41688178,"latency":null},{"benchmarkId":"epoch-ai:epoch-swe-bench-verified:overall","benchmarkName":"SWE-Bench verified","category":"coding","domain":null,"eligible":true,"ranked":true,"score":75.20661157024794,"unit":"percent","family":"swe-bench-verified","index":56.6971337925427,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.9648182701390322,"costPerTest":null,"latency":null},{"benchmarkId":"osWorldVerified","benchmarkName":"OSWorld-Verified","category":"agentic","score":72.1,"unit":"percent","family":"osWorldVerified","index":58.68424471194735,"aboveFit":false,"run":null,"source":"Tianbao Xie, Danyang Zhang, Jixuan Chen, Xiaochuan Li, Siheng Zhao, Ruisheng Cao, Toh Jing Hua, Zhoujun Cheng, Dongchan Shin, Fangyu Lei, Yitao Liu, Yiheng Xu, Shuyan Zhou, Silvio Savarese, Caiming Xiong, Victor Zhong, Tao Yu","sourceUrl":"https://os-world.github.io/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-design:overall","benchmarkName":"LiveBench Design","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":71.8,"unit":"percent","family":"livebench-2-agentic","index":71.09904419765901,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Design","measuredAt":"2026-10-14","stderr":null,"costPerTest":2.138588,"latency":null},{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":71.11111111111111,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":50.291111906473084,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":6.832943242540508,"costPerTest":null,"latency":null},{"benchmarkId":"aaMmmuPro","benchmarkName":"Artificial Analysis MMMU-Pro","category":"multimodal","score":70.6,"unit":"percent","family":"mmmu-pro","index":47.032520945248734,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmmu-pro","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-french:overall","benchmarkName":"EuroEval French","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":68.47125000000001,"unit":"percent","family":"euroeval","index":65.11996692273279,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/french","measuredAt":null,"stderr":0.38038837061613756,"costPerTest":null,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":68.3,"unit":"percent","family":"lcr","index":54.15734935667129,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-browser:overall","benchmarkName":"LiveBench Browser Use","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":68,"unit":"percent","family":"livebench-2-agentic","index":68.31998603696127,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Browser","measuredAt":"2026-10-14","stderr":null,"costPerTest":3.375256,"latency":null},{"benchmarkId":"clawEval","benchmarkName":"Claw-Eval","category":"agentic","score":67.8,"unit":"percent","family":"clawEval","index":63.16568629631021,"aboveFit":false,"run":null,"source":"Bowen Ye, Rang Li, Qibin Yang, Yuanxin Liu, Linli Yao, Hanglong Lv, Zhihui Xie, Chenxin An, Lei Li, Lingpeng Kong, Qi Liu, Zhifang Sui, Tong Yang","sourceUrl":"https://arxiv.org/abs/2604.06132","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:mortgage_tax:overall","benchmarkName":"MortgageTax","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":67.726,"unit":"percent","family":"vals-ai:mortgage_tax:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mortgage_tax","measuredAt":"2026-09-01","stderr":0.921,"costPerTest":0.04467,"latency":29.74},{"benchmarkId":"euroeval:euroeval-portuguese:overall","benchmarkName":"EuroEval Portuguese","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":66.98,"unit":"percent","family":"euroeval","index":63.0966837995822,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/portuguese","measuredAt":null,"stderr":0.5091199514849128,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-coding-2:overall","benchmarkName":"LiveBench Coding 2.0","category":"coding","domain":null,"eligible":true,"ranked":true,"score":66.6,"unit":"percent","family":"livebench:livebench-coding-2:overall","index":56.41196894042153,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Coding","measuredAt":"2026-10-14","stderr":null,"costPerTest":2.628662068965517,"latency":null},{"benchmarkId":"vals-ai:corp_fin_v2:overall","benchmarkName":"CorpFin v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":65.307,"unit":"percent","family":"vals-ai:corp_fin_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","measuredAt":"2026-08-12","stderr":0.938,"costPerTest":0.307405,"latency":18.483},{"benchmarkId":"euroeval:euroeval-italian:overall","benchmarkName":"EuroEval Italian","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":65.28625,"unit":"percent","family":"euroeval","index":60.798654770354176,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/italian","measuredAt":null,"stderr":0.48060346180609226,"costPerTest":null,"latency":null},{"benchmarkId":"cyberGym","benchmarkName":"CyberGym","category":"agentic","score":65.2,"unit":"percent","family":"cyberGym","index":57.55669247170988,"aboveFit":false,"run":null,"source":"Zhun Wang, Tianneng Shi, Jingxuan He, Matthew Cai, Jialin Zhang, Dawn Song","sourceUrl":"https://www.cybergym.io/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":64.50625000000001,"unit":"percent","family":"euroeval","index":59.74037424324107,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":0.7353453695373351,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:case_law_v2:overall","benchmarkName":"CaseLaw v2","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":63.987,"unit":"percent","family":"vals-ai:case_law_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/case_law_v2","measuredAt":"2026-05-04","stderr":0.557,"costPerTest":0.144942,"latency":36.196},{"benchmarkId":"livebench:livebench-research:overall","benchmarkName":"LiveBench Research","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":63.6,"unit":"percent","family":"livebench-2-agentic","index":65.10212921931127,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Research","measuredAt":"2026-10-14","stderr":null,"costPerTest":3.901772727272727,"latency":null},{"benchmarkId":"vals-ai:finance_agent:overall","benchmarkName":"Finance Agent v1.1","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":63.331,"unit":"percent","family":"vals-ai:finance_agent:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/finance_agent","measuredAt":"2026-06-04","stderr":2.841,"costPerTest":1.436698,"latency":348.995},{"benchmarkId":"gertLabs","benchmarkName":"Gert Labs Composite Game Benchmark","category":"agentic","score":62.92,"unit":"percent","family":"gertLabs","index":66.49413709579824,"aboveFit":false,"run":null,"source":"Gert Labs","sourceUrl":"https://gertlabs.com/rankings","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:public-benefits-bench:overall","benchmarkName":"Public Benefits Bench v1.1","category":null,"domain":"Public benefits","eligible":false,"ranked":true,"score":62.449,"unit":"percent","family":"vals-ai:public-benefits-bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/public-benefits-bench","measuredAt":"2026-10-12","stderr":1.26,"costPerTest":1.105228,"latency":882.995},{"benchmarkId":"arc-prize:v2-public-eval:overall","benchmarkName":"ARC-AGI-2 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":62.36000000000001,"unit":"percent","family":"arc-prize:v2-public-eval:overall","index":null,"aboveFit":false,"run":"max effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":2.8905,"latency":null},{"benchmarkId":"sweRebench","benchmarkName":"SWE-Rebench","category":"coding","score":60.7,"unit":"percent","family":"sweRebench","index":null,"aboveFit":false,"run":null,"source":"Nebius","sourceUrl":"https://swe-rebench.com","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:emb:overall","benchmarkName":"EMB","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":60.154,"unit":"percent","family":"vals-ai:emb:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/emb","measuredAt":"2026-10-20","stderr":3.276,"costPerTest":7.22787,"latency":3255.853},{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":59.88000000000001,"unit":"percent","family":"euroeval","index":53.463617463039675,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.47877181305878047,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-dutch:overall","benchmarkName":"EuroEval Dutch","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":59.71,"unit":"percent","family":"euroeval","index":53.232966578925264,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/dutch","measuredAt":null,"stderr":0.4257672812438478,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:terminal-bench-2:overall","benchmarkName":"Terminal-Bench 2.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":59.551,"unit":"percent","family":"vals-ai:terminal-bench-2:overall","index":62.40759189859753,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2","measuredAt":"2026-06-04","stderr":5.232,"costPerTest":0.518966,"latency":615.5},{"benchmarkId":"terminalBench2","benchmarkName":"Terminal-Bench 2.0","category":"agentic","score":59.1,"unit":"percent","family":"terminalBench2","index":55.938157355309386,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-spanish:overall","benchmarkName":"EuroEval Spanish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":58.5325,"unit":"percent","family":"euroeval","index":51.63537001395641,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/spanish","measuredAt":null,"stderr":0.4538464084908021,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:public-benefits-bench-v1:overall","benchmarkName":"Public Benefits Bench v1","category":null,"domain":"Public benefits","eligible":false,"ranked":true,"score":58.525,"unit":"percent","family":"vals-ai:public-benefits-bench-v1:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/public-benefits-bench-v1","measuredAt":"2026-06-09","stderr":1.282,"costPerTest":0.816309,"latency":787.612},{"benchmarkId":"arc-prize:v2-semi-private:overall","benchmarkName":"ARC-AGI-2 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":58.330000000000005,"unit":"percent","family":"arc-prize:v2-semi-private:overall","index":65.96984251018173,"aboveFit":false,"run":"max effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":2.7179,"latency":null},{"benchmarkId":"vals-ai:terminal-bench-2-1:overall","benchmarkName":"Terminal-Bench 2.1","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":57.303,"unit":"percent","family":"terminal-bench-2-1","index":54.65508815775005,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2-1","measuredAt":"2026-10-01","stderr":0.649,"costPerTest":0.572392,"latency":704.225},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":54.785000000000004,"unit":"percent","family":"euroeval","index":46.55087478914048,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.5314249711859615,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:vibe-code:overall","benchmarkName":"Vibe Code Bench v1.1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":51.476,"unit":"percent","family":"vals-ai:vibe-code:overall","index":60.82287213957719,"aboveFit":false,"run":"OpenHands","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","measuredAt":"2026-11-09","stderr":4.645,"costPerTest":5.907892,"latency":1572.118},{"benchmarkId":"vals-ai:fabv2:overall","benchmarkName":"Finance Agent (v2)","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":51.035,"unit":"percent","family":"vals-ai:fabv2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/fabv2","measuredAt":"2026-10-29","stderr":0.329,"costPerTest":2.409559,"latency":695.588},{"benchmarkId":"vals-ai:skillsbench:overall","benchmarkName":"SkillsBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":49.052,"unit":"percent","family":"vals-ai:skillsbench:overall","index":61.65313561547145,"aboveFit":false,"run":"OpenHands","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/skillsbench","measuredAt":"2026-10-29","stderr":4.64,"costPerTest":1.382228,"latency":860.178},{"benchmarkId":"hle","benchmarkName":"Humanity's Last Exam","category":"knowledge","score":49,"unit":"percent","family":"hle","index":73.5479888197734,"aboveFit":false,"run":null,"source":"Center for AI Safety, Scale AI, and thousands of expert contributors","sourceUrl":"https://lastexam.ai/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"cursorBench31","benchmarkName":"cursorBench31","category":"coding","score":48.8,"unit":"percent","family":"cursorBench31","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaIfBench","benchmarkName":"Artificial Analysis IFBench","category":"instruction","score":41.2,"unit":"percent","family":"ifbench-family","index":27.525466723040424,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:code-migration:overall","benchmarkName":"Code Migration","category":"coding","domain":null,"eligible":true,"ranked":true,"score":39.892,"unit":"percent","family":"vals-ai:code-migration:overall","index":66.38136726257144,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/code-migration","measuredAt":"2026-10-29","stderr":4.138,"costPerTest":38.507642,"latency":5979.033},{"benchmarkId":"omniscienceAccuracy","benchmarkName":"Artificial Analysis Omniscience Accuracy","category":"knowledge","score":38.6,"unit":"percent","family":"omniscienceAccuracy","index":59.33583145132269,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:legal_research:overall","benchmarkName":"Legal Research Bench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":38.462,"unit":"percent","family":"vals-ai:legal_research:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_research","measuredAt":"2026-11-09","stderr":3.381,"costPerTest":2.302948,"latency":989.604},{"benchmarkId":"jobBench","benchmarkName":"JobBench","category":"agentic","score":36.9,"unit":"percent","family":"jobBench","index":56.902949918823836,"aboveFit":false,"run":null,"source":"Yuetai Li, Yichen Feng, Zhangchen Xu, Zixian Ma, Kaiyuan Zheng, Fengqing Jiang, Xinghua Sun, Rulin Shao, Zichen Chen, Yue Huang, Xinyang Han, Brian Lee, Kayla Xu, Shenglai Zeng, Hang Hua, Xiangliang Zhang, Basel Alomair, Ranjay Krishna, Luke Zettlemoyer, Pang Wei Koh, Bhaskar Ramasubramanian, Luyao Niu, Xiang Yue, Radha Poovendran","sourceUrl":"https://arxiv.org/abs/2605.26329","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-simpleqa-verified:overall","benchmarkName":"SimpleQA Verified","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":32.800000000000004,"unit":"percent","family":"epoch-ai:epoch-simpleqa-verified:overall","index":51.24434783612443,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.4853842487270312,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-2025-02-28-private:overall","benchmarkName":"FrontierMath-2025-02-28-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":32.4,"unit":"percent","family":"epoch-ai:epoch-frontiermath-2025-02-28-private:overall","index":58.51616845161297,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.812163604063986,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:tax_agent_bench:overall","benchmarkName":"Tax Agent Bench","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":25.84,"unit":"percent","family":"vals-ai:tax_agent_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench","measuredAt":"2026-11-09","stderr":3.154,"costPerTest":1.193501,"latency":707.307},{"benchmarkId":"frontierCode","benchmarkName":"FrontierCode 1.1 Main","category":"coding","score":24.3,"unit":"percent","family":"frontierCode","index":52.6649240068159,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-long-horizon:overall","benchmarkName":"LiveBench Long Agentic","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":23.1,"unit":"percent","family":"livebench-2-agentic","index":35.48321987503276,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Long%20Agentic","measuredAt":"2026-10-14","stderr":null,"costPerTest":20.41961111111111,"latency":null},{"benchmarkId":"epoch-ai:epoch-chess-puzzles:overall","benchmarkName":"Chess Puzzles","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":23,"unit":"percent","family":"epoch-ai:epoch-chess-puzzles:overall","index":51.442025213591144,"aboveFit":false,"run":"max effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":4.229525846816507,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-mystery-game-puzzles:overall","benchmarkName":"Mystery Game Puzzles","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":16,"unit":"percent","family":"epoch-ai:epoch-mystery-game-puzzles:overall","index":51.071636549352945,"aboveFit":false,"run":"low effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":3.684529491774706,"costPerTest":null,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":13.3,"unit":"percent","family":"hle","index":37.730437822209744,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"osWorld2","benchmarkName":"OSWorld 2.0","category":"agentic","score":8.3,"unit":"percent","family":"osWorld2","index":57.427094413842696,"aboveFit":false,"run":null,"source":"Mengqi Yuan, Zilong Zhou, Xinzhuang Xiong, Weiming Wu, Jiayang Sun, Jiamin Song, Kaiqian Cui, Bowen Wang, Haoyuan Wu, Yitong Li, Dunjie Lu, Haikong Lu, Qi Zhen, Xinyuan Wang, Jiaqi Deng, Yuhao Yang, Cheng Chen, Boyuan Zheng, Alex Su, Xiao Yu, Hao Zou, Saaket Agashe, Xing Han Lu, Manpreet Kaur, Zhengyang Qi, Vincent Sunn Chen, Frederic Sala, Dayiheng Liu, Junyang Lin, Zhou Yu, Yu Su, Siva Reddy, Xin Eric Wang, Peng Qi, Tianbao Xie, Tao Yu","sourceUrl":"https://arxiv.org/abs/2606.29537","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-private:overall","benchmarkName":"FrontierMath-Tier-4-2025-07-01-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":8.3,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-private:overall","index":51.63252397292058,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":4,"costPerTest":null,"latency":null},{"benchmarkId":"apprenticeBench","benchmarkName":"ApprenticeBench: end-to-end computer use, continual learning, and long-horizon agency on a real accounts-payable job","category":"agentic","score":2,"unit":"percent","family":"apprenticeBench","index":59.22247859396124,"aboveFit":false,"run":null,"source":"NeoCognition","sourceUrl":"https://neocognition.io/blog/apprentice-bench/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":0.9,"unit":"percent","family":"critpt","index":38.5914109978019,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:programbench:overall","benchmarkName":"ProgramBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":0.5,"unit":"percent","family":"vals-ai:programbench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/programbench","measuredAt":"2026-11-09","stderr":0.5,"costPerTest":null,"latency":4349.841},{"benchmarkId":"mazeBench","benchmarkName":"MazeBench: visual spatial reasoning in a 3D open world","category":"agentic","score":0,"unit":"percent","family":"mazeBench","index":61.97866734834783,"aboveFit":false,"run":null,"source":"Jonathan Pappas and David Pappas","sourceUrl":"https://mazebench.com/blog?post=maze-bench-results","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-public:overall","benchmarkName":"FrontierMath-Tier-4-2025-07-01-Public","category":"math","domain":null,"eligible":false,"ranked":true,"score":0,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-public:overall","index":null,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":3,"outputPerMillion":15,"batch":{"inputPerMillion":1.5,"outputPerMillion":7.5},"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":44,"timeToFirstToken":0.608,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":{"elo":1457.7744544346779,"eloUpper":1461.2809504906522,"eloLower":1454.2679583787035,"votes":70017,"rank":43,"rankUpper":null,"rankLower":null,"arenaName":"Claude Sonnet 4.6","organization":"anthropic","updatedAt":"2026-11-09"},"quality":{"coveragePercent":100,"benchmarkCount":42,"categoryCount":8,"confidence":"high","state":"available"},"hosts":[{"host":"amazon-bedrock","name":"Amazon Bedrock","kind":"cloud","url":"https://docs.aws.amazon.com/bedrock/latest/userguide/models-supported.html","inputPerMillion":3,"outputPerMillion":15,"cacheReadPerMillion":0.3,"contextTokens":1000000,"maxOutputTokens":128000,"deprecated":false},{"host":"databricks","name":"Databricks","kind":"cloud","url":"https://docs.databricks.com/aws/en/machine-learning/foundation-models/","inputPerMillion":3,"outputPerMillion":15,"cacheReadPerMillion":0.3,"contextTokens":1000000,"maxOutputTokens":64000,"deprecated":false},{"host":"digitalocean","name":"DigitalOcean","kind":"cloud","url":"https://docs.digitalocean.com/products/gradient-ai-platform/details/models/","inputPerMillion":3,"outputPerMillion":15,"cacheReadPerMillion":0.3,"contextTokens":200000,"maxOutputTokens":8192,"deprecated":false},{"host":"gmi-cloud","name":"GMI Cloud","kind":"host","url":"https://docs.gmicloud.ai/inference-engine/api-reference/llm-api-reference","inputPerMillion":3,"outputPerMillion":15,"cacheReadPerMillion":0.3,"contextTokens":409600,"maxOutputTokens":64000,"deprecated":false},{"host":"google-vertex","name":"Google Vertex AI","kind":"cloud","url":"https://cloud.google.com/vertex-ai/generative-ai/docs/models","inputPerMillion":3,"outputPerMillion":15,"cacheReadPerMillion":0.3,"contextTokens":1000000,"maxOutputTokens":128000,"deprecated":false},{"host":"azure","name":"Microsoft Azure","kind":"cloud","url":"https://learn.microsoft.com/en-us/azure/ai-services/openai/concepts/models","inputPerMillion":3,"outputPerMillion":15,"cacheReadPerMillion":0.3,"contextTokens":1000000,"maxOutputTokens":64000,"deprecated":false},{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":3,"outputPerMillion":15,"cacheReadPerMillion":0.3,"contextTokens":1000000,"maxOutputTokens":128000,"deprecated":false},{"host":"sap-ai-core","name":"SAP AI Core","kind":"cloud","url":"https://help.sap.com/docs/sap-ai-core","inputPerMillion":3,"outputPerMillion":15,"cacheReadPerMillion":0.3,"contextTokens":1000000,"maxOutputTokens":64000,"deprecated":false},{"host":"snowflake","name":"Snowflake Cortex","kind":"cloud","url":"https://docs.snowflake.com/en/user-guide/snowflake-cortex/cortex-rest-api","inputPerMillion":3,"outputPerMillion":15,"cacheReadPerMillion":0.3,"contextTokens":1000000,"maxOutputTokens":64000,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:livebench","name":"LiveBench","url":"https://livebench.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:arc-prize-foundation","name":"ARC Prize Foundation","url":"https://arcprize.org","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"mimo-v2-5-pro","providerId":"xiaomi","slug":"mimo-v2-5-pro","name":"MiMo-V2.5-Pro","provider":"Xiaomi","description":"MiMo-V2.5-Pro is a reasoning model from Xiaomi in the MiMo-V2.5 family. 27 benchmarks count toward its index, in 6 categories.","logoUrl":null,"releaseDate":"2026-04-22","contextWindowTokens":1024000,"modalities":["text"],"openWeights":true,"details":{"apiId":"xiaomi/mimo-v2.5-pro","maxOutputTokens":235929,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.096,"weightsUrl":"https://huggingface.co/XiaomiMiMo/MiMo-V2.5-Pro"},"reasoning":true,"overallScore":56.347074912978364,"overallScoreRange":{"low":51.73515696204628,"high":60.958992863910446},"badges":[],"capabilityScores":{"agentic":58.211043296242586,"coding":57.201812812193225,"reasoning":53.28565831395814,"multimodal":null,"knowledge":51.24377136253388,"multilingual":null,"instruction":63.6910275169422,"math":54.75200401249156},"capabilityCounts":{"agentic":13,"knowledge":4,"coding":6,"instruction":1,"reasoning":2,"math":1},"benchmarks":[{"benchmarkId":"tau2Bench","benchmarkName":"τ²-Bench Tool-Agent-User Evaluation","category":"agentic","score":94.2,"unit":"percent","family":"tau2-bench","index":65.29802728835415,"aboveFit":false,"run":null,"source":"Victor Barres, Honghua Dong, Soham Ray, Xujie Si, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2506.07982","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":86.6,"unit":"percent","family":"gpqa-diamond","index":56.63625438590291,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:mmlu_pro:overall","benchmarkName":"MMLU Pro","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":84.595,"unit":"percent","family":"mmlu-pro","index":51.060302665735875,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","measuredAt":"2026-09-01","stderr":0.453,"costPerTest":0.002085,"latency":67.757},{"benchmarkId":"vals-ai:medscribe:overall","benchmarkName":"MedScribe","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":83.73,"unit":"percent","family":"vals-ai:medscribe:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medscribe","measuredAt":"2026-10-29","stderr":2.063,"costPerTest":0.00603,"latency":90.71},{"benchmarkId":"vals-ai:gpqa:overall","benchmarkName":"GPQA Diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":82.576,"unit":"percent","family":"gpqa-diamond","index":52.94555154202569,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","measuredAt":"2026-09-01","stderr":2.468,"costPerTest":0.006654,"latency":251.332},{"benchmarkId":"vals-ai:lcb:overall","benchmarkName":"LiveCodeBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":81.354,"unit":"percent","family":"livecodebench","index":56.84653444778762,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","measuredAt":"2026-09-01","stderr":1.069,"costPerTest":0.006808,"latency":271.074},{"benchmarkId":"aaIfBench","benchmarkName":"Artificial Analysis IFBench","category":"instruction","score":79.9,"unit":"percent","family":"ifbench-family","index":63.6910275169422,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":79.7,"unit":"percent","family":"lcr","index":61.97866734834783,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:legal_bench:overall","benchmarkName":"LegalBench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":77.129,"unit":"percent","family":"vals-ai:legal_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","measuredAt":"2026-10-13","stderr":0.55,"costPerTest":0.000845,"latency":27.142},{"benchmarkId":"vals-ai:swebench:overall","benchmarkName":"SWE-bench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":74,"unit":"percent","family":"swe-bench-verified","index":55.807188285464846,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/swebench","measuredAt":"2026-09-01","stderr":1.964,"costPerTest":0.022968,"latency":470.814},{"benchmarkId":"vals-ai:tax_eval_v2:overall","benchmarkName":"TaxEval v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":73.794,"unit":"percent","family":"vals-ai:tax_eval_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","measuredAt":"2026-09-01","stderr":0.863,"costPerTest":0.002547,"latency":62.829},{"benchmarkId":"tau3Bench","benchmarkName":"τ³-Bench Tool-Agent-User Evaluation","category":"agentic","score":72.9,"unit":"percent","family":"tau3Bench","index":63.329436292779334,"aboveFit":false,"run":null,"source":"Sierra Research","sourceUrl":"https://github.com/sierra-research/tau2-bench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"terminalBench2","benchmarkName":"Terminal-Bench 2.0","category":"agentic","score":68.4,"unit":"percent","family":"terminalBench2","index":62.55367050875807,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"clawEval","benchmarkName":"Claw-Eval","category":"agentic","score":63.8,"unit":"percent","family":"clawEval","index":57.37721715640845,"aboveFit":false,"run":null,"source":"Bowen Ye, Rang Li, Qibin Yang, Yuanxin Liu, Linli Yao, Hanglong Lv, Zhihui Xie, Chenxin An, Lei Li, Lingpeng Kong, Qi Liu, Zhifang Sui, Tong Yang","sourceUrl":"https://arxiv.org/abs/2604.06132","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gertLabs","benchmarkName":"Gert Labs Composite Game Benchmark","category":"agentic","score":62.7,"unit":"percent","family":"gertLabs","index":66.3159994027788,"aboveFit":false,"run":null,"source":"Gert Labs","sourceUrl":"https://gertlabs.com/rankings","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:corp_fin_v2:overall","benchmarkName":"CorpFin v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":61.422,"unit":"percent","family":"vals-ai:corp_fin_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","measuredAt":"2026-08-12","stderr":0.959,"costPerTest":0.037238,"latency":18.899},{"benchmarkId":"vals-ai:terminal-bench-2-1:overall","benchmarkName":"Terminal-Bench 2.1","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":57.303,"unit":"percent","family":"terminal-bench-2-1","index":54.65508815775005,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2-1","measuredAt":"2026-10-01","stderr":5.273,"costPerTest":0.043613,"latency":866.253},{"benchmarkId":"swePro","benchmarkName":"SWE-bench Pro","category":"coding","score":57.2,"unit":"percent","family":"swePro","index":57.551474597866154,"aboveFit":false,"run":null,"source":"Xiang Deng, Jeff Da, Edwin Pan, Yannis Yiming He, Charles Ide, Kanak Garg, Niklas Lauffer, Andrew Park, Nitin Pasari, Chetan Rane, Karmini Sampath, Maya Krishnan, Srivatsa Kundurthy, Sean Hendryx, Zifan Wang, Chen Bo Calvin Zhang, Noah Jacobson, Bing Liu, Brad Kenstler","sourceUrl":"https://arxiv.org/abs/2509.16941","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:emb:overall","benchmarkName":"EMB","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":55.228,"unit":"percent","family":"vals-ai:emb:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/emb","measuredAt":"2026-10-20","stderr":3.112,"costPerTest":0.221264,"latency":1747.98},{"benchmarkId":"aaSciCode","benchmarkName":"Artificial Analysis SciCode","category":"coding","score":50.6,"unit":"percent","family":"scicode-family","index":61.80657045109676,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/scicode","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hle","benchmarkName":"Humanity's Last Exam","category":"knowledge","score":48,"unit":"percent","family":"hle","index":72.5446960747436,"aboveFit":false,"run":null,"source":"Center for AI Safety, Scale AI, and thousands of expert contributors","sourceUrl":"https://lastexam.ai/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"terminalBenchHard","benchmarkName":"Terminal-Bench Hard","category":"agentic","score":43.2,"unit":"percent","family":"terminalBenchHard","index":54.5377851211709,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/terminalbench-hard","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:fabv2:overall","benchmarkName":"Finance Agent (v2)","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":41.502,"unit":"percent","family":"vals-ai:fabv2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/fabv2","measuredAt":"2026-10-29","stderr":1.193,"costPerTest":0.206445,"latency":497.373},{"benchmarkId":"aaItbench","benchmarkName":"Artificial Analysis ITBench-AA","category":"agentic","score":38.2,"unit":"percent","family":"aaItbench","index":55.65822961558741,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/itbench-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":35.7,"unit":"percent","family":"hle","index":60.20419531087714,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:vibe-code:overall","benchmarkName":"Vibe Code Bench v1.1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":34.113,"unit":"percent","family":"vals-ai:vibe-code:overall","index":54.55802894638785,"aboveFit":false,"run":"OpenHands","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","measuredAt":"2026-11-09","stderr":4.529,"costPerTest":0.166747,"latency":2403.764},{"benchmarkId":"hleNoTools","benchmarkName":"Humanity's Last Exam without tools","category":"knowledge","score":34,"unit":"percent","family":"hle","index":58.49859764432648,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:medcode:overall","benchmarkName":"MedCode","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":32.484,"unit":"percent","family":"vals-ai:medcode:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medcode","measuredAt":"2026-10-29","stderr":1.907,"costPerTest":0.006719,"latency":38.448},{"benchmarkId":"gdpvalAaNormalized","benchmarkName":"GDPval-AA normalized","category":"agentic","score":31,"unit":"percent","family":"gdpvalAaNormalized","index":56.19805452963958,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"omniscienceAccuracy","benchmarkName":"Artificial Analysis Omniscience Accuracy","category":"knowledge","score":22.4,"unit":"percent","family":"omniscienceAccuracy","index":38.919684509558195,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:proof_bench:overall","benchmarkName":"ProofBench v1.1","category":"math","domain":null,"eligible":true,"ranked":true,"score":22,"unit":"percent","family":"vals-ai:proof_bench:overall","index":54.75200401249156,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/proof_bench","measuredAt":"2026-11-09","stderr":4.163,"costPerTest":0.089853,"latency":1177.352},{"benchmarkId":"vals-ai:code-migration:overall","benchmarkName":"Code Migration","category":"coding","domain":null,"eligible":true,"ranked":true,"score":21.557,"unit":"percent","family":"vals-ai:code-migration:overall","index":56.641080144556135,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/code-migration","measuredAt":"2026-10-29","stderr":4.127,"costPerTest":0.2383,"latency":2932.892},{"benchmarkId":"vals-ai:legal_research:overall","benchmarkName":"Legal Research Bench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":15.865,"unit":"percent","family":"vals-ai:legal_research:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_research","measuredAt":"2026-11-09","stderr":2.539,"costPerTest":0.082648,"latency":416.437},{"benchmarkId":"vals-ai:tax_agent_bench:overall","benchmarkName":"Tax Agent Bench","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":11.503,"unit":"percent","family":"vals-ai:tax_agent_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench","measuredAt":"2026-11-09","stderr":1.957,"costPerTest":0.040042,"latency":376.514},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":4,"unit":"percent","family":"critpt","index":44.592649279568455,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"apexAgentsAa","benchmarkName":"APEX-Agents-AA","category":"agentic","score":2.4,"unit":"percent","family":"apexAgentsAa","index":43.72510625261542,"aboveFit":false,"run":null,"source":"Artificial Analysis / Mercor","sourceUrl":"https://artificialanalysis.ai/evaluations/apex-agents-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:terminal-bench-4:overall","benchmarkName":"Terminal-Bench 4.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":0.505,"unit":"percent","family":"terminal-bench-4","index":54.755806974722255,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-4","measuredAt":"2026-11-09","stderr":0.505,"costPerTest":0.23303,"latency":7622.037},{"benchmarkId":"mazeBench","benchmarkName":"MazeBench: visual spatial reasoning in a 3D open world","category":"agentic","score":0,"unit":"percent","family":"mazeBench","index":61.97866734834783,"aboveFit":false,"run":null,"source":"Jonathan Pappas and David Pappas","sourceUrl":"https://mazebench.com/blog?post=maze-bench-results","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lmarena:agent-bash-recovery:overall","benchmarkName":"Agent Arena command recovery","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":-7.505049308821143,"unit":"index","family":"agent-arena","index":60.36047420224139,"aboveFit":false,"run":null,"source":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset","measuredAt":"2026-10-08","stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":0.48,"outputPerMillion":1.7999999999999998,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":47,"timeToFirstToken":1.561,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":{"elo":1464.0935084668895,"eloUpper":1467.6075025379887,"eloLower":1460.5795143957903,"votes":76793,"rank":38,"rankUpper":null,"rankLower":null,"arenaName":"MiMo-V2.5-Pro","organization":"xiaomi","updatedAt":"2026-11-09"},"quality":{"coveragePercent":85,"benchmarkCount":27,"categoryCount":6,"confidence":"medium","state":"available"},"hosts":[{"host":"xiaomi","name":"Xiaomi","kind":"maker","url":"https://platform.xiaomimimo.com/#/docs","inputPerMillion":0.435,"outputPerMillion":0.87,"cacheReadPerMillion":0.0036,"contextTokens":1048576,"maxOutputTokens":131072,"deprecated":true},{"host":"novita-ai","name":"Novita AI","kind":"host","url":"https://novita.ai/docs/guides/introduction","inputPerMillion":0.522,"outputPerMillion":1.044,"cacheReadPerMillion":0.0043,"contextTokens":1048576,"maxOutputTokens":131072,"deprecated":false},{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":0.48,"outputPerMillion":1.8,"cacheReadPerMillion":0.096,"contextTokens":1024000,"maxOutputTokens":235929,"deprecated":true},{"host":"digitalocean","name":"DigitalOcean","kind":"cloud","url":"https://docs.digitalocean.com/products/gradient-ai-platform/details/models/","inputPerMillion":0.8,"outputPerMillion":3,"cacheReadPerMillion":0.16,"contextTokens":262144,"maxOutputTokens":262144,"deprecated":false},{"host":"deepinfra","name":"Deep Infra","kind":"host","url":"https://deepinfra.com/models","inputPerMillion":1,"outputPerMillion":3,"cacheReadPerMillion":0.2,"contextTokens":1048576,"maxOutputTokens":16384,"deprecated":true}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:lmarena","name":"LMArena","url":"https://huggingface.co","license":"CC BY 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"claude-opus-4-5-thinking","providerId":"anthropic","slug":"claude-opus-4-5-thinking","name":"Claude Opus 4.5 Thinking","provider":"Anthropic","description":"Claude Opus 4.5 Thinking is a reasoning model from Anthropic in the Claude Opus 4.5 family. 19 benchmarks count toward its index, in 7 categories.","logoUrl":null,"releaseDate":"2025-11-24","contextWindowTokens":200000,"modalities":["text"],"openWeights":false,"details":null,"reasoning":true,"overallScore":56.270516702257304,"overallScoreRange":{"low":52.06086428557032,"high":60.48016911894429},"badges":[],"capabilityScores":{"agentic":59.673360441255326,"coding":54.419404169988056,"reasoning":54.17565253963818,"multimodal":57.27984112079703,"knowledge":59.03225669630906,"multilingual":null,"instruction":43.22524505217608,"math":57.20416131359742},"capabilityCounts":{"math":1,"agentic":4,"knowledge":4,"coding":4,"multimodal":1,"reasoning":4,"instruction":1},"benchmarks":[{"benchmarkId":"vals-ai:medqa:overall","benchmarkName":"MedQA","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":95.875,"unit":"percent","family":"vals-ai:medqa:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medqa","measuredAt":"2026-04-16","stderr":0.182,"costPerTest":0.14813,"latency":31.625},{"benchmarkId":"vals-ai:aime:overall","benchmarkName":"AIME","category":"math","domain":null,"eligible":true,"ranked":true,"score":95.417,"unit":"percent","family":"vals-ai:aime:overall","index":57.20416131359742,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/aime","measuredAt":"2026-04-16","stderr":0.369,"costPerTest":1.320246,"latency":212.201},{"benchmarkId":"vals-ai:mgsm:overall","benchmarkName":"MGSM","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":95.2,"unit":"percent","family":"vals-ai:mgsm:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mgsm","measuredAt":"2026-01-09","stderr":1.284,"costPerTest":0.063151,"latency":11.56},{"benchmarkId":"aaGlobalMmluLite","benchmarkName":"Artificial Analysis Global-MMLU-Lite","category":"multilingual","score":91.3,"unit":"percent","family":"aaGlobalMmluLite","index":null,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/global-mmlu-lite","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaAime2025","benchmarkName":"Artificial Analysis AIME 2025","category":"math","score":91.3,"unit":"percent","family":"aaAime2025","index":null,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/aime-2025","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"tau2Bench","benchmarkName":"τ²-Bench Tool-Agent-User Evaluation","category":"agentic","score":89.5,"unit":"percent","family":"tau2-bench","index":62.179233662322595,"aboveFit":false,"run":null,"source":"Victor Barres, Honghua Dong, Soham Ray, Xujie Si, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2506.07982","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaMmluPro","benchmarkName":"Artificial Analysis MMLU-Pro","category":"knowledge","score":89.5,"unit":"percent","family":"mmlu-pro","index":57.24001555232793,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmlu-pro","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:mmlu_pro:overall","benchmarkName":"MMLU Pro","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":87.26,"unit":"percent","family":"mmlu-pro","index":54.417883673435746,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","measuredAt":"2026-09-01","stderr":0.381,"costPerTest":0.215138,"latency":38.92},{"benchmarkId":"aaLiveCodeBench","benchmarkName":"Artificial Analysis LiveCodeBench","category":"coding","score":87.1,"unit":"percent","family":"livecodebench","index":61.614227549637356,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/livecodebench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":86.6,"unit":"percent","family":"gpqa-diamond","index":56.63625438590291,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:gpqa:overall","benchmarkName":"GPQA Diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":85.859,"unit":"percent","family":"gpqa-diamond","index":55.95662943378735,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","measuredAt":"2026-09-01","stderr":2.476,"costPerTest":0.712758,"latency":138.525},{"benchmarkId":"vals-ai:medscribe:overall","benchmarkName":"MedScribe","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":85.321,"unit":"percent","family":"vals-ai:medscribe:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medscribe","measuredAt":"2026-10-29","stderr":1.896,"costPerTest":0.410224,"latency":72.112},{"benchmarkId":"vals-ai:legal_bench:overall","benchmarkName":"LegalBench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":84.604,"unit":"percent","family":"vals-ai:legal_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","measuredAt":"2026-10-13","stderr":0.391,"costPerTest":0.037097,"latency":7.638},{"benchmarkId":"vals-ai:lcb:overall","benchmarkName":"LiveCodeBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":83.67,"unit":"percent","family":"livecodebench","index":58.76821513415796,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","measuredAt":"2026-09-01","stderr":1.037,"costPerTest":0.971,"latency":164.427},{"benchmarkId":"vals-ai:mmmu:overall","benchmarkName":"MMMU Pro","category":"multimodal","domain":null,"eligible":true,"ranked":true,"score":82.948,"unit":"percent","family":"mmmu-pro","index":63.10235959494018,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmmu","measuredAt":"2026-09-01","stderr":0.904,"costPerTest":0.126986,"latency":75.28},{"benchmarkId":"arc-prize:v1-public-eval:overall","benchmarkName":"ARC-AGI-1 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":81.61,"unit":"percent","family":"arc-prize:v1-public-eval:overall","index":null,"aboveFit":false,"run":"16k effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.52,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":77.3,"unit":"percent","family":"lcr","index":60.332074086942235,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:swebench:overall","benchmarkName":"SWE-bench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":76.4,"unit":"percent","family":"swe-bench-verified","index":57.577326472145714,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/swebench","measuredAt":"2026-09-01","stderr":1.901,"costPerTest":1.054837,"latency":320.542},{"benchmarkId":"vals-ai:tax_eval_v2:overall","benchmarkName":"TaxEval v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":74.856,"unit":"percent","family":"vals-ai:tax_eval_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","measuredAt":"2026-09-01","stderr":0.849,"costPerTest":0.218987,"latency":47.448},{"benchmarkId":"aaMmmuPro","benchmarkName":"Artificial Analysis MMMU-Pro","category":"multimodal","score":74,"unit":"percent","family":"mmmu-pro","index":51.4573226466539,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmmu-pro","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"arc-prize:v1-semi-private:overall","benchmarkName":"ARC-AGI-1 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":73.91499999999999,"unit":"percent","family":"arc-prize:v1-semi-private:overall","index":60.968752911543305,"aboveFit":false,"run":"16k effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.62,"latency":null},{"benchmarkId":"vals-ai:mortgage_tax:overall","benchmarkName":"MortgageTax","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":67.686,"unit":"percent","family":"vals-ai:mortgage_tax:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mortgage_tax","measuredAt":"2026-09-01","stderr":0.917,"costPerTest":0.178388,"latency":28.166},{"benchmarkId":"vals-ai:corp_fin_v2:overall","benchmarkName":"CorpFin v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":65.074,"unit":"percent","family":"vals-ai:corp_fin_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","measuredAt":"2026-08-12","stderr":0.939,"costPerTest":0.173423,"latency":17.725},{"benchmarkId":"vals-ai:case_law_v2:overall","benchmarkName":"CaseLaw v2","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":62.594,"unit":"percent","family":"vals-ai:case_law_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/case_law_v2","measuredAt":"2026-05-04","stderr":0.107,"costPerTest":0.229072,"latency":16.819},{"benchmarkId":"vals-ai:finance_agent:overall","benchmarkName":"Finance Agent v1.1","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":58.81,"unit":"percent","family":"vals-ai:finance_agent:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/finance_agent","measuredAt":"2026-06-04","stderr":2.812,"costPerTest":1.501497,"latency":181.87},{"benchmarkId":"aaIfBench","benchmarkName":"Artificial Analysis IFBench","category":"instruction","score":58,"unit":"percent","family":"ifbench-family","index":43.22524505217608,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:terminal-bench:overall","benchmarkName":"Terminal-Bench 1.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":57.5,"unit":"percent","family":"vals-ai:terminal-bench:overall","index":59.82968138441703,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench","measuredAt":"2026-01-12","stderr":5.527,"costPerTest":0.960696,"latency":309.102},{"benchmarkId":"vals-ai:terminal-bench-2:overall","benchmarkName":"Terminal-Bench 2.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":53.933,"unit":"percent","family":"vals-ai:terminal-bench-2:overall","index":59.024930864585315,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2","measuredAt":"2026-06-04","stderr":5.314,"costPerTest":1.122133,"latency":661.913},{"benchmarkId":"vals-ai:medcode:overall","benchmarkName":"MedCode","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":49.156,"unit":"percent","family":"vals-ai:medcode:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medcode","measuredAt":"2026-10-29","stderr":2.012,"costPerTest":0.095846,"latency":60.839},{"benchmarkId":"terminalBenchHard","benchmarkName":"Terminal-Bench Hard","category":"agentic","score":47,"unit":"percent","family":"terminalBenchHard","index":57.65959585369635,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/terminalbench-hard","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"omniscienceAccuracy","benchmarkName":"Artificial Analysis Omniscience Accuracy","category":"knowledge","score":46.6,"unit":"percent","family":"omniscienceAccuracy","index":69.41787932379899,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":30.1,"unit":"percent","family":"hle","index":54.585755938710285,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"arc-prize:v2-public-eval:overall","benchmarkName":"ARC-AGI-2 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":24.169999999999998,"unit":"percent","family":"arc-prize:v2-public-eval:overall","index":null,"aboveFit":false,"run":"16k effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.82,"latency":null},{"benchmarkId":"arc-prize:v2-semi-private:overall","benchmarkName":"ARC-AGI-2 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":22.78,"unit":"percent","family":"arc-prize:v2-semi-private:overall","index":49.64760389047938,"aboveFit":false,"run":"16k effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.79,"latency":null},{"benchmarkId":"vals-ai:vibe-code:overall","benchmarkName":"Vibe Code Bench v1.1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":20.63,"unit":"percent","family":"vals-ai:vibe-code:overall","index":49.69315071257043,"aboveFit":false,"run":"OpenHands","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","measuredAt":"2026-11-09","stderr":3.159,"costPerTest":32.873283,"latency":2283.142},{"benchmarkId":"vals-ai:ioi-v1:overall","benchmarkName":"IOI v1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":20.25,"unit":"percent","family":"vals-ai:ioi-v1:overall","index":50.215918153338436,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/ioi-v1","measuredAt":"2026-08-09","stderr":5.157,"costPerTest":3.9235,"latency":8211.492},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":4.6,"unit":"percent","family":"critpt","index":45.75417926958779,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":95,"benchmarkCount":19,"categoryCount":7,"confidence":"medium","state":"available"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:arc-prize-foundation","name":"ARC Prize Foundation","url":"https://arcprize.org","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"glm-5-1","providerId":"z-ai","slug":"glm-5-1","name":"GLM-5.1","provider":"Z.AI","description":"GLM-5.1 is a reasoning model from Z.AI in the GLM-5 family. 38 benchmarks count toward its index, in 6 categories.","logoUrl":null,"releaseDate":"2026-04-07","contextWindowTokens":204800,"modalities":["text"],"openWeights":true,"details":{"apiId":"z-ai/glm-5.1","maxOutputTokens":131072,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.26,"weightsUrl":"https://huggingface.co/zai-org/GLM-5.1"},"reasoning":true,"overallScore":55.972585654363954,"overallScoreRange":{"low":51.849465274679524,"high":60.09570603404838},"badges":[],"capabilityScores":{"agentic":59.20101239705854,"coding":56.95108321882735,"reasoning":50.9271143641875,"multimodal":null,"knowledge":51.66963452313214,"multilingual":null,"instruction":60.326789303555984,"math":54.88234422141211},"capabilityCounts":{"agentic":13,"math":8,"knowledge":5,"coding":8,"instruction":1,"reasoning":3},"benchmarks":[{"benchmarkId":"tau2Bench","benchmarkName":"τ²-Bench Tool-Agent-User Evaluation","category":"agentic","score":97.7,"unit":"percent","family":"tau2-bench","index":67.62053318007978,"aboveFit":false,"run":null,"source":"Victor Barres, Honghua Dong, Soham Ray, Xujie Si, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2506.07982","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aime2026","benchmarkName":"AIME 2026","category":"math","score":95.3,"unit":"percent","family":"aime2026","index":53.901308033618,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hmmtNov2025","benchmarkName":"Harvard-MIT Mathematics Tournament November 2025","category":"math","score":94,"unit":"percent","family":"hmmtNov2025","index":55.728423671587684,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":93.33333333333333,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":63.219868478319434,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":3.760507165451774,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:aime:overall","benchmarkName":"AIME","category":"math","domain":null,"eligible":true,"ranked":true,"score":91.875,"unit":"percent","family":"vals-ai:aime:overall","index":55.656145230137575,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/aime","measuredAt":"2026-04-16","stderr":0.438,"costPerTest":0.070948,"latency":375.201},{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":89.8989898989899,"unit":"percent","family":"gpqa-diamond","index":59.66199777609283,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.1469735576055355,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:mmlu_pro:overall","benchmarkName":"MMLU Pro","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":86.9,"unit":"percent","family":"mmlu-pro","index":53.96432676432807,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","measuredAt":"2026-09-01","stderr":0.334,"costPerTest":0.012383,"latency":69.318},{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":86.8,"unit":"percent","family":"gpqa-diamond","index":56.81968892088687,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gpqaDiamond","benchmarkName":"GPQA Diamond","category":"knowledge","score":86.2,"unit":"percent","family":"gpqa-diamond","index":56.269385315935004,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:gpqa:overall","benchmarkName":"GPQA Diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":84.518,"unit":"percent","family":"gpqa-diamond","index":54.726700876719924,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","measuredAt":"2026-09-01","stderr":1.821,"costPerTest":0.044897,"latency":249.704},{"benchmarkId":"vals-ai:legal_bench:overall","benchmarkName":"LegalBench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":84.394,"unit":"percent","family":"vals-ai:legal_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","measuredAt":"2026-10-13","stderr":0.429,"costPerTest":0.002183,"latency":13.749},{"benchmarkId":"mmAnswerBench","benchmarkName":"MMAnswerBench","category":"math","score":83.8,"unit":"percent","family":"mmAnswerBench","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hmmtFeb2026","benchmarkName":"Harvard-MIT Mathematics Tournament February 2026","category":"math","score":82.6,"unit":"percent","family":"hmmtFeb2026","index":51.205736313860605,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:lcb:overall","benchmarkName":"LiveCodeBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":81.38,"unit":"percent","family":"livecodebench","index":56.86810771974169,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","measuredAt":"2026-09-01","stderr":1.061,"costPerTest":0.045983,"latency":226.424},{"benchmarkId":"epoch-ai:epoch-frontiermath-2025-02-28-public:overall","benchmarkName":"FrontierMath-2025-02-28-Public","category":"math","domain":null,"eligible":false,"ranked":true,"score":80,"unit":"percent","family":"epoch-ai:epoch-frontiermath-2025-02-28-public:overall","index":null,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":13.333333333333334,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:swebench:overall","benchmarkName":"SWE-bench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":76.4,"unit":"percent","family":"swe-bench-verified","index":57.577326472145714,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/swebench","measuredAt":"2026-09-01","stderr":1.901,"costPerTest":0.456603,"latency":527.071},{"benchmarkId":"aaIfBench","benchmarkName":"Artificial Analysis IFBench","category":"instruction","score":76.3,"unit":"percent","family":"ifbench-family","index":60.326789303555984,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-swe-bench-verified:overall","benchmarkName":"SWE-Bench verified","category":"coding","domain":null,"eligible":true,"ranked":true,"score":74.17,"unit":"percent","family":"swe-bench-verified","index":55.932573073688076,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.9900000000000002,"costPerTest":null,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":73.7,"unit":"percent","family":"lcr","index":57.86218419483386,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:medscribe:overall","benchmarkName":"MedScribe","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":72.27,"unit":"percent","family":"vals-ai:medscribe:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medscribe","measuredAt":"2026-10-29","stderr":2.064,"costPerTest":0.023717,"latency":95.701},{"benchmarkId":"mcpAtlas","benchmarkName":"MCP Atlas","category":"agentic","score":71.8,"unit":"percent","family":"mcpAtlas","index":59.41208710956293,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:tax_eval_v2:overall","benchmarkName":"TaxEval v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":71.194,"unit":"percent","family":"vals-ai:tax_eval_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","measuredAt":"2026-09-01","stderr":0.895,"costPerTest":0.016958,"latency":84.072},{"benchmarkId":"tau3Bench","benchmarkName":"τ³-Bench Tool-Agent-User Evaluation","category":"agentic","score":70.6,"unit":"percent","family":"tau3Bench","index":60.68378860350336,"aboveFit":false,"run":null,"source":"Sierra Research","sourceUrl":"https://github.com/sierra-research/tau2-bench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"cyberGym","benchmarkName":"CyberGym","category":"agentic","score":68.7,"unit":"percent","family":"cyberGym","index":59.72045479942245,"aboveFit":false,"run":null,"source":"Zhun Wang, Tianneng Shi, Jingxuan He, Matthew Cai, Jialin Zhang, Dawn Song","sourceUrl":"https://www.cybergym.io/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"browseComp","benchmarkName":"BrowseComp","category":"agentic","score":68,"unit":"percent","family":"browseComp","index":53.63034085855132,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:corp_fin_v2:overall","benchmarkName":"CorpFin v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":64.452,"unit":"percent","family":"vals-ai:corp_fin_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","measuredAt":"2026-08-12","stderr":0.943,"costPerTest":0.078922,"latency":68.17},{"benchmarkId":"terminalBench2","benchmarkName":"Terminal-Bench 2.0","category":"agentic","score":63.5,"unit":"percent","family":"terminalBench2","index":59.06807755694102,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"sweRebench","benchmarkName":"SWE-Rebench","category":"coding","score":62.7,"unit":"percent","family":"sweRebench","index":null,"aboveFit":false,"run":null,"source":"Nebius","sourceUrl":"https://swe-rebench.com","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"clawEval","benchmarkName":"Claw-Eval","category":"agentic","score":62.3,"unit":"percent","family":"clawEval","index":55.20654122894529,"aboveFit":false,"run":null,"source":"Bowen Ye, Rang Li, Qibin Yang, Yuanxin Liu, Linli Yao, Hanglong Lv, Zhihui Xie, Chenxin An, Lei Li, Lingpeng Kong, Qi Liu, Zhifang Sui, Tong Yang","sourceUrl":"https://arxiv.org/abs/2604.06132","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:public-benefits-bench:overall","benchmarkName":"Public Benefits Bench v1.1","category":null,"domain":"Public benefits","eligible":false,"ranked":true,"score":61.84,"unit":"percent","family":"vals-ai:public-benefits-bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/public-benefits-bench","measuredAt":"2026-10-12","stderr":1.264,"costPerTest":0.340666,"latency":580.381},{"benchmarkId":"gertLabs","benchmarkName":"Gert Labs Composite Game Benchmark","category":"agentic","score":60.11,"unit":"percent","family":"gertLabs","index":64.21883292586793,"aboveFit":false,"run":null,"source":"Gert Labs","sourceUrl":"https://gertlabs.com/rankings","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"swePro","benchmarkName":"SWE-bench Pro","category":"coding","score":58.4,"unit":"percent","family":"swePro","index":58.57340657529173,"aboveFit":false,"run":null,"source":"Xiang Deng, Jeff Da, Edwin Pan, Yannis Yiming He, Charles Ide, Kanak Garg, Niklas Lauffer, Andrew Park, Nitin Pasari, Chetan Rane, Karmini Sampath, Maya Krishnan, Srivatsa Kundurthy, Sean Hendryx, Zifan Wang, Chen Bo Calvin Zhang, Noah Jacobson, Bing Liu, Brad Kenstler","sourceUrl":"https://arxiv.org/abs/2509.16941","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:public-benefits-bench-v1:overall","benchmarkName":"Public Benefits Bench v1","category":null,"domain":"Public benefits","eligible":false,"ranked":true,"score":57.916,"unit":"percent","family":"vals-ai:public-benefits-bench-v1:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/public-benefits-bench-v1","measuredAt":"2026-06-09","stderr":1.284,"costPerTest":0.339125,"latency":437.877},{"benchmarkId":"vals-ai:finance_agent:overall","benchmarkName":"Finance Agent v1.1","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":57.655,"unit":"percent","family":"vals-ai:finance_agent:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/finance_agent","measuredAt":"2026-06-04","stderr":2.797,"costPerTest":0.291851,"latency":501.808},{"benchmarkId":"vals-ai:terminal-bench-2-1:overall","benchmarkName":"Terminal-Bench 2.1","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":56.929,"unit":"percent","family":"terminal-bench-2-1","index":54.436352251437164,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2-1","measuredAt":"2026-10-01","stderr":1.35,"costPerTest":0.232386,"latency":940.943},{"benchmarkId":"vals-ai:terminal-bench-2:overall","benchmarkName":"Terminal-Bench 2.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":53.933,"unit":"percent","family":"vals-ai:terminal-bench-2:overall","index":59.024930864585315,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2","measuredAt":"2026-06-04","stderr":5.314,"costPerTest":0.158286,"latency":881.675},{"benchmarkId":"hleWithTools","benchmarkName":"Humanity's Last Exam with tools","category":"agentic","score":52.3,"unit":"percent","family":"hleWithTools","index":62.001275956566175,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"openHarmonyBench","benchmarkName":"OpenHarmony Bench v1.0","category":"coding","score":52.3,"unit":"percent","family":"openHarmonyBench","index":59.80149799285386,"aboveFit":false,"run":null,"source":"OpenHarmony Bench authors","sourceUrl":"https://arxiv.org/abs/2608.16022","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:case_law_v2:overall","benchmarkName":"CaseLaw v2","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":51.554,"unit":"percent","family":"vals-ai:case_law_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/case_law_v2","measuredAt":"2026-05-04","stderr":1.134,"costPerTest":0.034033,"latency":48.961},{"benchmarkId":"aaSciCode","benchmarkName":"Artificial Analysis SciCode","category":"coding","score":44.8,"unit":"percent","family":"scicode-family","index":54.24516180189131,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/scicode","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:fabv2:overall","benchmarkName":"Finance Agent (v2)","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":44.792,"unit":"percent","family":"vals-ai:fabv2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/fabv2","measuredAt":"2026-10-29","stderr":0.606,"costPerTest":0.618185,"latency":924.078},{"benchmarkId":"nl2Repo","benchmarkName":"NL2Repo","category":"coding","score":42.7,"unit":"percent","family":"nl2Repo","index":56.88806810700351,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:medcode:overall","benchmarkName":"MedCode","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":41.604,"unit":"percent","family":"vals-ai:medcode:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medcode","measuredAt":"2026-10-29","stderr":2.124,"costPerTest":0.024196,"latency":77.578},{"benchmarkId":"aaItbench","benchmarkName":"Artificial Analysis ITBench-AA","category":"agentic","score":40.3,"unit":"percent","family":"aaItbench","index":58.53488664899706,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/itbench-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-simpleqa-verified:overall","benchmarkName":"SimpleQA Verified","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":34,"unit":"percent","family":"epoch-ai:epoch-simpleqa-verified:overall","index":52.24405377011239,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.4987482264363972,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-2025-02-28-private:overall","benchmarkName":"FrontierMath-2025-02-28-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":33.44827586206897,"unit":"percent","family":"epoch-ai:epoch-frontiermath-2025-02-28-private:overall","index":59.441562928919716,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.775348993336878,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:vibe-code:overall","benchmarkName":"Vibe Code Bench v1.1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":31.456,"unit":"percent","family":"vals-ai:vibe-code:overall","index":53.599341601787025,"aboveFit":false,"run":"OpenHands","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","measuredAt":"2026-11-09","stderr":4.553,"costPerTest":2.894911,"latency":2014.732},{"benchmarkId":"hle","benchmarkName":"Humanity's Last Exam","category":"knowledge","score":31,"unit":"percent","family":"hle","index":55.4887194092371,"aboveFit":false,"run":null,"source":"Center for AI Safety, Scale AI, and thousands of expert contributors","sourceUrl":"https://lastexam.ai/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gdpvalAaNormalized","benchmarkName":"GDPval-AA normalized","category":"agentic","score":30.8,"unit":"percent","family":"gdpvalAaNormalized","index":56.05505917730121,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":30.1,"unit":"percent","family":"hle","index":54.585755938710285,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:legal_research:overall","benchmarkName":"Legal Research Bench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":27.885,"unit":"percent","family":"vals-ai:legal_research:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_research","measuredAt":"2026-11-09","stderr":3.117,"costPerTest":0.385737,"latency":325.824},{"benchmarkId":"vals-ai:code-migration:overall","benchmarkName":"Code Migration","category":"coding","domain":null,"eligible":true,"ranked":true,"score":25.768,"unit":"percent","family":"vals-ai:code-migration:overall","index":58.87813217913276,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/code-migration","measuredAt":"2026-10-29","stderr":4.087,"costPerTest":5.701313,"latency":3502.363},{"benchmarkId":"epoch-ai:epoch-frontiermath-tiers-1-3-v2-private:overall","benchmarkName":"FrontierMath-Tiers-1-3-v2-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":24.912280701754387,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tiers-1-3-v2-private:overall","index":44.00731788504523,"aboveFit":false,"run":"none effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.566445410232946,"costPerTest":null,"latency":null},{"benchmarkId":"omniscienceAccuracy","benchmarkName":"Artificial Analysis Omniscience Accuracy","category":"knowledge","score":23.7,"unit":"percent","family":"omniscienceAccuracy","index":40.55801728883559,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-chess-puzzles:overall","benchmarkName":"Chess Puzzles","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":20,"unit":"percent","family":"epoch-ai:epoch-chess-puzzles:overall","index":49.164979628140856,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":4.020151261036848,"costPerTest":null,"latency":null},{"benchmarkId":"researchClawBench","benchmarkName":"ResearchClawBench","category":"agentic","score":18.2,"unit":"percent","family":"researchClawBench","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-private:overall","benchmarkName":"FrontierMath-Tier-4-2025-07-01-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":12.5,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-private:overall","index":55.89839122980863,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":4.824031156174749,"costPerTest":null,"latency":null},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":4.6,"unit":"percent","family":"critpt","index":45.75417926958779,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:programbench:overall","benchmarkName":"ProgramBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":0,"unit":"percent","family":"vals-ai:programbench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/programbench","measuredAt":"2026-11-09","stderr":0,"costPerTest":null,"latency":3931.735},{"benchmarkId":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-public:overall","benchmarkName":"FrontierMath-Tier-4-2025-07-01-Public","category":"math","domain":null,"eligible":false,"ranked":true,"score":0,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-public:overall","index":null,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":1.4,"outputPerMillion":4.4,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":101.5,"timeToFirstToken":0.54,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":{"elo":1462.70651715235,"eloUpper":1466.219331766602,"eloLower":1459.193702538098,"votes":67074,"rank":40,"rankUpper":null,"rankLower":null,"arenaName":"GLM-5.1","organization":"zai","updatedAt":"2026-11-09"},"quality":{"coveragePercent":85,"benchmarkCount":38,"categoryCount":6,"confidence":"medium","state":"available"},"hosts":[{"host":"chutes","name":"Chutes","kind":"host","url":"https://llm.chutes.ai/v1/models","inputPerMillion":0.9,"outputPerMillion":2.9,"cacheReadPerMillion":0.09,"contextTokens":202752,"maxOutputTokens":65535,"deprecated":false},{"host":"gmi-cloud","name":"GMI Cloud","kind":"host","url":"https://docs.gmicloud.ai/inference-engine/api-reference/llm-api-reference","inputPerMillion":0.98,"outputPerMillion":3.08,"cacheReadPerMillion":0.182,"contextTokens":202752,"maxOutputTokens":131072,"deprecated":false},{"host":"deepinfra","name":"Deep Infra","kind":"host","url":"https://deepinfra.com/models","inputPerMillion":1.05,"outputPerMillion":3.5,"cacheReadPerMillion":0.205,"contextTokens":202752,"maxOutputTokens":16384,"deprecated":true},{"host":"zhipu-ai","name":"Zhipu AI","kind":"maker","url":"https://docs.bigmodel.cn/cn/guide/start/pricing","inputPerMillion":0.89637,"outputPerMillion":3.58548,"cacheReadPerMillion":0.19421,"contextTokens":200000,"maxOutputTokens":131072,"deprecated":false},{"host":"siliconflow","name":"SiliconFlow","kind":"host","url":"https://cloud.siliconflow.com/models","inputPerMillion":1.19,"outputPerMillion":3.74,"cacheReadPerMillion":0.6,"contextTokens":205000,"maxOutputTokens":131000,"deprecated":false},{"host":"crusoe","name":"Crusoe","kind":"host","url":"https://docs.crusoecloud.com/managed-inference/overview","inputPerMillion":1.2,"outputPerMillion":4.4,"cacheReadPerMillion":0.25,"contextTokens":200000,"maxOutputTokens":131072,"deprecated":true},{"host":"novita-ai","name":"Novita AI","kind":"host","url":"https://novita.ai/docs/guides/introduction","inputPerMillion":1.38,"outputPerMillion":4.4,"cacheReadPerMillion":0.26,"contextTokens":204800,"maxOutputTokens":131072,"deprecated":false},{"host":"friendli","name":"Friendli","kind":"host","url":"https://friendli.ai/docs/guides/serverless_endpoints/introduction","inputPerMillion":1.4,"outputPerMillion":4.4,"cacheReadPerMillion":0.26,"contextTokens":202752,"maxOutputTokens":131072,"deprecated":false},{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":1.4,"outputPerMillion":4.4,"cacheReadPerMillion":0.26,"contextTokens":204800,"maxOutputTokens":131072,"deprecated":false},{"host":"together-ai","name":"Together AI","kind":"host","url":"https://docs.together.ai/docs/serverless-models","inputPerMillion":1.4,"outputPerMillion":4.4,"cacheReadPerMillion":0.26,"contextTokens":202752,"maxOutputTokens":131072,"deprecated":true},{"host":"z-ai","name":"Z.AI","kind":"maker","url":"https://docs.z.ai/guides/overview/pricing","inputPerMillion":1.4,"outputPerMillion":4.4,"cacheReadPerMillion":0.26,"contextTokens":200000,"maxOutputTokens":131072,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"minimax-m3","providerId":"minimax","slug":"minimax-m3","name":"MiniMax M3","provider":"MiniMax","description":"MiniMax M3 is a standard model from MiniMax. 47 benchmarks count toward its index, in 7 categories.","logoUrl":null,"releaseDate":"2026-06-01","contextWindowTokens":1048576,"modalities":["text","image","video"],"openWeights":true,"details":{"apiId":"minimax/minimax-m3","maxOutputTokens":512000,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.06,"weightsUrl":"https://huggingface.co/MiniMaxAI/Minimax-M3"},"reasoning":false,"overallScore":55.71774291801226,"overallScoreRange":{"low":52.859106005491306,"high":58.57637983053322},"badges":[],"capabilityScores":{"agentic":57.358718589417535,"coding":57.70328109287676,"reasoning":50.476825989994325,"multimodal":56.48271238118185,"knowledge":61.0065202239895,"multilingual":null,"instruction":54.722687574997785,"math":41.60755452003772},"capabilityCounts":{"knowledge":5,"agentic":18,"reasoning":5,"instruction":2,"coding":12,"multimodal":2,"math":3},"benchmarks":[{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":92.9,"unit":"percent","family":"gpqa-diamond","index":62.41444223789757,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:gpqa:overall","benchmarkName":"GPQA Diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":92.676,"unit":"percent","family":"gpqa-diamond","index":62.20899555871553,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","measuredAt":"2026-09-01","stderr":1.443,"costPerTest":0.0255,"latency":278.908},{"benchmarkId":"omniDocBench15","benchmarkName":"OmniDocBench 1.5","category":"multimodal","score":91.6,"unit":"percent","family":"omniDocBench15","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"tau2Bench","benchmarkName":"τ²-Bench Tool-Agent-User Evaluation","category":"agentic","score":88.9,"unit":"percent","family":"tau2-bench","index":61.781089795169635,"aboveFit":false,"run":null,"source":"Victor Barres, Honghua Dong, Soham Ray, Xujie Si, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2506.07982","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaHarveyLab","benchmarkName":"Artificial Analysis Harvey LAB-AA v1.0","category":"agentic","score":88.4,"unit":"percent","family":"aaHarveyLab","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:medscribe:overall","benchmarkName":"MedScribe","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":87.252,"unit":"percent","family":"vals-ai:medscribe:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medscribe","measuredAt":"2026-10-29","stderr":1.957,"costPerTest":0.013748,"latency":123.967},{"benchmarkId":"usamo2026","benchmarkName":"United States of America Mathematical Olympiad 2026","category":"math","score":85.71,"unit":"percent","family":"usamo2026","index":null,"aboveFit":false,"run":null,"source":"Mathematical Association of America","sourceUrl":"https://www.maa.org/math-competitions/usamo","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:sage:overall","benchmarkName":"SAGE","category":null,"domain":"Education","eligible":false,"ranked":true,"score":85.531,"unit":"percent","family":"vals-ai:sage:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/sage","measuredAt":"2026-11-05","stderr":1.755,"costPerTest":0.014772,"latency":51.873},{"benchmarkId":"vals-ai:legal_bench:overall","benchmarkName":"LegalBench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":85.415,"unit":"percent","family":"vals-ai:legal_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","measuredAt":"2026-10-13","stderr":0.436,"costPerTest":0.00091,"latency":8.205},{"benchmarkId":"videoMmeWithSub","benchmarkName":"Video-MME with subtitle","category":"multimodal","score":85.4,"unit":"percent","family":"videoMmeWithSub","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"videoMmmu","benchmarkName":"VideoMMMU","category":"multimodal","score":84.6,"unit":"percent","family":"videoMmmu","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:mmlu_pro:overall","benchmarkName":"MMLU Pro","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":84.22,"unit":"percent","family":"mmlu-pro","index":50.587847552082046,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","measuredAt":"2026-09-01","stderr":0.364,"costPerTest":0.004811,"latency":41.39},{"benchmarkId":"browseComp","benchmarkName":"BrowseComp","category":"agentic","score":83.52,"unit":"percent","family":"browseComp","index":65.8914482476608,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":83,"unit":"percent","family":"lcr","index":64.2427330827805,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaIfBench","benchmarkName":"Artificial Analysis IFBench","category":"instruction","score":82.9,"unit":"percent","family":"ifbench-family","index":66.49455936143072,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:lcb:overall","benchmarkName":"LiveCodeBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":82.154,"unit":"percent","family":"livecodebench","index":57.510327430989804,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","measuredAt":"2026-09-01","stderr":1.055,"costPerTest":0.03743,"latency":366.506},{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":81.31313131313132,"unit":"percent","family":"gpqa-diamond","index":51.78728289041792,"aboveFit":false,"run":"none effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.7772533334218945,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:mmmu:overall","benchmarkName":"MMMU Pro","category":"multimodal","domain":null,"eligible":true,"ranked":true,"score":81.156,"unit":"percent","family":"mmmu-pro","index":60.77022881584665,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmmu","measuredAt":"2026-09-01","stderr":0.94,"costPerTest":0.009431,"latency":68.354},{"benchmarkId":"sweVerified","benchmarkName":"Software Engineering Benchmark Verified","category":"coding","score":80.5,"unit":"percent","family":"swe-bench-verified","index":60.601312541058846,"aboveFit":false,"run":null,"source":"Carlos E. Jimenez, John Yang, Alexander Wettig, Shunyu Yao, Kexin Pei, Ofir Press, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2310.06770","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaMmmuPro","benchmarkName":"Artificial Analysis MMMU-Pro","category":"multimodal","score":78.6,"unit":"percent","family":"mmmu-pro","index":57.44381906620205,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmmu-pro","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmmuPro","benchmarkName":"Massive Multi-discipline Multimodal Understanding Pro","category":"multimodal","score":78.1,"unit":"percent","family":"mmmu-pro","index":56.79311293364247,"aboveFit":false,"run":null,"source":"MMMU-Pro authors","sourceUrl":"https://arxiv.org/abs/2409.02813","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-math:overall","benchmarkName":"LiveBench Mathematics","category":"math","domain":null,"eligible":true,"ranked":true,"score":76.94749999999999,"unit":"percent","family":"livebench:livebench-math:overall","index":47.01803958880753,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Mathematics","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.015987862796833775,"latency":null},{"benchmarkId":"livebench:livebench-language:overall","benchmarkName":"LiveBench Language","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":76.83566666666667,"unit":"percent","family":"livebench:livebench-language:overall","index":61.44525248877669,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Language","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.010721333333333334,"latency":null},{"benchmarkId":"livebench:livebench-data-analysis:overall","benchmarkName":"LiveBench Data Analysis","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":76.16633333333334,"unit":"percent","family":"livebench","index":60.288178897250084,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Data%20Analysis","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.028752666666666666,"latency":null},{"benchmarkId":"bankerToolBench","benchmarkName":"BankerToolBench","category":"agentic","score":76.1,"unit":"percent","family":"bankerToolBench","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:swebench:overall","benchmarkName":"SWE-bench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":75,"unit":"percent","family":"swe-bench-verified","index":56.544745863248544,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/swebench","measuredAt":"2026-09-01","stderr":1.938,"costPerTest":0.416355,"latency":726.041},{"benchmarkId":"gdpvalRubrics","benchmarkName":"GDPval rubrics","category":"agentic","score":74.7,"unit":"percent","family":"gdpvalRubrics","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"clawEval","benchmarkName":"Claw-Eval","category":"agentic","score":74.5,"unit":"percent","family":"clawEval","index":72.86137210564567,"aboveFit":true,"run":null,"source":"Bowen Ye, Rang Li, Qibin Yang, Yuanxin Liu, Linli Yao, Hanglong Lv, Zhihui Xie, Chenxin An, Lei Li, Lingpeng Kong, Qi Liu, Zhifang Sui, Tong Yang","sourceUrl":"https://arxiv.org/abs/2604.06132","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-reasoning:overall","benchmarkName":"LiveBench Reasoning","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":74.48075,"unit":"percent","family":"livebench","index":58.10461694391204,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Reasoning","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.019356435643564356,"latency":null},{"benchmarkId":"mcpAtlas","benchmarkName":"MCP Atlas","category":"agentic","score":74.2,"unit":"percent","family":"mcpAtlas","index":61.02823654013521,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:tax_eval_v2:overall","benchmarkName":"TaxEval v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":72.731,"unit":"percent","family":"vals-ai:tax_eval_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","measuredAt":"2026-09-01","stderr":0.863,"costPerTest":0.00728,"latency":94.036},{"benchmarkId":"livebench:livebench-coding-2:overall","benchmarkName":"LiveBench Coding 2.0","category":"coding","domain":null,"eligible":true,"ranked":true,"score":70.9,"unit":"percent","family":"livebench:livebench-coding-2:overall","index":59.52656676728189,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Coding","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.45509310344827586,"latency":null},{"benchmarkId":"osWorldVerified","benchmarkName":"OSWorld-Verified","category":"agentic","score":70.06,"unit":"percent","family":"osWorldVerified","index":57.03744031020724,"aboveFit":false,"run":null,"source":"Tianbao Xie, Danyang Zhang, Jixuan Chen, Xiaochuan Li, Siheng Zhao, Ruisheng Cao, Toh Jing Hua, Zhoujun Cheng, Dongchan Shin, Fangyu Lei, Yitao Liu, Yiheng Xu, Shuyan Zhou, Silvio Savarese, Caiming Xiong, Victor Zhong, Tao Yu","sourceUrl":"https://os-world.github.io/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:mortgage_tax:overall","benchmarkName":"MortgageTax","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":68.362,"unit":"percent","family":"vals-ai:mortgage_tax:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mortgage_tax","measuredAt":"2026-09-01","stderr":0.91,"costPerTest":0.008326,"latency":26.756},{"benchmarkId":"livebench:livebench-coding:overall","benchmarkName":"LiveBench Coding","category":"coding","domain":null,"eligible":true,"ranked":true,"score":68.2025,"unit":"percent","family":"livebench:livebench-coding:overall","index":50.74502751322684,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Coding","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.006314529914529915,"latency":null},{"benchmarkId":"vals-ai:corp_fin_v2:overall","benchmarkName":"CorpFin v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":68.104,"unit":"percent","family":"vals-ai:corp_fin_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","measuredAt":"2026-08-12","stderr":0.918,"costPerTest":0.049736,"latency":24.459},{"benchmarkId":"livebench:livebench-research:overall","benchmarkName":"LiveBench Research","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":68,"unit":"percent","family":"livebench-2-agentic","index":68.31998603696127,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Research","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.2516,"latency":null},{"benchmarkId":"livebench:livebench-design:overall","benchmarkName":"LiveBench Design","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":66.2,"unit":"percent","family":"livebench-2-agentic","index":67.00359006610446,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Design","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.20789200000000002,"latency":null},{"benchmarkId":"terminalBench21","benchmarkName":"Terminal-Bench 2.1","category":"agentic","score":66,"unit":"percent","family":"terminal-bench-2-1","index":59.74157526257154,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:cyber:overall","benchmarkName":"CyberBench v1.1","category":null,"domain":"Other","eligible":false,"ranked":true,"score":65,"unit":"percent","family":"vals-ai:cyber:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/cyber","measuredAt":"2026-11-05","stderr":6.104,"costPerTest":3.408292,"latency":2456.402},{"benchmarkId":"vals-ai:public-benefits-bench:overall","benchmarkName":"Public Benefits Bench v1.1","category":null,"domain":"Public benefits","eligible":false,"ranked":true,"score":64.141,"unit":"percent","family":"vals-ai:public-benefits-bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/public-benefits-bench","measuredAt":"2026-10-12","stderr":1.247,"costPerTest":0.2543,"latency":691.436},{"benchmarkId":"svgBench","benchmarkName":"SVG-Bench","category":"coding","score":63.7,"unit":"percent","family":"svgBench","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:public-benefits-bench-v1:overall","benchmarkName":"Public Benefits Bench v1","category":null,"domain":"Public benefits","eligible":false,"ranked":true,"score":60.69,"unit":"percent","family":"vals-ai:public-benefits-bench-v1:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/public-benefits-bench-v1","measuredAt":"2026-06-09","stderr":1.27,"costPerTest":0.198621,"latency":705.544},{"benchmarkId":"swePro","benchmarkName":"SWE-bench Pro","category":"coding","score":59,"unit":"percent","family":"swePro","index":59.08437256400452,"aboveFit":false,"run":null,"source":"Xiang Deng, Jeff Da, Edwin Pan, Yannis Yiming He, Charles Ide, Kanak Garg, Niklas Lauffer, Andrew Park, Nitin Pasari, Chetan Rane, Karmini Sampath, Maya Krishnan, Srivatsa Kundurthy, Sean Hendryx, Zifan Wang, Chen Bo Calvin Zhang, Noah Jacobson, Bing Liu, Brad Kenstler","sourceUrl":"https://arxiv.org/abs/2509.16941","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-instruction-following:overall","benchmarkName":"LiveBench Instruction Following","category":"instruction","domain":null,"eligible":true,"ranked":true,"score":57.50825,"unit":"percent","family":"livebench:livebench-instruction-following:overall","index":42.95081578856486,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=IF","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.019085,"latency":null},{"benchmarkId":"vals-ai:terminal-bench-2-1:overall","benchmarkName":"Terminal-Bench 2.1","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":53.558,"unit":"percent","family":"terminal-bench-2-1","index":52.46480481779876,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2-1","measuredAt":"2026-10-01","stderr":0.749,"costPerTest":0.205973,"latency":1023.416},{"benchmarkId":"vals-ai:skillsbench:overall","benchmarkName":"SkillsBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":51.497,"unit":"percent","family":"vals-ai:skillsbench:overall","index":63.4728810653352,"aboveFit":false,"run":"OpenHands","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/skillsbench","measuredAt":"2026-10-29","stderr":4.498,"costPerTest":0.504553,"latency":927.977},{"benchmarkId":"vibeV2","benchmarkName":"VIBE V2","category":"coding","score":50.1,"unit":"percent","family":"vibeV2","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"openHarmonyBench","benchmarkName":"OpenHarmony Bench v1.0","category":"coding","score":48.4,"unit":"percent","family":"openHarmonyBench","index":56.327276616367506,"aboveFit":false,"run":null,"source":"OpenHarmony Bench authors","sourceUrl":"https://arxiv.org/abs/2608.16022","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:fabv2:overall","benchmarkName":"Finance Agent (v2)","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":48.269,"unit":"percent","family":"vals-ai:fabv2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/fabv2","measuredAt":"2026-10-29","stderr":0.437,"costPerTest":0.32345,"latency":496.894},{"benchmarkId":"vals-ai:emb:overall","benchmarkName":"EMB","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":47.759,"unit":"percent","family":"vals-ai:emb:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/emb","measuredAt":"2026-10-20","stderr":2.946,"costPerTest":2.091762,"latency":1890.343},{"benchmarkId":"vals-ai:vibe-code:overall","benchmarkName":"Vibe Code Bench v1.1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":47.566,"unit":"percent","family":"vals-ai:vibe-code:overall","index":59.412082708869924,"aboveFit":false,"run":"OpenHands","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","measuredAt":"2026-11-09","stderr":5.442,"costPerTest":6.453899,"latency":5073.23},{"benchmarkId":"aaSciCode","benchmarkName":"Artificial Analysis SciCode","category":"coding","score":47.1,"unit":"percent","family":"scicode-family","index":57.2436514386452,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/scicode","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:medcode:overall","benchmarkName":"MedCode","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":46.289,"unit":"percent","family":"vals-ai:medcode:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medcode","measuredAt":"2026-10-29","stderr":2.104,"costPerTest":0.012125,"latency":63.12},{"benchmarkId":"vals-ai:terminal-bench-2:overall","benchmarkName":"Terminal-Bench 2.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":46.067,"unit":"percent","family":"vals-ai:terminal-bench-2:overall","index":54.28872372796373,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2","measuredAt":"2026-06-04","stderr":5.314,"costPerTest":0.216209,"latency":1019.457},{"benchmarkId":"officeQaPro","benchmarkName":"OfficeQA Pro","category":"multimodal","score":45.1,"unit":"percent","family":"officeQaPro","index":55.52160569616164,"aboveFit":false,"run":null,"source":"OfficeQA Pro authors","sourceUrl":"https://arxiv.org/abs/2603.08655","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"omniscienceAccuracy","benchmarkName":"Artificial Analysis Omniscience Accuracy","category":"knowledge","score":44.9,"unit":"percent","family":"omniscienceAccuracy","index":67.27544415089775,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"nl2Repo","benchmarkName":"NL2Repo","category":"coding","score":42.13,"unit":"percent","family":"nl2Repo","index":56.48619894599511,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaCweBench","benchmarkName":"Artificial Analysis CWE-Bench-AA","category":"agentic","score":41.7,"unit":"percent","family":"aaCweBench","index":56.35685760929913,"aboveFit":false,"run":null,"source":"Artificial Analysis; dataset by Collinear AI","sourceUrl":"https://artificialanalysis.ai/evaluations/cwe-bench-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-agentic-coding:overall","benchmarkName":"LiveBench Agentic Coding","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":40.65633333333333,"unit":"percent","family":"livebench:livebench-agentic-coding:overall","index":51.090874483611664,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Agentic%20Coding","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.42418194444444446,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":39,"unit":"percent","family":"hle","index":63.51506136947546,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gdpvalAaNormalized","benchmarkName":"GDPval-AA normalized","category":"agentic","score":37.1,"unit":"percent","family":"gdpvalAaNormalized","index":60.55941277595976,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:legal_research:overall","benchmarkName":"Legal Research Bench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":29.808,"unit":"percent","family":"vals-ai:legal_research:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_research","measuredAt":"2026-11-09","stderr":3.179,"costPerTest":0.342955,"latency":813.838},{"benchmarkId":"kernelBenchHard","benchmarkName":"KernelBench Hard","category":"coding","score":28.8,"unit":"percent","family":"kernelBenchHard","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":26.666666666666668,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":24.43359876278037,"aboveFit":false,"run":"none effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":6.666666666666668,"costPerTest":null,"latency":null},{"benchmarkId":"aaCyberGymE2e","benchmarkName":"Artificial Analysis CyberGym-E2E-AA","category":"agentic","score":24.4,"unit":"percent","family":"aaCyberGymE2e","index":55.8345123760445,"aboveFit":false,"run":null,"source":"Artificial Analysis; dataset by Berkeley RDI","sourceUrl":"https://artificialanalysis.ai/evaluations/cybergym-e2e-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaCyberIndex","benchmarkName":"Artificial Analysis Cyber Index","category":"agentic","score":22.4,"unit":"percent","family":"aaCyberIndex","index":50.2450982491225,"aboveFit":false,"run":null,"source":"Artificial Analysis with the Cyber Index Alliance (Collinear AI, IBM, NVIDIA, Pasta Lab, Vercel)","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-cyber-index","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:tax_agent_bench:overall","benchmarkName":"Tax Agent Bench","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":20.588,"unit":"percent","family":"vals-ai:tax_agent_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench","measuredAt":"2026-11-09","stderr":2.874,"costPerTest":0.164063,"latency":314.575},{"benchmarkId":"vals-ai:code-migration:overall","benchmarkName":"Code Migration","category":"coding","domain":null,"eligible":true,"ranked":true,"score":19.925,"unit":"percent","family":"vals-ai:code-migration:overall","index":55.77409631163544,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/code-migration","measuredAt":"2026-10-29","stderr":3.944,"costPerTest":7.072659,"latency":4428.262},{"benchmarkId":"researchClawBench","benchmarkName":"ResearchClawBench","category":"agentic","score":19.8,"unit":"percent","family":"researchClawBench","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:proof_bench:overall","benchmarkName":"ProofBench v1.1","category":"math","domain":null,"eligible":true,"ranked":true,"score":18,"unit":"percent","family":"vals-ai:proof_bench:overall","index":53.37102520852528,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/proof_bench","measuredAt":"2026-11-09","stderr":3.861,"costPerTest":0.419224,"latency":718.66},{"benchmarkId":"livebench:livebench-long-horizon:overall","benchmarkName":"LiveBench Long Agentic","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":12.4,"unit":"percent","family":"livebench-2-agentic","index":27.657977159383865,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Long%20Agentic","measuredAt":"2026-10-14","stderr":null,"costPerTest":4.069118181818182,"latency":null},{"benchmarkId":"vals-ai:vcb-1-100:overall","benchmarkName":"Vibe Code Bench 1-100","category":"coding","domain":null,"eligible":true,"ranked":true,"score":9.222,"unit":"percent","family":"vals-ai:vcb-1-100:overall","index":58.283862550015954,"aboveFit":false,"run":"OpenHands","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vcb-1-100","measuredAt":"2026-10-29","stderr":2.857,"costPerTest":3.028103,"latency":1570.129},{"benchmarkId":"epoch-ai:epoch-mystery-game-puzzles:overall","benchmarkName":"Mystery Game Puzzles","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":8,"unit":"percent","family":"epoch-ai:epoch-mystery-game-puzzles:overall","index":44.876317375711565,"aboveFit":false,"run":"none effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.7265992434429074,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-browser:overall","benchmarkName":"LiveBench Browser Use","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":8,"unit":"percent","family":"livebench-2-agentic","index":24.440120341733856,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Browser","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.42920400000000003,"latency":null},{"benchmarkId":"epoch-ai:epoch-chess-puzzles:overall","benchmarkName":"Chess Puzzles","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":8,"unit":"percent","family":"epoch-ai:epoch-chess-puzzles:overall","index":40.056797286339695,"aboveFit":false,"run":"none effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.7265992434429074,"costPerTest":null,"latency":null},{"benchmarkId":"osWorld2","benchmarkName":"OSWorld 2.0","category":"agentic","score":4.6,"unit":"percent","family":"osWorld2","index":55.931067343110335,"aboveFit":false,"run":null,"source":"Mengqi Yuan, Zilong Zhou, Xinzhuang Xiong, Weiming Wu, Jiayang Sun, Jiamin Song, Kaiqian Cui, Bowen Wang, Haoyuan Wu, Yitong Li, Dunjie Lu, Haikong Lu, Qi Zhen, Xinyuan Wang, Jiaqi Deng, Yuhao Yang, Cheng Chen, Boyuan Zheng, Alex Su, Xiao Yu, Hao Zou, Saaket Agashe, Xing Han Lu, Manpreet Kaur, Zhengyang Qi, Vincent Sunn Chen, Frederic Sala, Dayiheng Liu, Junyang Lin, Zhou Yu, Yu Su, Siva Reddy, Xin Eric Wang, Peng Qi, Tianbao Xie, Tao Yu","sourceUrl":"https://arxiv.org/abs/2606.29537","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":3.7,"unit":"percent","family":"critpt","index":44.01188428455879,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaDeepsecBench","benchmarkName":"Artificial Analysis DeepsecBench-AA","category":"agentic","score":1.1,"unit":"percent","family":"aaDeepsecBench","index":51.13007430674119,"aboveFit":false,"run":null,"source":"Artificial Analysis; dataset by Vercel","sourceUrl":"https://artificialanalysis.ai/evaluations/deepsecbench-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:terminal-bench-4:overall","benchmarkName":"Terminal-Bench 4.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":1.01,"unit":"percent","family":"terminal-bench-4","index":54.99560048358878,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-4","measuredAt":"2026-11-09","stderr":1.01,"costPerTest":7.229497,"latency":4582.924},{"benchmarkId":"lmarena:agent-bash-recovery:overall","benchmarkName":"Agent Arena command recovery","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":0.5799036305831118,"unit":"index","family":"agent-arena","index":67.00757300723842,"aboveFit":false,"run":null,"source":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset","measuredAt":"2026-10-08","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mazeBench","benchmarkName":"MazeBench: visual spatial reasoning in a 3D open world","category":"agentic","score":0,"unit":"percent","family":"mazeBench","index":61.97866734834783,"aboveFit":false,"run":null,"source":"Jonathan Pappas and David Pappas","sourceUrl":"https://mazebench.com/blog?post=maze-bench-results","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lmarena:agent-steerability:overall","benchmarkName":"Agent Arena steerability","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":-10.360938878971183,"unit":"index","family":"agent-arena","index":58.01248525397837,"aboveFit":false,"run":null,"source":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset","measuredAt":"2026-11-11","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lmarena:agent-task-outcome:overall","benchmarkName":"Agent Arena task outcome","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":-18.01655139011944,"unit":"index","family":"agent-arena","index":51.71837158927674,"aboveFit":false,"run":null,"source":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset","measuredAt":"2026-11-11","stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":0.3,"outputPerMillion":1.2,"batch":{"inputPerMillion":0.224,"outputPerMillion":0.88},"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":79,"timeToFirstToken":1.552,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":{"elo":1428.2232151443436,"eloUpper":1431.8798647965818,"eloLower":1424.5665654921054,"votes":72963,"rank":100,"rankUpper":null,"rankLower":null,"arenaName":"MiniMax-M3","organization":"minimax","updatedAt":"2026-11-09"},"quality":{"coveragePercent":95,"benchmarkCount":47,"categoryCount":7,"confidence":"high","state":"available"},"hosts":[{"host":"vultr","name":"Vultr","kind":"cloud","url":"https://api.vultrinference.com/","inputPerMillion":0.2,"outputPerMillion":0.9,"cacheReadPerMillion":null,"contextTokens":524288,"maxOutputTokens":512000,"deprecated":false},{"host":"coreweave","name":"CoreWeave","kind":"host","url":"https://docs.wandb.ai/inference","inputPerMillion":0.23,"outputPerMillion":0.96,"cacheReadPerMillion":0.05,"contextTokens":262144,"maxOutputTokens":262144,"deprecated":false},{"host":"deepinfra","name":"Deep Infra","kind":"host","url":"https://deepinfra.com/models","inputPerMillion":0.28,"outputPerMillion":1.1,"cacheReadPerMillion":0.056,"contextTokens":524288,"maxOutputTokens":512000,"deprecated":false},{"host":"fireworks-ai","name":"Fireworks AI","kind":"host","url":"https://fireworks.ai/docs/","inputPerMillion":0.3,"outputPerMillion":1.2,"cacheReadPerMillion":0.06,"contextTokens":512000,"maxOutputTokens":512000,"deprecated":false},{"host":"nebius","name":"Nebius","kind":"host","url":"https://docs.tokenfactory.nebius.com/","inputPerMillion":0.3,"outputPerMillion":1.2,"cacheReadPerMillion":0.06,"contextTokens":1048576,"maxOutputTokens":1048576,"deprecated":false},{"host":"novita-ai","name":"Novita AI","kind":"host","url":"https://novita.ai/docs/guides/introduction","inputPerMillion":0.3,"outputPerMillion":1.2,"cacheReadPerMillion":0.06,"contextTokens":1000000,"maxOutputTokens":131072,"deprecated":false},{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":0.3,"outputPerMillion":1.2,"cacheReadPerMillion":0.06,"contextTokens":1048576,"maxOutputTokens":512000,"deprecated":false},{"host":"siliconflow","name":"SiliconFlow","kind":"host","url":"https://cloud.siliconflow.com/models","inputPerMillion":0.3,"outputPerMillion":1.2,"cacheReadPerMillion":0.06,"contextTokens":1048576,"maxOutputTokens":131000,"deprecated":false},{"host":"together-ai","name":"Together AI","kind":"host","url":"https://docs.together.ai/docs/serverless-models","inputPerMillion":0.3,"outputPerMillion":1.2,"cacheReadPerMillion":0.06,"contextTokens":524288,"maxOutputTokens":524288,"deprecated":false},{"host":"gmi-cloud","name":"GMI Cloud","kind":"host","url":"https://docs.gmicloud.ai/inference-engine/api-reference/llm-api-reference","inputPerMillion":0.6,"outputPerMillion":2.4,"cacheReadPerMillion":0.12,"contextTokens":1048576,"maxOutputTokens":512000,"deprecated":false},{"host":"sambanova","name":"SambaNova","kind":"host","url":"https://docs.sambanova.ai/cloud/docs/get-started/supported-models","inputPerMillion":0.6,"outputPerMillion":2.4,"cacheReadPerMillion":0.06,"contextTokens":1048576,"maxOutputTokens":1048576,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:livebench","name":"LiveBench","url":"https://livebench.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:lmarena","name":"LMArena","url":"https://huggingface.co","license":"CC BY 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"apodex-1-1-mini","providerId":"apodex","slug":"apodex-1-1-mini","name":"Apodex 1.1 Mini","provider":"Apodex","description":"Apodex 1.1 Mini is a reasoning model from Apodex in the Apodex 1.1 family. 10 benchmarks count toward its index, in 5 categories.","logoUrl":null,"releaseDate":"2026-08-24","contextWindowTokens":262000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":true,"overallScore":55.673564486460634,"overallScoreRange":{"low":48.56016150996837,"high":62.7869674629529},"badges":[],"capabilityScores":{"agentic":59.86645483401983,"coding":55.157745604381624,"reasoning":53.72920720385067,"multimodal":58.22466642527356,"knowledge":53.76030532895064,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"knowledge":3,"reasoning":2,"multimodal":1,"coding":1,"agentic":3},"benchmarks":[{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":86.4,"unit":"percent","family":"gpqa-diamond","index":56.452819850918964,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":79.3,"unit":"percent","family":"lcr","index":61.70423513811355,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaMmmuPro","benchmarkName":"Artificial Analysis MMMU-Pro","category":"multimodal","score":79.2,"unit":"percent","family":"mmmu-pro","index":58.22466642527356,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmmu-pro","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"frontierScienceResearch","benchmarkName":"FrontierScience Research","category":"knowledge","score":51.7,"unit":"percent","family":"frontierScienceResearch","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaSciCode","benchmarkName":"Artificial Analysis SciCode","category":"coding","score":45.5,"unit":"percent","family":"scicode-family","index":55.157745604381624,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/scicode","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gdpvalAaNormalized","benchmarkName":"GDPval-AA normalized","category":"agentic","score":34.4,"unit":"percent","family":"gdpvalAaNormalized","index":58.6289755193918,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":34.1,"unit":"percent","family":"hle","index":58.59892691882946,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"apexAgentsAa","benchmarkName":"APEX-Agents-AA","category":"agentic","score":31.2,"unit":"percent","family":"apexAgentsAa","index":63.51677212881744,"aboveFit":false,"run":null,"source":"Artificial Analysis / Mercor","sourceUrl":"https://artificialanalysis.ai/evaluations/apex-agents-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"omniscienceAccuracy","benchmarkName":"Artificial Analysis Omniscience Accuracy","category":"knowledge","score":28.2,"unit":"percent","family":"omniscienceAccuracy","index":46.229169217103504,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"apexAgents","benchmarkName":"APEX-Agents","category":"agentic","score":27.7,"unit":"percent","family":"apexAgents","index":57.453616853850235,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":4.6,"unit":"percent","family":"critpt","index":45.75417926958779,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":0,"outputPerMillion":0,"batch":null,"freeTier":true,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":80,"benchmarkCount":10,"categoryCount":5,"confidence":"low","state":"available"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"grok420reasoning","providerId":"xai","slug":"xai-grok-4-20-reasoning","name":"Grok 4.20 (Reasoning)","provider":"xAI","description":"Grok 4.20 (Reasoning) is a model from xAI. 13 benchmarks count toward its index, in 6 categories.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":55.546448805536656,"overallScoreRange":{"low":50.06169771905568,"high":61.03119989201763},"badges":[],"capabilityScores":{"agentic":48.94743336709489,"coding":51.592530794675056,"reasoning":68.61514495011431,"multimodal":63.77909397280216,"knowledge":55.82703102029,"multilingual":null,"instruction":null,"math":57.65912595020998},"capabilityCounts":{"math":1,"reasoning":2,"knowledge":2,"coding":5,"multimodal":1,"agentic":2},"benchmarks":[{"benchmarkId":"vals-ai:aime:overall","benchmarkName":"AIME","category":"math","domain":null,"eligible":true,"ranked":true,"score":96.458,"unit":"percent","family":"vals-ai:aime:overall","index":57.65912595020998,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/aime","measuredAt":"2026-04-16","stderr":0.516,"costPerTest":0.059397,"latency":106.799},{"benchmarkId":"arc-prize:v1-public-eval:overall","benchmarkName":"ARC-AGI-1 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":95.5,"unit":"percent","family":"arc-prize:v1-public-eval:overall","index":null,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.24,"latency":null},{"benchmarkId":"vals-ai:medqa:overall","benchmarkName":"MedQA","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":94.55,"unit":"percent","family":"vals-ai:medqa:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medqa","measuredAt":"2026-04-16","stderr":0.207,"costPerTest":0.009106,"latency":12.225},{"benchmarkId":"arc-prize:v1-semi-private:overall","benchmarkName":"ARC-AGI-1 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":89.5,"unit":"percent","family":"arc-prize:v1-semi-private:overall","index":68.13374007639926,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.35,"latency":null},{"benchmarkId":"vals-ai:gpqa:overall","benchmarkName":"GPQA Diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":88.636,"unit":"percent","family":"gpqa-diamond","index":58.50361795203959,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","measuredAt":"2026-09-01","stderr":1.594,"costPerTest":0.034461,"latency":59.421},{"benchmarkId":"vals-ai:mmlu_pro:overall","benchmarkName":"MMLU Pro","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":86.254,"unit":"percent","family":"mmlu-pro","index":53.15044408854041,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","measuredAt":"2026-09-01","stderr":0.34,"costPerTest":0.012836,"latency":11.654},{"benchmarkId":"vals-ai:lcb:overall","benchmarkName":"LiveCodeBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":84.265,"unit":"percent","family":"livecodebench","index":59.261911165414595,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","measuredAt":"2026-09-01","stderr":1.028,"costPerTest":0.05249,"latency":76.09},{"benchmarkId":"vals-ai:mmmu:overall","benchmarkName":"MMMU Pro","category":"multimodal","domain":null,"eligible":true,"ranked":true,"score":83.468,"unit":"percent","family":"mmmu-pro","index":63.77909397280216,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmmu","measuredAt":"2026-09-01","stderr":0.893,"costPerTest":0.026669,"latency":44.461},{"benchmarkId":"vals-ai:legal_bench:overall","benchmarkName":"LegalBench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":77.738,"unit":"percent","family":"vals-ai:legal_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","measuredAt":"2026-10-13","stderr":0.48,"costPerTest":0.006065,"latency":7.574},{"benchmarkId":"vals-ai:tax_eval_v2:overall","benchmarkName":"TaxEval v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":74.121,"unit":"percent","family":"vals-ai:tax_eval_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","measuredAt":"2026-09-01","stderr":0.859,"costPerTest":0.017152,"latency":13.394},{"benchmarkId":"vals-ai:swebench:overall","benchmarkName":"SWE-bench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":72.2,"unit":"percent","family":"swe-bench-verified","index":54.47958464545421,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/swebench","measuredAt":"2026-09-01","stderr":2.006,"costPerTest":0.515946,"latency":134.759},{"benchmarkId":"arc-prize:v2-semi-private:overall","benchmarkName":"ARC-AGI-2 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":65.14,"unit":"percent","family":"arc-prize:v2-semi-private:overall","index":69.09654982382935,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.92,"latency":null},{"benchmarkId":"vals-ai:corp_fin_v2:overall","benchmarkName":"CorpFin v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":63.675,"unit":"percent","family":"vals-ai:corp_fin_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","measuredAt":"2026-08-12","stderr":0.947,"costPerTest":0.163386,"latency":9.67},{"benchmarkId":"arc-prize:v2-public-eval:overall","benchmarkName":"ARC-AGI-2 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":63.61,"unit":"percent","family":"arc-prize:v2-public-eval:overall","index":null,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.96,"latency":null},{"benchmarkId":"vals-ai:medscribe:overall","benchmarkName":"MedScribe","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":63.412,"unit":"percent","family":"vals-ai:medscribe:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medscribe","measuredAt":"2026-10-29","stderr":2.095,"costPerTest":0.031303,"latency":18.596},{"benchmarkId":"vals-ai:case_law_v2:overall","benchmarkName":"CaseLaw v2","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":54.448,"unit":"percent","family":"vals-ai:case_law_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/case_law_v2","measuredAt":"2026-05-04","stderr":0.284,"costPerTest":0.068241,"latency":10.663},{"benchmarkId":"vals-ai:finance_agent:overall","benchmarkName":"Finance Agent v1.1","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":52.295,"unit":"percent","family":"vals-ai:finance_agent:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/finance_agent","measuredAt":"2026-06-04","stderr":0,"costPerTest":0.362389,"latency":127.028},{"benchmarkId":"vals-ai:mortgage_tax:overall","benchmarkName":"MortgageTax","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":45.35,"unit":"percent","family":"vals-ai:mortgage_tax:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mortgage_tax","measuredAt":"2026-09-01","stderr":0.988,"costPerTest":0.024144,"latency":11.124},{"benchmarkId":"vals-ai:terminal-bench-2-1:overall","benchmarkName":"Terminal-Bench 2.1","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":44.195,"unit":"percent","family":"terminal-bench-2-1","index":46.988804040238264,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2-1","measuredAt":"2026-10-01","stderr":0.991,"costPerTest":0.407362,"latency":305.726},{"benchmarkId":"vals-ai:terminal-bench-2:overall","benchmarkName":"Terminal-Bench 2.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":40.449,"unit":"percent","family":"vals-ai:terminal-bench-2:overall","index":50.90606269395151,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2","measuredAt":"2026-06-04","stderr":5.232,"costPerTest":0.154109,"latency":161.408},{"benchmarkId":"vals-ai:medcode:overall","benchmarkName":"MedCode","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":32.156,"unit":"percent","family":"vals-ai:medcode:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medcode","measuredAt":"2026-10-29","stderr":2.124,"costPerTest":0.036189,"latency":16.546},{"benchmarkId":"vals-ai:ioi-v1:overall","benchmarkName":"IOI v1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":30.166,"unit":"percent","family":"vals-ai:ioi-v1:overall","index":55.14969450297032,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/ioi-v1","measuredAt":"2026-08-09","stderr":7.489,"costPerTest":0.709499,"latency":896.93},{"benchmarkId":"vals-ai:fabv2:overall","benchmarkName":"Finance Agent (v2)","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":28.492,"unit":"percent","family":"vals-ai:fabv2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/fabv2","measuredAt":"2026-10-29","stderr":0.315,"costPerTest":0.791809,"latency":351.413},{"benchmarkId":"vals-ai:legal_research:overall","benchmarkName":"Legal Research Bench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":13.942,"unit":"percent","family":"vals-ai:legal_research:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_research","measuredAt":"2026-11-09","stderr":2.408,"costPerTest":0.261499,"latency":244.487},{"benchmarkId":"vals-ai:emb:overall","benchmarkName":"EMB","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":11.739,"unit":"percent","family":"vals-ai:emb:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/emb","measuredAt":"2026-10-20","stderr":1.553,"costPerTest":0.469926,"latency":367.615},{"benchmarkId":"vals-ai:tax_agent_bench:overall","benchmarkName":"Tax Agent Bench","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":6.952,"unit":"percent","family":"vals-ai:tax_agent_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench","measuredAt":"2026-11-09","stderr":1.75,"costPerTest":0.119262,"latency":89.262},{"benchmarkId":"vals-ai:vibe-code:overall","benchmarkName":"Vibe Code Bench v1.1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":4.063,"unit":"percent","family":"vals-ai:vibe-code:overall","index":43.715516825479035,"aboveFit":false,"run":"OpenHands","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","measuredAt":"2026-11-09","stderr":2.058,"costPerTest":0.771885,"latency":301.651},{"benchmarkId":"vals-ai:code-migration:overall","benchmarkName":"Code Migration","category":"coding","domain":null,"eligible":true,"ranked":true,"score":0.314,"unit":"percent","family":"vals-ai:code-migration:overall","index":45.35594683405713,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/code-migration","measuredAt":"2026-10-29","stderr":0.11,"costPerTest":0.289486,"latency":373.673}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":90,"benchmarkCount":13,"categoryCount":6,"confidence":"medium","state":"available"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:arc-prize-foundation","name":"ARC Prize Foundation","url":"https://arcprize.org","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"qwen3-5-397b-reasoning","providerId":"alibaba","slug":"qwen3-5-397b-reasoning","name":"Qwen3.5 397B (Reasoning)","provider":"Alibaba","description":"Qwen3.5 397B (Reasoning) is a reasoning model from Alibaba in the Qwen3.5 397B family. 9 benchmarks count toward its index, in 6 categories.","logoUrl":null,"releaseDate":"2026-02-16","contextWindowTokens":128000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":true,"overallScore":55.30939408185501,"overallScoreRange":{"low":48.41939408185501,"high":62.19939408185501},"badges":[],"capabilityScores":{"agentic":59.40857645235432,"coding":54.24516180189131,"reasoning":50.236095869051624,"multimodal":55.75198312154714,"knowledge":56.29737726368192,"multilingual":null,"instruction":62.66306584062974,"math":null},"capabilityCounts":{"agentic":2,"knowledge":2,"instruction":1,"reasoning":2,"multimodal":1,"coding":1},"benchmarks":[{"benchmarkId":"tau2Bench","benchmarkName":"τ²-Bench Tool-Agent-User Evaluation","category":"agentic","score":95.6,"unit":"percent","family":"tau2-bench","index":66.22702964504438,"aboveFit":false,"run":null,"source":"Victor Barres, Honghua Dong, Soham Ray, Xujie Si, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2506.07982","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":89.3,"unit":"percent","family":"gpqa-diamond","index":59.11262060818633,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaIfBench","benchmarkName":"Artificial Analysis IFBench","category":"instruction","score":78.8,"unit":"percent","family":"ifbench-family","index":62.66306584062974,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":77.3,"unit":"percent","family":"lcr","index":60.332074086942235,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaMmmuPro","benchmarkName":"Artificial Analysis MMMU-Pro","category":"multimodal","score":77.3,"unit":"percent","family":"mmmu-pro","index":55.75198312154714,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmmu-pro","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaSciCode","benchmarkName":"Artificial Analysis SciCode","category":"coding","score":44.8,"unit":"percent","family":"scicode-family","index":54.24516180189131,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/scicode","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":29,"unit":"percent","family":"hle","index":53.482133919177514,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"apexAgentsAa","benchmarkName":"APEX-Agents-AA","category":"agentic","score":15.3,"unit":"percent","family":"apexAgentsAa","index":52.590123259664246,"aboveFit":false,"run":null,"source":"Artificial Analysis / Mercor","sourceUrl":"https://artificialanalysis.ai/evaluations/apex-agents-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":1.7,"unit":"percent","family":"critpt","index":40.14011765116101,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":0.6,"outputPerMillion":3.6,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":85,"benchmarkCount":9,"categoryCount":6,"confidence":"low","state":"available"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gpt-5-1","providerId":"openai","slug":"gpt-5-1","name":"GPT-5.1","provider":"OpenAI","description":"GPT-5.1 is a reasoning model from OpenAI. 25 benchmarks count toward its index, in 7 categories.","logoUrl":null,"releaseDate":"2025-11-13","contextWindowTokens":400000,"modalities":["image","text","file"],"openWeights":false,"details":{"apiId":"openai/gpt-5.1","maxOutputTokens":128000,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.125,"weightsUrl":null},"reasoning":true,"overallScore":55.26283292232105,"overallScoreRange":{"low":51.52883292232105,"high":58.996832922321055},"badges":[],"capabilityScores":{"agentic":52.3993591705334,"coding":53.450567738930566,"reasoning":56.363596560354225,"multimodal":58.40621343625769,"knowledge":57.21310312350888,"multilingual":null,"instruction":57.14945321313568,"math":57.493744316987964},"capabilityCounts":{"math":4,"knowledge":5,"coding":4,"multimodal":1,"agentic":6,"reasoning":4,"instruction":1},"benchmarks":[{"benchmarkId":"vals-ai:medqa:overall","benchmarkName":"MedQA","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":96.383,"unit":"percent","family":"vals-ai:medqa:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medqa","measuredAt":"2026-04-16","stderr":0.17,"costPerTest":0.009789,"latency":12.291},{"benchmarkId":"aaAime2025","benchmarkName":"Artificial Analysis AIME 2025","category":"math","score":94,"unit":"percent","family":"aaAime2025","index":null,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/aime-2025","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:aime:overall","benchmarkName":"AIME","category":"math","domain":null,"eligible":true,"ranked":true,"score":93.333,"unit":"percent","family":"vals-ai:aime:overall","index":56.29335794885142,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/aime","measuredAt":"2026-04-16","stderr":0.62,"costPerTest":0.150354,"latency":163.891},{"benchmarkId":"vals-ai:mgsm:overall","benchmarkName":"MGSM","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":92.982,"unit":"percent","family":"vals-ai:mgsm:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mgsm","measuredAt":"2026-01-09","stderr":1.536,"costPerTest":0.00606,"latency":8.347},{"benchmarkId":"aaGlobalMmluLite","benchmarkName":"Artificial Analysis Global-MMLU-Lite","category":"multilingual","score":90.6,"unit":"percent","family":"aaGlobalMmluLite","index":null,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/global-mmlu-lite","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":88.61111111111111,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":60.47250770680209,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":4.0479727089927025,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:medscribe:overall","benchmarkName":"MedScribe","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":88.09,"unit":"percent","family":"vals-ai:medscribe:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medscribe","measuredAt":"2026-10-29","stderr":1.942,"costPerTest":0.096508,"latency":77.983},{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":87.62626262626263,"unit":"percent","family":"gpqa-diamond","index":57.577514424002416,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.8753682810775951,"costPerTest":null,"latency":null},{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":87.3,"unit":"percent","family":"gpqa-diamond","index":57.27827525834676,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaMmluPro","benchmarkName":"Artificial Analysis MMLU-Pro","category":"knowledge","score":87,"unit":"percent","family":"mmlu-pro","index":54.09031479463575,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmlu-pro","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaLiveCodeBench","benchmarkName":"Artificial Analysis LiveCodeBench","category":"coding","score":86.8,"unit":"percent","family":"livecodebench","index":61.36530518093654,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/livecodebench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:gpqa:overall","benchmarkName":"GPQA Diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":86.616,"unit":"percent","family":"gpqa-diamond","index":56.65092914870163,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","measuredAt":"2026-09-01","stderr":1.711,"costPerTest":0.074563,"latency":85.708},{"benchmarkId":"vals-ai:lcb:overall","benchmarkName":"LiveCodeBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":86.486,"unit":"percent","family":"livecodebench","index":61.10476643502968,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","measuredAt":"2026-09-01","stderr":0.983,"costPerTest":0.125706,"latency":141.24},{"benchmarkId":"vals-ai:mmlu_pro:overall","benchmarkName":"MMLU Pro","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":86.377,"unit":"percent","family":"mmlu-pro","index":53.30540936581886,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","measuredAt":"2026-09-01","stderr":0.337,"costPerTest":0.018514,"latency":23.056},{"benchmarkId":"vals-ai:legal_bench:overall","benchmarkName":"LegalBench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":85.683,"unit":"percent","family":"vals-ai:legal_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","measuredAt":"2026-10-13","stderr":0.397,"costPerTest":0.004796,"latency":7.048},{"benchmarkId":"vals-ai:mmmu:overall","benchmarkName":"MMMU Pro","category":"multimodal","domain":null,"eligible":true,"ranked":true,"score":83.179,"unit":"percent","family":"mmmu-pro","index":63.40298582818272,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmmu","measuredAt":"2026-09-01","stderr":0.899,"costPerTest":0.002764,"latency":75.307},{"benchmarkId":"tau2Bench","benchmarkName":"τ²-Bench Tool-Agent-User Evaluation","category":"agentic","score":81.9,"unit":"percent","family":"tau2-bench","index":57.136078011718396,"aboveFit":false,"run":null,"source":"Victor Barres, Honghua Dong, Soham Ray, Xujie Si, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2506.07982","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":80,"unit":"percent","family":"lcr","index":62.18449150602352,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaMmmuPro","benchmarkName":"Artificial Analysis MMMU-Pro","category":"multimodal","score":75.5,"unit":"percent","family":"mmmu-pro","index":53.409441044332645,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmmu-pro","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:tax_eval_v2:overall","benchmarkName":"TaxEval v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":74.857,"unit":"percent","family":"vals-ai:tax_eval_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","measuredAt":"2026-09-01","stderr":0.854,"costPerTest":0.007193,"latency":44.176},{"benchmarkId":"vals-ai:case_law_v2:overall","benchmarkName":"CaseLaw v2","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":73.419,"unit":"percent","family":"vals-ai:case_law_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/case_law_v2","measuredAt":"2026-05-04","stderr":0.75,"costPerTest":0.042966,"latency":23.591},{"benchmarkId":"aaIfBench","benchmarkName":"Artificial Analysis IFBench","category":"instruction","score":72.9,"unit":"percent","family":"ifbench-family","index":57.14945321313568,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:swebench:overall","benchmarkName":"SWE-bench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":69.8,"unit":"percent","family":"swe-bench-verified","index":52.70944645877334,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/swebench","measuredAt":"2026-09-01","stderr":2.055,"costPerTest":0.713431,"latency":500.651},{"benchmarkId":"epoch-ai:epoch-swe-bench-verified:overall","benchmarkName":"SWE-Bench verified","category":"coding","domain":null,"eligible":true,"ranked":true,"score":66.94214876033058,"unit":"percent","family":"swe-bench-verified","index":50.601616620776646,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.122974956319103,"costPerTest":null,"latency":null},{"benchmarkId":"swebench-experiments:swe-bench-verified:overall","benchmarkName":"SWE-bench Verified","category":"coding","domain":null,"eligible":true,"ranked":true,"score":66,"unit":"percent","family":"swe-bench-verified","index":49.90672766319531,"aboveFit":false,"run":"medium effort · mini-SWE-agent","source":"SWE-bench team","sourceUrl":"https://github.com/SWE-bench/experiments/tree/main/evaluation/verified","measuredAt":"2026-09-01","stderr":null,"costPerTest":0.3061554725,"latency":null},{"benchmarkId":"vals-ai:corp_fin_v2:overall","benchmarkName":"CorpFin v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":63.831,"unit":"percent","family":"vals-ai:corp_fin_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","measuredAt":"2026-08-12","stderr":0.947,"costPerTest":0.091991,"latency":35.289},{"benchmarkId":"vals-ai:mortgage_tax:overall","benchmarkName":"MortgageTax","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":61.368,"unit":"percent","family":"vals-ai:mortgage_tax:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mortgage_tax","measuredAt":"2026-09-01","stderr":0.931,"costPerTest":0.026408,"latency":46.254},{"benchmarkId":"vals-ai:finance_agent:overall","benchmarkName":"Finance Agent v1.1","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":55.309,"unit":"percent","family":"vals-ai:finance_agent:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/finance_agent","measuredAt":"2026-06-04","stderr":2.802,"costPerTest":0.473742,"latency":578.064},{"benchmarkId":"vals-ai:medcode:overall","benchmarkName":"MedCode","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":52.732,"unit":"percent","family":"vals-ai:medcode:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medcode","measuredAt":"2026-10-29","stderr":2.151,"costPerTest":0.014371,"latency":54.546},{"benchmarkId":"epoch-ai:epoch-simpleqa-verified:overall","benchmarkName":"SimpleQA Verified","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":48,"unit":"percent","family":"epoch-ai:epoch-simpleqa-verified:overall","index":63.9072896666386,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.5806639423035178,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:terminal-bench:overall","benchmarkName":"Terminal-Bench 1.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":47.5,"unit":"percent","family":"vals-ai:terminal-bench:overall","index":52.59284878179137,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench","measuredAt":"2026-01-12","stderr":5.583,"costPerTest":0.10176,"latency":685.951},{"benchmarkId":"terminalBenchHard","benchmarkName":"Terminal-Bench Hard","category":"agentic","score":45.5,"unit":"percent","family":"terminalBenchHard","index":56.42730214348893,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/terminalbench-hard","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:terminal-bench-2:overall","benchmarkName":"Terminal-Bench 2.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":44.944,"unit":"percent","family":"vals-ai:terminal-bench-2:overall","index":53.61255278791465,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2","measuredAt":"2026-06-04","stderr":5.303,"costPerTest":0.407628,"latency":886.169},{"benchmarkId":"epoch-ai:epoch-chess-puzzles:overall","benchmarkName":"Chess Puzzles","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":42,"unit":"percent","family":"epoch-ai:epoch-chess-puzzles:overall","index":65.86331392144298,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":4.9604496374885825,"costPerTest":null,"latency":null},{"benchmarkId":"gertLabs","benchmarkName":"Gert Labs Composite Game Benchmark","category":"agentic","score":41.24,"unit":"percent","family":"gertLabs","index":48.9394771655174,"aboveFit":false,"run":null,"source":"Gert Labs","sourceUrl":"https://gertlabs.com/rankings","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"omniscienceAccuracy","benchmarkName":"Artificial Analysis Omniscience Accuracy","category":"knowledge","score":37.7,"unit":"percent","family":"omniscienceAccuracy","index":58.20160106566911,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-2025-02-28-private:overall","benchmarkName":"FrontierMath-2025-02-28-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":31.03448275862069,"unit":"percent","family":"epoch-ai:epoch-frontiermath-2025-02-28-private:overall","index":57.31072038248972,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.721380905679257,"costPerTest":null,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":28.5,"unit":"percent","family":"hle","index":52.980487546662616,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:vibe-code:overall","benchmarkName":"Vibe Code Bench v1.1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":24.606,"unit":"percent","family":"vals-ai:vibe-code:overall","index":51.127753980215466,"aboveFit":false,"run":"OpenHands","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","measuredAt":"2026-11-09","stderr":4.254,"costPerTest":2.574926,"latency":1836.188},{"benchmarkId":"vals-ai:ioi-v1:overall","benchmarkName":"IOI v1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":21.5,"unit":"percent","family":"vals-ai:ioi-v1:overall","index":50.837864546747056,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/ioi-v1","measuredAt":"2026-08-09","stderr":7.336,"costPerTest":0.368193,"latency":1750.069},{"benchmarkId":"gdpvalAaNormalized","benchmarkName":"GDPval-AA normalized","category":"agentic","score":16.3,"unit":"percent","family":"gdpvalAaNormalized","index":45.68789613276965,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-mystery-game-puzzles:overall","benchmarkName":"Mystery Game Puzzles","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":16,"unit":"percent","family":"epoch-ai:epoch-mystery-game-puzzles:overall","index":51.071636549352945,"aboveFit":false,"run":"medium effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":3.684529491774706,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-private:overall","benchmarkName":"FrontierMath-Tier-4-2025-07-01-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":12.5,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-private:overall","index":55.89839122980863,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":4.824031156174749,"costPerTest":null,"latency":null},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":4.9,"unit":"percent","family":"critpt","index":46.33494426459745,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":1.25,"outputPerMillion":10,"batch":{"inputPerMillion":0.625,"outputPerMillion":5},"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":50,"timeToFirstToken":1.423,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":{"elo":1422.309575464516,"eloUpper":1425.8686671339924,"eloLower":1418.7504837950396,"votes":44207,"rank":113,"rankUpper":null,"rankLower":null,"arenaName":"GPT-5.1","organization":"openai","updatedAt":"2026-11-09"},"quality":{"coveragePercent":95,"benchmarkCount":25,"categoryCount":7,"confidence":"medium","state":"available"},"hosts":[{"host":"databricks","name":"Databricks","kind":"cloud","url":"https://docs.databricks.com/aws/en/machine-learning/foundation-models/","inputPerMillion":1.25,"outputPerMillion":10,"cacheReadPerMillion":0.125,"contextTokens":400000,"maxOutputTokens":128000,"deprecated":false},{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":1.25,"outputPerMillion":10,"cacheReadPerMillion":0.125,"contextTokens":400000,"maxOutputTokens":128000,"deprecated":false},{"host":"snowflake","name":"Snowflake Cortex","kind":"cloud","url":"https://docs.snowflake.com/en/user-guide/snowflake-cortex/cortex-rest-api","inputPerMillion":1.25,"outputPerMillion":10,"cacheReadPerMillion":0.13,"contextTokens":272000,"maxOutputTokens":8192,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:swe-bench-team","name":"SWE-bench team","url":"https://github.com","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"step-3-7-flash","providerId":"stepfun","slug":"step-3-7-flash","name":"Step 3.7 Flash","provider":"StepFun","description":"Step 3.7 Flash is a reasoning model from StepFun. 21 benchmarks count toward its index, in 6 categories.","logoUrl":null,"releaseDate":"2026-05-29","contextWindowTokens":262144,"modalities":["text","image","video"],"openWeights":true,"details":{"apiId":"stepfun/step-3.7-flash","maxOutputTokens":230400,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.04,"weightsUrl":"https://huggingface.co/stepfun-ai/Step-3.7-Flash"},"reasoning":true,"overallScore":55.258825254169835,"overallScoreRange":{"low":50.230241969053836,"high":60.287408539285835},"badges":[],"capabilityScores":{"agentic":59.592302225592256,"coding":54.928432692457505,"reasoning":49.581915918007105,"multimodal":59.2870574343638,"knowledge":46.823344683723946,"multilingual":null,"instruction":51.91619377009045,"math":null},"capabilityCounts":{"agentic":10,"multimodal":3,"knowledge":3,"reasoning":2,"instruction":1,"coding":2},"benchmarks":[{"benchmarkId":"tau2Bench","benchmarkName":"τ²-Bench Tool-Agent-User Evaluation","category":"agentic","score":98.5,"unit":"percent","family":"tau2-bench","index":68.15139166961706,"aboveFit":false,"run":null,"source":"Victor Barres, Honghua Dong, Soham Ray, Xujie Si, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2506.07982","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vStar","benchmarkName":"V*","category":"multimodal","score":95.3,"unit":"percent","family":"vStar","index":55.709959024426944,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"deepSearchQa","benchmarkName":"DeepSearchQA","category":"agentic","score":92.82,"unit":"percent","family":"deepSearchQa","index":67.91713885031233,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":80.9,"unit":"percent","family":"gpqa-diamond","index":51.40837013886014,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"simpleVqa","benchmarkName":"SimpleVQA","category":"multimodal","score":79.2,"unit":"percent","family":"simpleVqa","index":69.00205468735565,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"browseComp","benchmarkName":"BrowseComp","category":"agentic","score":75.82,"unit":"percent","family":"browseComp","index":59.808295741466026,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaMmmuPro","benchmarkName":"Artificial Analysis MMMU-Pro","category":"multimodal","score":75.3,"unit":"percent","family":"mmmu-pro","index":53.14915859130881,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmmu-pro","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":73.7,"unit":"percent","family":"lcr","index":57.86218419483386,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaIfBench","benchmarkName":"Artificial Analysis IFBench","category":"instruction","score":67.3,"unit":"percent","family":"ifbench-family","index":51.91619377009045,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"clawEval","benchmarkName":"Claw-Eval","category":"agentic","score":67.1,"unit":"percent","family":"clawEval","index":62.1527041968274,"aboveFit":false,"run":null,"source":"Bowen Ye, Rang Li, Qibin Yang, Yuanxin Liu, Linli Yao, Hanglong Lv, Zhihui Xie, Chenxin An, Lei Li, Lingpeng Kong, Qi Liu, Zhifang Sui, Tong Yang","sourceUrl":"https://arxiv.org/abs/2604.06132","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"terminalBench21","benchmarkName":"Terminal-Bench 2.1","category":"agentic","score":59.5,"unit":"percent","family":"terminal-bench-2-1","index":55.940015393497184,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"swePro","benchmarkName":"SWE-bench Pro","category":"coding","score":56.3,"unit":"percent","family":"swePro","index":56.785025614796965,"aboveFit":false,"run":null,"source":"Xiang Deng, Jeff Da, Edwin Pan, Yannis Yiming He, Charles Ide, Kanak Garg, Niklas Lauffer, Andrew Park, Nitin Pasari, Chetan Rane, Karmini Sampath, Maya Krishnan, Srivatsa Kundurthy, Sean Hendryx, Zifan Wang, Chen Bo Calvin Zhang, Noah Jacobson, Bing Liu, Brad Kenstler","sourceUrl":"https://arxiv.org/abs/2509.16941","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gertLabs","benchmarkName":"Gert Labs Composite Game Benchmark","category":"agentic","score":51.57,"unit":"percent","family":"gertLabs","index":57.30385156956727,"aboveFit":false,"run":null,"source":"Gert Labs","sourceUrl":"https://gertlabs.com/rankings","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"toolathlon","benchmarkName":"Toolathlon","category":"agentic","score":49.5,"unit":"percent","family":"toolathlon","index":62.267303397859365,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hleWithTools","benchmarkName":"Humanity's Last Exam with tools","category":"agentic","score":47.2,"unit":"percent","family":"hleWithTools","index":57.655628118620385,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaSciCode","benchmarkName":"Artificial Analysis SciCode","category":"coding","score":43.9,"unit":"percent","family":"scicode-family","index":53.071839770118046,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/scicode","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gdpvalAaNormalized","benchmarkName":"GDPval-AA normalized","category":"agentic","score":25.8,"unit":"percent","family":"gdpvalAaNormalized","index":52.480175368842055,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"omniscienceAccuracy","benchmarkName":"Artificial Analysis Omniscience Accuracy","category":"knowledge","score":25.8,"unit":"percent","family":"omniscienceAccuracy","index":43.20455485536062,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":21.4,"unit":"percent","family":"hle","index":45.857109056951074,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"apexAgentsAa","benchmarkName":"APEX-Agents-AA","category":"agentic","score":14.8,"unit":"percent","family":"apexAgentsAa","index":52.246517949313514,"aboveFit":false,"run":null,"source":"Artificial Analysis / Mercor","sourceUrl":"https://artificialanalysis.ai/evaluations/apex-agents-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":2.3,"unit":"percent","family":"critpt","index":41.301647641180345,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":0.19999999999999998,"outputPerMillion":1.15,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":112,"timeToFirstToken":3.532,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":85,"benchmarkCount":21,"categoryCount":6,"confidence":"medium","state":"available"},"hosts":[{"host":"deepinfra","name":"Deep Infra","kind":"host","url":"https://deepinfra.com/models","inputPerMillion":0.2,"outputPerMillion":1.15,"cacheReadPerMillion":0.04,"contextTokens":262144,"maxOutputTokens":256000,"deprecated":true},{"host":"novita-ai","name":"Novita AI","kind":"host","url":"https://novita.ai/docs/guides/introduction","inputPerMillion":0.2,"outputPerMillion":1.15,"cacheReadPerMillion":0.04,"contextTokens":262144,"maxOutputTokens":256000,"deprecated":false},{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":0.2,"outputPerMillion":1.15,"cacheReadPerMillion":0.04,"contextTokens":262144,"maxOutputTokens":230400,"deprecated":false},{"host":"stepfun","name":"StepFun","kind":"maker","url":"https://platform.stepfun.ai/docs/en/overview/concept","inputPerMillion":0.2,"outputPerMillion":1.15,"cacheReadPerMillion":0.04,"contextTokens":256000,"maxOutputTokens":256000,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"inkling-small","providerId":"thinking-machines-lab","slug":"inkling-small","name":"Inkling-Small","provider":"Thinking Machines Lab","description":"Inkling-Small is a reasoning model from Thinking Machines Lab in the Inkling family. 35 benchmarks count toward its index, in 7 categories.","logoUrl":null,"releaseDate":"2026-07-30","contextWindowTokens":524288,"modalities":["text","image","audio"],"openWeights":true,"details":{"apiId":"thinkingmachines/inkling-small","maxOutputTokens":262144,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":false,"cacheReadPerMillion":0.09999999999999999,"weightsUrl":"https://huggingface.co/thinkingmachines/Inkling-Small"},"reasoning":true,"overallScore":55.143871239018026,"overallScoreRange":{"low":51.910642379226005,"high":58.37710009881005},"badges":[],"capabilityScores":{"agentic":55.72069016568139,"coding":54.94062165441569,"reasoning":55.74032880589557,"multimodal":54.68663317851449,"knowledge":51.9946246977625,"multilingual":null,"instruction":65.84040193105005,"math":55.25781615670602},"capabilityCounts":{"math":6,"knowledge":5,"coding":8,"reasoning":5,"instruction":1,"multimodal":3,"agentic":7},"benchmarks":[{"benchmarkId":"aime2026","benchmarkName":"AIME 2026","category":"math","score":95.5,"unit":"percent","family":"aime2026","index":54.04529613678822,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hmmtFeb2026","benchmarkName":"Harvard-MIT Mathematics Tournament February 2026","category":"math","score":90.2,"unit":"percent","family":"hmmtFeb2026","index":56.52436537781885,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":90,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":61.280554992542484,"aboveFit":false,"run":"xhigh effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":3.964807305493795,"costPerTest":null,"latency":null},{"benchmarkId":"gpqa","benchmarkName":"Graduate-Level Google-Proof Q&A","category":"knowledge","score":89.5,"unit":"percent","family":"gpqa-diamond","index":59.29605514317029,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gpqaDiamond","benchmarkName":"GPQA Diamond","category":"knowledge","score":89.5,"unit":"percent","family":"gpqa-diamond","index":59.29605514317029,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":89.5,"unit":"percent","family":"gpqa-diamond","index":59.29605514317029,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"arc-prize:v1-public-eval:overall","benchmarkName":"ARC-AGI-1 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":89.375,"unit":"percent","family":"arc-prize:v1-public-eval:overall","index":null,"aboveFit":false,"run":"xhigh effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.08376425325000007,"latency":null},{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":88.51010101010101,"unit":"percent","family":"gpqa-diamond","index":58.388146838704245,"aboveFit":false,"run":"xhigh effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.9100012831823399,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:lcb:overall","benchmarkName":"LiveCodeBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":85.931,"unit":"percent","family":"livecodebench","index":60.644260052933156,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","measuredAt":"2026-09-01","stderr":0.992,"costPerTest":0.020896,"latency":302.324},{"benchmarkId":"vals-ai:mmlu_pro:overall","benchmarkName":"MMLU Pro","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":85.566,"unit":"percent","family":"mmlu-pro","index":52.283646440023524,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","measuredAt":"2026-09-01","stderr":0.346,"costPerTest":0.005706,"latency":71.685},{"benchmarkId":"vals-ai:medscribe:overall","benchmarkName":"MedScribe","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":84.112,"unit":"percent","family":"vals-ai:medscribe:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medscribe","measuredAt":"2026-10-29","stderr":1.87,"costPerTest":0.019018,"latency":229.993},{"benchmarkId":"arc-prize:v1-semi-private:overall","benchmarkName":"ARC-AGI-1 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":84,"unit":"percent","family":"arc-prize:v1-semi-private:overall","index":65.60519151004009,"aboveFit":false,"run":"xhigh effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.11498211,"latency":null},{"benchmarkId":"vals-ai:gpqa:overall","benchmarkName":"GPQA Diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":83.586,"unit":"percent","family":"gpqa-diamond","index":53.87189594369468,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","measuredAt":"2026-09-01","stderr":2.036,"costPerTest":0.020022,"latency":259.758},{"benchmarkId":"vals-ai:legal_bench:overall","benchmarkName":"LegalBench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":82.986,"unit":"percent","family":"vals-ai:legal_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","measuredAt":"2026-10-13","stderr":0.459,"costPerTest":0.001332,"latency":17.655},{"benchmarkId":"ifBench","benchmarkName":"Instruction Following Benchmark","category":"instruction","score":82.2,"unit":"percent","family":"ifbench-family","index":65.84040193105005,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:swebench:overall","benchmarkName":"SWE-bench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":82.2,"unit":"percent","family":"swe-bench-verified","index":61.855160423291125,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/swebench","measuredAt":"2026-09-01","stderr":1.712,"costPerTest":0.13413,"latency":245.037},{"benchmarkId":"charxiv","benchmarkName":"CharXiv Reasoning","category":"multimodal","score":81.3,"unit":"percent","family":"charxiv","index":56.744123029969224,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"sweVerified","benchmarkName":"Software Engineering Benchmark Verified","category":"coding","score":80.2,"unit":"percent","family":"swe-bench-verified","index":60.380045267723744,"aboveFit":false,"run":null,"source":"Carlos E. Jimenez, John Yang, Alexander Wettig, Shunyu Yao, Kexin Pei, Ofir Press, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2310.06770","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mcpAtlas","benchmarkName":"MCP Atlas","category":"agentic","score":79.6,"unit":"percent","family":"mcpAtlas","index":64.66457275892283,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"browseComp","benchmarkName":"BrowseComp","category":"agentic","score":77.4,"unit":"percent","family":"browseComp","index":61.05652703494496,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"charxivNoTools","benchmarkName":"CharXiv Reasoning without tools","category":"multimodal","score":77.4,"unit":"percent","family":"charxivNoTools","index":55.85845385892034,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":75.7,"unit":"percent","family":"lcr","index":59.234345246005184,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:tax_eval_v2:overall","benchmarkName":"TaxEval v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":75.511,"unit":"percent","family":"vals-ai:tax_eval_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","measuredAt":"2026-09-01","stderr":0.837,"costPerTest":0.017501,"latency":241.592},{"benchmarkId":"mmmuPro","benchmarkName":"Massive Multi-discipline Multimodal Understanding Pro","category":"multimodal","score":74,"unit":"percent","family":"mmmu-pro","index":51.4573226466539,"aboveFit":false,"run":null,"source":"MMMU-Pro authors","sourceUrl":"https://arxiv.org/abs/2409.02813","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaMmmuPro","benchmarkName":"Artificial Analysis MMMU-Pro","category":"multimodal","score":74,"unit":"percent","family":"mmmu-pro","index":51.4573226466539,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmmu-pro","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:corp_fin_v2:overall","benchmarkName":"CorpFin v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":69.619,"unit":"percent","family":"vals-ai:corp_fin_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","measuredAt":"2026-08-12","stderr":0.906,"costPerTest":0.025455,"latency":45.316},{"benchmarkId":"terminalBench21","benchmarkName":"Terminal-Bench 2.1","category":"agentic","score":64.7,"unit":"percent","family":"terminal-bench-2-1","index":58.98126328875667,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:cyber:overall","benchmarkName":"CyberBench v1.1","category":null,"domain":"Other","eligible":false,"ranked":true,"score":64.107,"unit":"percent","family":"vals-ai:cyber:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/cyber","measuredAt":"2026-11-05","stderr":6.17,"costPerTest":0.251693,"latency":1175.122},{"benchmarkId":"vals-ai:mortgage_tax:overall","benchmarkName":"MortgageTax","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":62.282,"unit":"percent","family":"vals-ai:mortgage_tax:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mortgage_tax","measuredAt":"2026-09-01","stderr":0.952,"costPerTest":0.011526,"latency":112.214},{"benchmarkId":"vals-ai:public-benefits-bench:overall","benchmarkName":"Public Benefits Bench v1.1","category":null,"domain":"Public benefits","eligible":false,"ranked":true,"score":59.405,"unit":"percent","family":"vals-ai:public-benefits-bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/public-benefits-bench","measuredAt":"2026-10-12","stderr":1.277,"costPerTest":0.095937,"latency":521.336},{"benchmarkId":"swePro","benchmarkName":"SWE-bench Pro","category":"coding","score":55.9,"unit":"percent","family":"swePro","index":56.44438162232177,"aboveFit":false,"run":null,"source":"Xiang Deng, Jeff Da, Edwin Pan, Yannis Yiming He, Charles Ide, Kanak Garg, Niklas Lauffer, Andrew Park, Nitin Pasari, Chetan Rane, Karmini Sampath, Maya Krishnan, Srivatsa Kundurthy, Sean Hendryx, Zifan Wang, Chen Bo Calvin Zhang, Noah Jacobson, Bing Liu, Brad Kenstler","sourceUrl":"https://arxiv.org/abs/2509.16941","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:terminal-bench-2-1:overall","benchmarkName":"Terminal-Bench 2.1","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":55.056,"unit":"percent","family":"terminal-bench-2-1","index":53.34091815377927,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2-1","measuredAt":"2026-10-01","stderr":2.339,"costPerTest":0.127848,"latency":603.689},{"benchmarkId":"toolathlonVerified","benchmarkName":"Toolathlon-Verified","category":"agentic","score":54.4,"unit":"percent","family":"toolathlonVerified","index":53.50517632196294,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaSciCode","benchmarkName":"Artificial Analysis SciCode","category":"coding","score":49.7,"unit":"percent","family":"scicode-family","index":60.63324841932351,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/scicode","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"sciCode","benchmarkName":"Scientific Code Benchmark","category":"coding","score":48.7,"unit":"percent","family":"scicode-family","index":59.32955727290877,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hle","benchmarkName":"Humanity's Last Exam","category":"knowledge","score":47.8,"unit":"percent","family":"hle","index":72.34403752573765,"aboveFit":true,"run":null,"source":"Center for AI Safety, Scale AI, and thousands of expert contributors","sourceUrl":"https://lastexam.ai/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tiers-1-3-v2-private:overall","benchmarkName":"FrontierMath-Tiers-1-3-v2-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":46.31578947368421,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tiers-1-3-v2-private:overall","index":55.27309135494766,"aboveFit":false,"run":"xhigh effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.958888847874604,"costPerTest":null,"latency":null},{"benchmarkId":"aaEnterpriseOpsGym","benchmarkName":"Artificial Analysis EnterpriseOps-Gym","category":"agentic","score":42.7,"unit":"percent","family":"aaEnterpriseOpsGym","index":null,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/enterprise-ops-gym-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:fabv2:overall","benchmarkName":"Finance Agent (v2)","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":41.257,"unit":"percent","family":"vals-ai:fabv2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/fabv2","measuredAt":"2026-10-29","stderr":0.832,"costPerTest":0.230271,"latency":994.255},{"benchmarkId":"arc-prize:v2-semi-private:overall","benchmarkName":"ARC-AGI-2 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":40.138888888888886,"unit":"percent","family":"arc-prize:v2-semi-private:overall","index":57.617672149757695,"aboveFit":false,"run":"xhigh effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.23255201249999993,"latency":null},{"benchmarkId":"arc-prize:v2-public-eval:overall","benchmarkName":"ARC-AGI-2 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":39.30555555555556,"unit":"percent","family":"arc-prize:v2-public-eval:overall","index":null,"aboveFit":false,"run":"xhigh effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.2580068700000001,"latency":null},{"benchmarkId":"vals-ai:medcode:overall","benchmarkName":"MedCode","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":37.893,"unit":"percent","family":"vals-ai:medcode:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medcode","measuredAt":"2026-10-29","stderr":2.206,"costPerTest":0.016656,"latency":191.049},{"benchmarkId":"vals-ai:skillsbench:overall","benchmarkName":"SkillsBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":33.582,"unit":"percent","family":"vals-ai:skillsbench:overall","index":50.139245182182144,"aboveFit":false,"run":"OpenHands","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/skillsbench","measuredAt":"2026-10-29","stderr":3.66,"costPerTest":0.148059,"latency":700.919},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":33.3,"unit":"percent","family":"hle","index":57.796292722805624,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"omniscienceAccuracy","benchmarkName":"Artificial Analysis Omniscience Accuracy","category":"knowledge","score":31.8,"unit":"percent","family":"omniscienceAccuracy","index":50.76609075971784,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:emb:overall","benchmarkName":"EMB","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":31.762,"unit":"percent","family":"vals-ai:emb:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/emb","measuredAt":"2026-10-20","stderr":2.956,"costPerTest":0.249301,"latency":1173.625},{"benchmarkId":"hleNoTools","benchmarkName":"Humanity's Last Exam without tools","category":"knowledge","score":31.6,"unit":"percent","family":"hle","index":56.09069505625498,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gdpvalAaNormalized","benchmarkName":"GDPval-AA normalized","category":"agentic","score":31.2,"unit":"percent","family":"gdpvalAaNormalized","index":56.341049881977945,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:legal_research:overall","benchmarkName":"Legal Research Bench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":25.481,"unit":"percent","family":"vals-ai:legal_research:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_research","measuredAt":"2026-11-09","stderr":3.029,"costPerTest":0.141548,"latency":774.823},{"benchmarkId":"epoch-ai:epoch-simpleqa-verified:overall","benchmarkName":"SimpleQA Verified","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":19.1,"unit":"percent","family":"epoch-ai:epoch-simpleqa-verified:overall","index":39.83103842309521,"aboveFit":false,"run":"xhigh effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.2436787112179482,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:vibe-code:overall","benchmarkName":"Vibe Code Bench v1.1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":19.052,"unit":"percent","family":"vals-ai:vibe-code:overall","index":49.123783520331024,"aboveFit":false,"run":"OpenHands","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","measuredAt":"2026-11-09","stderr":3.403,"costPerTest":0.432264,"latency":2949.469},{"benchmarkId":"epoch-ai:epoch-frontiermath-tier-4-v2-private:overall","benchmarkName":"FrontierMath-Tier-4-v2-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":17.073170731707318,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tier-4-v2-private:overall","index":55.1955002815125,"aboveFit":false,"run":"xhigh effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":5.949419959829496,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:tax_agent_bench:overall","benchmarkName":"Tax Agent Bench","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":15.389,"unit":"percent","family":"vals-ai:tax_agent_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench","measuredAt":"2026-11-09","stderr":2.365,"costPerTest":0.091767,"latency":385.252},{"benchmarkId":"vals-ai:code-migration:overall","benchmarkName":"Code Migration","category":"coding","domain":null,"eligible":true,"ranked":true,"score":13.675,"unit":"percent","family":"vals-ai:code-migration:overall","index":52.45384572600164,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/code-migration","measuredAt":"2026-10-29","stderr":3.81,"costPerTest":0.596991,"latency":1232.702},{"benchmarkId":"vals-ai:ioi:overall","benchmarkName":"IOI","category":"coding","domain":null,"eligible":true,"ranked":true,"score":9.333,"unit":"percent","family":"vals-ai:ioi:overall","index":49.62045143993224,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/ioi","measuredAt":"2026-10-29","stderr":4.475,"costPerTest":0.337871,"latency":678.056},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":8.3,"unit":"percent","family":"critpt","index":52.91694754137367,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:proof_bench:overall","benchmarkName":"ProofBench v1.1","category":"math","domain":null,"eligible":true,"ranked":true,"score":6,"unit":"percent","family":"vals-ai:proof_bench:overall","index":49.22808879662642,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/proof_bench","measuredAt":"2026-11-09","stderr":2.387,"costPerTest":0.082234,"latency":507.301},{"benchmarkId":"epoch-ai:epoch-mystery-game-puzzles:overall","benchmarkName":"Mystery Game Puzzles","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":6,"unit":"percent","family":"epoch-ai:epoch-mystery-game-puzzles:overall","index":43.327487582301224,"aboveFit":false,"run":"xhigh effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.3868325657594203,"costPerTest":null,"latency":null},{"benchmarkId":"lmarena:agent-bash-recovery:overall","benchmarkName":"Agent Arena command recovery","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":5.433972301988699,"unit":"index","family":"agent-arena","index":70.9983784339242,"aboveFit":false,"run":null,"source":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset","measuredAt":"2026-10-08","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:hlab:overall","benchmarkName":"HLAB-Vals-1.1","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":2.917,"unit":"percent","family":"vals-ai:hlab:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/hlab","measuredAt":"2026-11-09","stderr":1.362,"costPerTest":0.17869,"latency":455.265},{"benchmarkId":"vals-ai:programbench:overall","benchmarkName":"ProgramBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":0.5,"unit":"percent","family":"vals-ai:programbench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/programbench","measuredAt":"2026-11-09","stderr":0.5,"costPerTest":0.479317,"latency":1710.815},{"benchmarkId":"vals-ai:terminal-bench-4:overall","benchmarkName":"Terminal-Bench 4.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":0,"unit":"percent","family":"terminal-bench-4","index":54.516013465855735,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-4","measuredAt":"2026-11-09","stderr":0,"costPerTest":0.321679,"latency":705.115},{"benchmarkId":"lmarena:agent-task-outcome:overall","benchmarkName":"Agent Arena task outcome","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":-27.647282280595043,"unit":"index","family":"agent-arena","index":43.80040097483733,"aboveFit":false,"run":null,"source":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset","measuredAt":"2026-11-11","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lmarena:agent-steerability:overall","benchmarkName":"Agent Arena steerability","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":-27.718147269256654,"unit":"index","family":"agent-arena","index":43.74213884455912,"aboveFit":false,"run":null,"source":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset","measuredAt":"2026-11-11","stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":0.44999999999999996,"outputPerMillion":1.2,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":80,"timeToFirstToken":0.688,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":{"elo":1413.994063429453,"eloUpper":1418.5123939639798,"eloLower":1409.4757328949263,"votes":37665,"rank":137,"rankUpper":null,"rankLower":null,"arenaName":"Inkling-Small","organization":"thinky","updatedAt":"2026-11-09"},"quality":{"coveragePercent":95,"benchmarkCount":35,"categoryCount":7,"confidence":"high","state":"available"},"hosts":[{"host":"deepinfra","name":"Deep Infra","kind":"host","url":"https://deepinfra.com/models","inputPerMillion":0.45,"outputPerMillion":1.2,"cacheReadPerMillion":0.1,"contextTokens":524288,"maxOutputTokens":1048576,"deprecated":false},{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":0.45,"outputPerMillion":1.2,"cacheReadPerMillion":0.1,"contextTokens":524288,"maxOutputTokens":262144,"deprecated":false},{"host":"arcee-ai","name":"Arcee AI","kind":"host","url":"https://docs.arcee.ai","inputPerMillion":0.5,"outputPerMillion":1.2,"cacheReadPerMillion":0.1,"contextTokens":262144,"maxOutputTokens":262144,"deprecated":true}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:arc-prize-foundation","name":"ARC Prize Foundation","url":"https://arcprize.org","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:lmarena","name":"LMArena","url":"https://huggingface.co","license":"CC BY 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gpt5","providerId":"openai","slug":"openai-gpt-5","name":"GPT-5","provider":"OpenAI","description":"GPT-5 is a model from OpenAI. 25 benchmarks count toward its index, in 6 categories.","logoUrl":null,"releaseDate":"2025-08-07T17:23:33.000Z","contextWindowTokens":400000,"modalities":["text","image","file"],"openWeights":null,"details":{"apiId":"openai/gpt-5","maxOutputTokens":128000,"knowledgeCutoff":"2024-09-30","retiresAt":"2026-12-11","tools":true,"structuredOutput":true,"cacheReadPerMillion":0.125,"weightsUrl":null},"reasoning":null,"overallScore":55.02726529918664,"overallScoreRange":{"low":50.793265299186636,"high":59.26126529918664},"badges":[],"capabilityScores":{"agentic":53.09914557918639,"coding":52.475717048111065,"reasoning":55.38999959572726,"multimodal":61.221818871843,"knowledge":58.38591912890185,"multilingual":null,"instruction":null,"math":55.28179178277721},"capabilityCounts":{"math":8,"agentic":3,"knowledge":3,"coding":5,"multimodal":1,"reasoning":5},"benchmarks":[{"benchmarkId":"epoch-ai:epoch-math-level-5:overall","benchmarkName":"MATH level 5","category":"math","domain":null,"eligible":true,"ranked":true,"score":98.13066465256797,"unit":"percent","family":"epoch-ai:epoch-math-level-5:overall","index":46.66540921007715,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0.31922231523012307,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:medqa:overall","benchmarkName":"MedQA","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":96.317,"unit":"percent","family":"vals-ai:medqa:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medqa","measuredAt":"2026-04-16","stderr":0.172,"costPerTest":0.020888,"latency":35.496},{"benchmarkId":"vals-ai:math500:overall","benchmarkName":"MATH 500","category":"math","domain":null,"eligible":true,"ranked":true,"score":96,"unit":"percent","family":"math-500","index":45.57557164964577,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/math500","measuredAt":"2026-01-09","stderr":0.876,"costPerTest":0.020071,"latency":32.217},{"benchmarkId":"tau2-bench:telecom:overall","benchmarkName":"τ²-bench Telecom","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":95.83333333333334,"unit":"percent","family":"tau2-bench","index":66.38186337115944,"aboveFit":false,"run":"Sierra","source":"Sierra Research","sourceUrl":"https://github.com/sierra-research/tau2-bench","measuredAt":"2026-03-02","stderr":null,"costPerTest":0.16471152192982458,"latency":null},{"benchmarkId":"vals-ai:aime:overall","benchmarkName":"AIME","category":"math","domain":null,"eligible":true,"ranked":true,"score":93.374,"unit":"percent","family":"vals-ai:aime:overall","index":56.311276825029246,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/aime","measuredAt":"2026-04-16","stderr":1.305,"costPerTest":0.092693,"latency":292.202},{"benchmarkId":"vals-ai:mgsm:overall","benchmarkName":"MGSM","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":92.836,"unit":"percent","family":"vals-ai:mgsm:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mgsm","measuredAt":"2026-01-09","stderr":1.559,"costPerTest":0.014278,"latency":20.353},{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":91.38888888888889,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":62.08860227828288,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":3.658872186755414,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:mmlu_pro:overall","benchmarkName":"MMLU Pro","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":86.544,"unit":"percent","family":"mmlu-pro","index":53.5158093764327,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","measuredAt":"2026-09-01","stderr":0.336,"costPerTest":0.024135,"latency":37.239},{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":86.17424242424242,"unit":"percent","family":"gpqa-diamond","index":56.24576117127798,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.0787836099851997,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:legal_bench:overall","benchmarkName":"LegalBench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":86.023,"unit":"percent","family":"vals-ai:legal_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","measuredAt":"2026-10-13","stderr":0.379,"costPerTest":null,"latency":18.59},{"benchmarkId":"vals-ai:lcb:overall","benchmarkName":"LiveCodeBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":85.911,"unit":"percent","family":"livecodebench","index":60.6276652283531,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","measuredAt":"2026-09-01","stderr":0.974,"costPerTest":0.063751,"latency":150.441},{"benchmarkId":"vals-ai:gpqa:overall","benchmarkName":"GPQA Diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":85.606,"unit":"percent","family":"gpqa-diamond","index":55.724584747032644,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","measuredAt":"2026-09-01","stderr":1.764,"costPerTest":0.072708,"latency":169.716},{"benchmarkId":"vals-ai:medscribe:overall","benchmarkName":"MedScribe","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":83.65,"unit":"percent","family":"vals-ai:medscribe:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medscribe","measuredAt":"2026-10-29","stderr":1.936,"costPerTest":0.1015,"latency":184.536},{"benchmarkId":"tau2-bench:retail:overall","benchmarkName":"τ²-bench Retail","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":81.57894736842105,"unit":"percent","family":"tau2-bench","index":56.92303611789093,"aboveFit":false,"run":"Sierra","source":"Sierra Research","sourceUrl":"https://github.com/sierra-research/tau2-bench","measuredAt":"2026-04-30","stderr":null,"costPerTest":0.10552710142543861,"latency":null},{"benchmarkId":"vals-ai:mmmu:overall","benchmarkName":"MMMU Pro","category":"multimodal","domain":null,"eligible":true,"ranked":true,"score":81.503,"unit":"percent","family":"mmmu-pro","index":61.221818871843,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmmu","measuredAt":"2026-09-01","stderr":0.934,"costPerTest":0.032532,"latency":72.114},{"benchmarkId":"epoch-ai:epoch-swe-bench-verified:overall","benchmarkName":"SWE-Bench verified","category":"coding","domain":null,"eligible":true,"ranked":true,"score":73.55371900826447,"unit":"percent","family":"swe-bench-verified","index":55.478030358189486,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.0068321817102888,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:tax_eval_v2:overall","benchmarkName":"TaxEval v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":73.385,"unit":"percent","family":"vals-ai:tax_eval_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","measuredAt":"2026-09-01","stderr":0.868,"costPerTest":0.053873,"latency":78.668},{"benchmarkId":"vals-ai:swebench:overall","benchmarkName":"SWE-bench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":69,"unit":"percent","family":"swe-bench-verified","index":52.11940039654639,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/swebench","measuredAt":"2026-09-01","stderr":2.07,"costPerTest":0.27029,"latency":232.842},{"benchmarkId":"vals-ai:case_law_v2:overall","benchmarkName":"CaseLaw v2","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":66.452,"unit":"percent","family":"vals-ai:case_law_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/case_law_v2","measuredAt":"2026-05-04","stderr":0.652,"costPerTest":0.066591,"latency":123.423},{"benchmarkId":"arc-prize:v1-public-eval:overall","benchmarkName":"ARC-AGI-1 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":65.88000000000001,"unit":"percent","family":"arc-prize:v1-public-eval:overall","index":null,"aboveFit":false,"run":"high effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.3924,"latency":null},{"benchmarkId":"arc-prize:v1-semi-private:overall","benchmarkName":"ARC-AGI-1 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":65.67,"unit":"percent","family":"arc-prize:v1-semi-private:overall","index":57.17822874251941,"aboveFit":false,"run":"high effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.5087,"latency":null},{"benchmarkId":"vals-ai:mortgage_tax:overall","benchmarkName":"MortgageTax","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":65.454,"unit":"percent","family":"vals-ai:mortgage_tax:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mortgage_tax","measuredAt":"2026-09-01","stderr":0.879,"costPerTest":0.028906,"latency":62.828},{"benchmarkId":"swe-bench:swe-bench-verified:overall","benchmarkName":"SWE-bench Verified","category":"coding","domain":null,"eligible":true,"ranked":true,"score":65,"unit":"percent","family":"swe-bench-verified","index":49.16917008541162,"aboveFit":false,"run":"medium effort · mini-SWE-agent","source":"SWE-bench team","sourceUrl":"https://www.swebench.com/","measuredAt":"2026-02-26","stderr":null,"costPerTest":0.2803830175,"latency":null},{"benchmarkId":"swebench-experiments:swe-bench-verified:overall","benchmarkName":"SWE-bench Verified","category":"coding","domain":null,"eligible":true,"ranked":true,"score":65,"unit":"percent","family":"swe-bench-verified","index":49.16917008541162,"aboveFit":false,"run":"medium effort · mini-SWE-agent","source":"SWE-bench team","sourceUrl":"https://github.com/SWE-bench/experiments/tree/main/evaluation/verified","measuredAt":"2026-09-01","stderr":null,"costPerTest":0.2803830175,"latency":null},{"benchmarkId":"tau2-bench:airline:overall","benchmarkName":"τ²-bench Airline","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":62.5,"unit":"percent","family":"tau2-bench","index":44.26275964043924,"aboveFit":false,"run":"Sierra","source":"Sierra Research","sourceUrl":"https://github.com/sierra-research/tau2-bench","measuredAt":"2026-03-02","stderr":null,"costPerTest":0.13399991125,"latency":null},{"benchmarkId":"vals-ai:corp_fin_v2:overall","benchmarkName":"CorpFin v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":61.072,"unit":"percent","family":"vals-ai:corp_fin_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","measuredAt":"2026-08-12","stderr":0.961,"costPerTest":0.086116,"latency":160.705},{"benchmarkId":"epoch-ai:epoch-frontiermath-2025-02-28-public:overall","benchmarkName":"FrontierMath-2025-02-28-Public","category":"math","domain":null,"eligible":false,"ranked":true,"score":60,"unit":"percent","family":"epoch-ai:epoch-frontiermath-2025-02-28-public:overall","index":null,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":16.329931618554518,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tiers-1-3-v2-private:overall","benchmarkName":"FrontierMath-Tiers-1-3-v2-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":55.43859649122807,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tiers-1-3-v2-private:overall","index":60.07489644047984,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.9493504108183504,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:finance_agent:overall","benchmarkName":"Finance Agent v1.1","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":52.151,"unit":"percent","family":"vals-ai:finance_agent:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/finance_agent","measuredAt":"2026-06-04","stderr":2.879,"costPerTest":0.586458,"latency":926.611},{"benchmarkId":"epoch-ai:epoch-simpleqa-verified:overall","benchmarkName":"SimpleQA Verified","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":50.1,"unit":"percent","family":"epoch-ai:epoch-simpleqa-verified:overall","index":65.65677505111753,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.5819268290576818,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:medcode:overall","benchmarkName":"MedCode","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":49.634,"unit":"percent","family":"vals-ai:medcode:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medcode","measuredAt":"2026-10-29","stderr":2.098,"costPerTest":0.045858,"latency":58.268},{"benchmarkId":"vals-ai:terminal-bench:overall","benchmarkName":"Terminal-Bench 1.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":48.75,"unit":"percent","family":"vals-ai:terminal-bench:overall","index":53.497452857119576,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench","measuredAt":"2026-01-12","stderr":5.588,"costPerTest":0.100635,"latency":957.628},{"benchmarkId":"vals-ai:terminal-bench-2:overall","benchmarkName":"Terminal-Bench 2.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":37.079,"unit":"percent","family":"vals-ai:terminal-bench-2:overall","index":48.87694776254867,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2","measuredAt":"2026-06-04","stderr":5.149,"costPerTest":0.477064,"latency":986.102},{"benchmarkId":"epoch-ai:epoch-chess-puzzles:overall","benchmarkName":"Chess Puzzles","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":36,"unit":"percent","family":"epoch-ai:epoch-chess-puzzles:overall","index":61.309222750542396,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":4.824181513244218,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-2025-02-28-private:overall","benchmarkName":"FrontierMath-2025-02-28-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":32.41379310344827,"unit":"percent","family":"epoch-ai:epoch-frontiermath-2025-02-28-private:overall","index":58.528344694735424,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.753246166483919,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-mystery-game-puzzles:overall","benchmarkName":"Mystery Game Puzzles","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":23,"unit":"percent","family":"epoch-ai:epoch-mystery-game-puzzles:overall","index":56.492540826289144,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":4.229525846816508,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tier-4-v2-private:overall","benchmarkName":"FrontierMath-Tier-4-v2-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":21.951219512195124,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tier-4-v2-private:overall","index":57.11184193415877,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":6.5445892024384085,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:vibe-code:overall","benchmarkName":"Vibe Code Bench v1.1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":20.088,"unit":"percent","family":"vals-ai:vibe-code:overall","index":49.49758859711177,"aboveFit":false,"run":"OpenHands","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","measuredAt":"2026-11-09","stderr":3.409,"costPerTest":1.525167,"latency":1852.149},{"benchmarkId":"vals-ai:ioi-v1:overall","benchmarkName":"IOI v1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":20,"unit":"percent","family":"vals-ai:ioi-v1:overall","index":50.09152887465671,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/ioi-v1","measuredAt":"2026-08-09","stderr":4.581,"costPerTest":1.629264,"latency":2608.252},{"benchmarkId":"epoch-ai:epoch-ebr-bench:overall","benchmarkName":"EBR-bench","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":12.698400000000001,"unit":"percent","family":"epoch-ai:epoch-ebr-bench:overall","index":58.254422105417895,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-private:overall","benchmarkName":"FrontierMath-Tier-4-2025-07-01-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":12.5,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-private:overall","index":55.89839122980863,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":4.824031156174749,"costPerTest":null,"latency":null},{"benchmarkId":"arc-prize:v2-semi-private:overall","benchmarkName":"ARC-AGI-2 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":9.86,"unit":"percent","family":"arc-prize:v2-semi-private:overall","index":43.71558355386745,"aboveFit":false,"run":"high effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.7302,"latency":null},{"benchmarkId":"arc-prize:v2-public-eval:overall","benchmarkName":"ARC-AGI-2 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":9.58,"unit":"percent","family":"arc-prize:v2-public-eval:overall","index":null,"aboveFit":false,"run":"high effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.774,"latency":null},{"benchmarkId":"epoch-ai:epoch-mirrorcode:overall","benchmarkName":"MirrorCode","category":"coding","domain":null,"eligible":true,"ranked":true,"score":4.444444444444445,"unit":"percent","family":"epoch-ai:epoch-mirrorcode:overall","index":51.51751729945473,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":3.4375250520170986,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-public:overall","benchmarkName":"FrontierMath-Tier-4-2025-07-01-Public","category":"math","domain":null,"eligible":false,"ranked":true,"score":0,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-public:overall","index":null,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":1.25,"outputPerMillion":10,"batch":{"inputPerMillion":0.625,"outputPerMillion":5},"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":76,"timeToFirstToken":4.5795,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":{"elo":1406.360510180139,"eloUpper":1410.7879860163412,"eloLower":1401.9330343439365,"votes":31638,"rank":149,"rankUpper":null,"rankLower":null,"arenaName":"GPT-5-high","organization":"openai","updatedAt":"2026-11-09"},"quality":{"coveragePercent":90,"benchmarkCount":25,"categoryCount":6,"confidence":"medium","state":"available"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:sierra-research","name":"Sierra Research","url":"https://github.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:arc-prize-foundation","name":"ARC Prize Foundation","url":"https://arcprize.org","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:swe-bench-team","name":"SWE-bench team","url":"https://www.swebench.com","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gemini-3-flash","providerId":"google","slug":"gemini-3-flash","name":"Gemini 3 Flash","provider":"Google","description":"Gemini 3 Flash is a standard model from Google. 21 benchmarks count toward its index, in 7 categories.","logoUrl":null,"releaseDate":"2025-12-01","contextWindowTokens":1000000,"modalities":["text"],"openWeights":false,"details":null,"reasoning":false,"overallScore":54.97879244866522,"overallScoreRange":{"low":50.95020916354922,"high":59.00737573378122},"badges":[],"capabilityScores":{"agentic":48.5419795783268,"coding":56.84952172183685,"reasoning":50.82805871188137,"multimodal":67.2548424652453,"knowledge":60.714052480449865,"multilingual":null,"instruction":40.51516426917052,"math":57.27555643305166},"capabilityCounts":{"math":5,"agentic":4,"knowledge":4,"multimodal":2,"coding":1,"reasoning":4,"instruction":1},"benchmarks":[{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":95.55555555555556,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":64.51274413550408,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":3.1067790907534745,"costPerTest":null,"latency":null},{"benchmarkId":"aaGlobalMmluLite","benchmarkName":"Artificial Analysis Global-MMLU-Lite","category":"multilingual","score":92.7,"unit":"percent","family":"aaGlobalMmluLite","index":null,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/global-mmlu-lite","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"tau2-bench:telecom:overall","benchmarkName":"τ²-bench Telecom","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":91.23,"unit":"percent","family":"tau2-bench","index":63.32721514594698,"aboveFit":false,"run":"high effort · Sierra","source":"Sierra Research","sourceUrl":"https://github.com/sierra-research/tau2-bench","measuredAt":"2026-03-02","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":89.39393939393939,"unit":"percent","family":"gpqa-diamond","index":59.19877925340607,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.1938047738853106,"costPerTest":null,"latency":null},{"benchmarkId":"tau2-bench:airline:overall","benchmarkName":"τ²-bench Airline","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":82.5,"unit":"percent","family":"tau2-bench","index":57.53422187887136,"aboveFit":false,"run":"high effort · Sierra","source":"Sierra Research","sourceUrl":"https://github.com/sierra-research/tau2-bench","measuredAt":"2026-03-02","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":81.2,"unit":"percent","family":"gpqa-diamond","index":51.683521941336075,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaMmmuPro","benchmarkName":"Artificial Analysis MMMU-Pro","category":"multimodal","score":78.6,"unit":"percent","family":"mmmu-pro","index":57.44381906620205,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmmu-pro","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"tau2-bench:retail:overall","benchmarkName":"τ²-bench Retail","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":76.75,"unit":"percent","family":"tau2-bench","index":53.71867648532212,"aboveFit":false,"run":"high effort · Sierra","source":"Sierra Research","sourceUrl":"https://github.com/sierra-research/tau2-bench","measuredAt":"2026-04-30","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-swe-bench-verified:overall","benchmarkName":"SWE-Bench verified","category":"coding","domain":null,"eligible":true,"ranked":true,"score":75.41322314049587,"unit":"percent","family":"swe-bench-verified","index":56.84952172183685,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.9593002139508684,"costPerTest":null,"latency":null},{"benchmarkId":"parseBenchLlamaIndex","benchmarkName":"ParseBench: document parsing for AI agents (LlamaIndex leaderboard)","category":"multimodal","score":74.07,"unit":"percent","family":"parseBenchLlamaIndex","index":77.06586586428857,"aboveFit":true,"run":null,"source":"Boyang Zhang, Sebastián G. Acosta, Preston Carlson, Sacha Bron, Pierre-Loïc Doulcet, Daniel B. Ospina and Simon Suo (LlamaIndex)","sourceUrl":"https://arxiv.org/abs/2604.08538","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"swe-bench:swe-bench-multilingual:overall","benchmarkName":"SWE-bench Multilingual","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":72.7,"unit":"percent","family":"swe-bench-multilingual","index":null,"aboveFit":false,"run":"mini-SWE-agent","source":"SWE-bench team","sourceUrl":"https://www.swebench.com/","measuredAt":"2026-02-20","stderr":null,"costPerTest":0.3518355868333333,"latency":null},{"benchmarkId":"swebench-experiments:swe-bench-multilingual:overall","benchmarkName":"SWE-bench Multilingual","category":"coding","domain":null,"eligible":true,"ranked":true,"score":72.7,"unit":"percent","family":"swe-bench-multilingual","index":null,"aboveFit":false,"run":"mini-SWE-agent","source":"SWE-bench team","sourceUrl":"https://github.com/SWE-bench/experiments/tree/main/evaluation/multilingual","measuredAt":"2026-09-02","stderr":null,"costPerTest":0.3518355868333333,"latency":null},{"benchmarkId":"epoch-ai:epoch-simpleqa-verified:overall","benchmarkName":"SimpleQA Verified","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":66.8,"unit":"percent","family":"epoch-ai:epoch-simpleqa-verified:overall","index":79.56934929911665,"aboveFit":true,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.4899597242811566,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-2025-02-28-public:overall","benchmarkName":"FrontierMath-2025-02-28-Public","category":"math","domain":null,"eligible":false,"ranked":true,"score":60,"unit":"percent","family":"epoch-ai:epoch-frontiermath-2025-02-28-public:overall","index":null,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":16.329931618554518,"costPerTest":null,"latency":null},{"benchmarkId":"gertLabs","benchmarkName":"Gert Labs Composite Game Benchmark","category":"agentic","score":56.63,"unit":"percent","family":"gertLabs","index":61.40101850901473,"aboveFit":false,"run":null,"source":"Gert Labs","sourceUrl":"https://gertlabs.com/rankings","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":55.3,"unit":"percent","family":"lcr","index":45.23830252405771,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaIfBench","benchmarkName":"Artificial Analysis IFBench","category":"instruction","score":55.1,"unit":"percent","family":"ifbench-family","index":40.51516426917052,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tiers-1-3-v2-private:overall","benchmarkName":"FrontierMath-Tiers-1-3-v2-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":51.2280701754386,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tiers-1-3-v2-private:overall","index":57.85867870869576,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.966059084324674,"costPerTest":null,"latency":null},{"benchmarkId":"clawEval","benchmarkName":"Claw-Eval","category":"agentic","score":49.2,"unit":"percent","family":"clawEval","index":36.24930479576704,"aboveFit":false,"run":null,"source":"Bowen Ye, Rang Li, Qibin Yang, Yuanxin Liu, Linli Yao, Hanglong Lv, Zhihui Xie, Chenxin An, Lei Li, Lingpeng Kong, Qi Liu, Zhifang Sui, Tong Yang","sourceUrl":"https://arxiv.org/abs/2604.06132","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"omniscienceAccuracy","benchmarkName":"Artificial Analysis Omniscience Accuracy","category":"knowledge","score":45.8,"unit":"percent","family":"omniscienceAccuracy","index":68.40967453655135,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"tau2Bench","benchmarkName":"τ²-Bench Tool-Agent-User Evaluation","category":"agentic","score":43.3,"unit":"percent","family":"tau2-bench","index":31.522155891544408,"aboveFit":false,"run":null,"source":"Victor Barres, Honghua Dong, Soham Ray, Xujie Si, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2506.07982","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-chess-puzzles:overall","benchmarkName":"Chess Puzzles","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":40,"unit":"percent","family":"epoch-ai:epoch-chess-puzzles:overall","index":64.34528353114278,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":4.92365963917331,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-2025-02-28-private:overall","benchmarkName":"FrontierMath-2025-02-28-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":35.6401384083045,"unit":"percent","family":"epoch-ai:epoch-frontiermath-2025-02-28-private:overall","index":61.37649014483267,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.8221558784583785,"costPerTest":null,"latency":null},{"benchmarkId":"tau2-bench:banking-knowledge:overall","benchmarkName":"τ²-bench Banking","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":27.32,"unit":"percent","family":"tau2-bench","index":20.918257563037148,"aboveFit":false,"run":"high effort · Sierra","source":"Sierra Research","sourceUrl":"https://github.com/sierra-research/tau2-bench","measuredAt":"2026-10-20","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-mystery-game-puzzles:overall","benchmarkName":"Mystery Game Puzzles","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":20,"unit":"percent","family":"epoch-ai:epoch-mystery-game-puzzles:overall","index":54.16929613617363,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":4.020151261036848,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tier-4-v2-private:overall","benchmarkName":"FrontierMath-Tier-4-v2-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":17.073170731707318,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tier-4-v2-private:overall","index":55.1955002815125,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":5.949419959829497,"costPerTest":null,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":15,"unit":"percent","family":"hle","index":39.43603548876039,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"jobBench","benchmarkName":"JobBench","category":"agentic","score":11.4,"unit":"percent","family":"jobBench","index":42.79891852320333,"aboveFit":false,"run":null,"source":"Yuetai Li, Yichen Feng, Zhangchen Xu, Zixian Ma, Kaiyuan Zheng, Fengqing Jiang, Xinghua Sun, Rulin Shao, Zichen Chen, Yue Huang, Xinyang Han, Brian Lee, Kayla Xu, Shenglai Zeng, Hang Hua, Xiangliang Zhang, Basel Alomair, Ranjay Krishna, Luke Zettlemoyer, Pang Wei Koh, Bhaskar Ramasubramanian, Luyao Niu, Xiang Yue, Radha Poovendran","sourceUrl":"https://arxiv.org/abs/2605.26329","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-private:overall","benchmarkName":"FrontierMath-Tier-4-2025-07-01-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":4.166666666666666,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-private:overall","index":47.43436889471329,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.914766351555603,"costPerTest":null,"latency":null},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":1.4,"unit":"percent","family":"critpt","index":39.55935265615135,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-public:overall","benchmarkName":"FrontierMath-Tier-4-2025-07-01-Public","category":"math","domain":null,"eligible":false,"ranked":true,"score":0,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-public:overall","index":null,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":0.5,"outputPerMillion":3,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":198,"timeToFirstToken":0.91,"source":"BenchLM","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":{"elo":1465.8447700363245,"eloUpper":1470.167888089586,"eloLower":1461.5216519830633,"votes":31133,"rank":37,"rankUpper":null,"rankLower":null,"arenaName":"Gemini 3 Flash","organization":"google","updatedAt":"2026-11-09"},"quality":{"coveragePercent":95,"benchmarkCount":21,"categoryCount":7,"confidence":"medium","state":"available"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:sierra-research","name":"Sierra Research","url":"https://github.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:swe-bench-team","name":"SWE-bench team","url":"https://www.swebench.com","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"mimo-v2-5","providerId":"xiaomi","slug":"mimo-v2-5","name":"MiMo-V2.5","provider":"Xiaomi","description":"MiMo-V2.5 is a reasoning model from Xiaomi. 14 benchmarks count toward its index, in 5 categories.","logoUrl":null,"releaseDate":"2026-04-22","contextWindowTokens":1000000,"modalities":["text","audio","image","video"],"openWeights":true,"details":{"apiId":"xiaomi/mimo-v2.5","maxOutputTokens":65536,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.08,"weightsUrl":"https://huggingface.co/XiaomiMiMo/MiMo-V2.5"},"reasoning":true,"overallScore":54.76800364455338,"overallScoreRange":{"low":47.94638936282948,"high":61.589617926277285},"badges":[],"capabilityScores":{"agentic":56.512931509274814,"coding":55.48020772493162,"reasoning":null,"multimodal":57.15094343064648,"knowledge":50.49153449088634,"multilingual":null,"instruction":null,"math":52.68053580654214},"capabilityCounts":{"knowledge":2,"coding":5,"multimodal":2,"agentic":4,"math":1},"benchmarks":[{"benchmarkId":"videoMmeWithSub","benchmarkName":"Video-MME with subtitle","category":"multimodal","score":87.7,"unit":"percent","family":"videoMmeWithSub","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:mmlu_pro:overall","benchmarkName":"MMLU Pro","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":82.931,"unit":"percent","family":"mmlu-pro","index":48.963861841415955,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","measuredAt":"2026-09-01","stderr":0.372,"costPerTest":0.000774,"latency":30.248},{"benchmarkId":"vals-ai:gpqa:overall","benchmarkName":"GPQA Diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":81.566,"unit":"percent","family":"gpqa-diamond","index":52.01920714035672,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","measuredAt":"2026-09-01","stderr":2.054,"costPerTest":0.002538,"latency":89.393},{"benchmarkId":"vals-ai:lcb:overall","benchmarkName":"LiveCodeBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":81.512,"unit":"percent","family":"livecodebench","index":56.97763356197005,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","measuredAt":"2026-09-01","stderr":1.072,"costPerTest":0.002243,"latency":108.607},{"benchmarkId":"charxiv","benchmarkName":"CharXiv Reasoning","category":"multimodal","score":81,"unit":"percent","family":"charxiv","index":56.402573502299184,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:mmmu:overall","benchmarkName":"MMMU Pro","category":"multimodal","domain":null,"eligible":true,"ranked":true,"score":80,"unit":"percent","family":"mmmu-pro","index":59.26579623736889,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmmu","measuredAt":"2026-09-01","stderr":0.962,"costPerTest":0.001396,"latency":42.47},{"benchmarkId":"vals-ai:legal_bench:overall","benchmarkName":"LegalBench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":78.886,"unit":"percent","family":"vals-ai:legal_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","measuredAt":"2026-10-13","stderr":0.506,"costPerTest":0.000264,"latency":11.184},{"benchmarkId":"mmmuPro","benchmarkName":"Massive Multi-discipline Multimodal Understanding Pro","category":"multimodal","score":77.9,"unit":"percent","family":"mmmu-pro","index":56.53283048061865,"aboveFit":false,"run":null,"source":"MMMU-Pro authors","sourceUrl":"https://arxiv.org/abs/2409.02813","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:medscribe:overall","benchmarkName":"MedScribe","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":72.151,"unit":"percent","family":"vals-ai:medscribe:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medscribe","measuredAt":"2026-10-29","stderr":1.851,"costPerTest":0.001351,"latency":20.261},{"benchmarkId":"vals-ai:tax_eval_v2:overall","benchmarkName":"TaxEval v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":71.832,"unit":"percent","family":"vals-ai:tax_eval_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","measuredAt":"2026-09-01","stderr":0.881,"costPerTest":0.000729,"latency":26.3},{"benchmarkId":"vals-ai:swebench:overall","benchmarkName":"SWE-bench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":71,"unit":"percent","family":"swe-bench-verified","index":53.594515552113776,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/swebench","measuredAt":"2026-09-01","stderr":2.031,"costPerTest":0.010032,"latency":244.053},{"benchmarkId":"terminalBench2","benchmarkName":"Terminal-Bench 2.0","category":"agentic","score":65.8,"unit":"percent","family":"terminalBench2","index":60.70417220779392,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"clawEval","benchmarkName":"Claw-Eval","category":"agentic","score":62.3,"unit":"percent","family":"clawEval","index":55.20654122894529,"aboveFit":false,"run":null,"source":"Bowen Ye, Rang Li, Qibin Yang, Yuanxin Liu, Linli Yao, Hanglong Lv, Zhihui Xie, Chenxin An, Lei Li, Lingpeng Kong, Qi Liu, Zhifang Sui, Tong Yang","sourceUrl":"https://arxiv.org/abs/2604.06132","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:terminal-bench-2-1:overall","benchmarkName":"Terminal-Bench 2.1","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":60.674,"unit":"percent","family":"terminal-bench-2-1","index":56.626635591388464,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2-1","measuredAt":"2026-10-01","stderr":5.207,"costPerTest":0.021658,"latency":775.239},{"benchmarkId":"vals-ai:corp_fin_v2:overall","benchmarkName":"CorpFin v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":59.907,"unit":"percent","family":"vals-ai:corp_fin_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","measuredAt":"2026-08-12","stderr":0.966,"costPerTest":0.011733,"latency":17.625},{"benchmarkId":"vals-ai:mortgage_tax:overall","benchmarkName":"MortgageTax","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":59.26,"unit":"percent","family":"vals-ai:mortgage_tax:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mortgage_tax","measuredAt":"2026-09-01","stderr":0.979,"costPerTest":0.001523,"latency":32.939},{"benchmarkId":"swePro","benchmarkName":"SWE-bench Pro","category":"coding","score":56.1,"unit":"percent","family":"swePro","index":56.61470361855937,"aboveFit":false,"run":null,"source":"Xiang Deng, Jeff Da, Edwin Pan, Yannis Yiming He, Charles Ide, Kanak Garg, Niklas Lauffer, Andrew Park, Nitin Pasari, Chetan Rane, Karmini Sampath, Maya Krishnan, Srivatsa Kundurthy, Sean Hendryx, Zifan Wang, Chen Bo Calvin Zhang, Noah Jacobson, Bing Liu, Brad Kenstler","sourceUrl":"https://arxiv.org/abs/2509.16941","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:emb:overall","benchmarkName":"EMB","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":55.092,"unit":"percent","family":"vals-ai:emb:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/emb","measuredAt":"2026-10-20","stderr":3.127,"costPerTest":0.083304,"latency":1837.949},{"benchmarkId":"gertLabs","benchmarkName":"Gert Labs Composite Game Benchmark","category":"agentic","score":46.89,"unit":"percent","family":"gertLabs","index":53.51437700897159,"aboveFit":false,"run":null,"source":"Gert Labs","sourceUrl":"https://gertlabs.com/rankings","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:vibe-code:overall","benchmarkName":"Vibe Code Bench v1.1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":42.174,"unit":"percent","family":"vals-ai:vibe-code:overall","index":57.46656439419637,"aboveFit":false,"run":"OpenHands","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","measuredAt":"2026-11-09","stderr":4.568,"costPerTest":0.068395,"latency":1580.348},{"benchmarkId":"vals-ai:fabv2:overall","benchmarkName":"Finance Agent (v2)","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":36.728,"unit":"percent","family":"vals-ai:fabv2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/fabv2","measuredAt":"2026-10-29","stderr":0.109,"costPerTest":0.08693,"latency":339.26},{"benchmarkId":"vals-ai:medcode:overall","benchmarkName":"MedCode","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":31.895,"unit":"percent","family":"vals-ai:medcode:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medcode","measuredAt":"2026-10-29","stderr":2.025,"costPerTest":0.002162,"latency":17.704},{"benchmarkId":"mmClawBench","benchmarkName":"MM-ClawBench","category":"agentic","score":23.8,"unit":"percent","family":"mmClawBench","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"researchClawBench","benchmarkName":"ResearchClawBench","category":"agentic","score":16.9,"unit":"percent","family":"researchClawBench","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:proof_bench:overall","benchmarkName":"ProofBench v1.1","category":"math","domain":null,"eligible":true,"ranked":true,"score":16,"unit":"percent","family":"vals-ai:proof_bench:overall","index":52.68053580654214,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/proof_bench","measuredAt":"2026-11-09","stderr":3.685,"costPerTest":0.05707,"latency":1255.813},{"benchmarkId":"vals-ai:code-migration:overall","benchmarkName":"Code Migration","category":"coding","domain":null,"eligible":true,"ranked":true,"score":14.228,"unit":"percent","family":"vals-ai:code-migration:overall","index":52.747621497818514,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/code-migration","measuredAt":"2026-10-29","stderr":3.687,"costPerTest":0.102802,"latency":2274.632},{"benchmarkId":"vals-ai:legal_research:overall","benchmarkName":"Legal Research Bench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":9.135,"unit":"percent","family":"vals-ai:legal_research:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_research","measuredAt":"2026-11-09","stderr":2.002,"costPerTest":0.033318,"latency":391.469},{"benchmarkId":"vals-ai:tax_agent_bench:overall","benchmarkName":"Tax Agent Bench","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":7.772,"unit":"percent","family":"vals-ai:tax_agent_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench","measuredAt":"2026-11-09","stderr":1.779,"costPerTest":0.017023,"latency":500.826}],"price":{"inputPerMillion":0.39999999999999997,"outputPerMillion":2,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":3.5,"timeToFirstToken":78.5365,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":{"elo":1427.6392544267055,"eloUpper":1431.8699332954488,"eloLower":1423.4085755579622,"votes":46820,"rank":102,"rankUpper":null,"rankLower":null,"arenaName":"MiMo-V2.5","organization":"xiaomi","updatedAt":"2026-11-09"},"quality":{"coveragePercent":75,"benchmarkCount":14,"categoryCount":5,"confidence":"low","state":"available"},"hosts":[{"host":"deepinfra","name":"Deep Infra","kind":"host","url":"https://deepinfra.com/models","inputPerMillion":0.14,"outputPerMillion":0.28,"cacheReadPerMillion":0.0028,"contextTokens":262144,"maxOutputTokens":16384,"deprecated":true},{"host":"xiaomi","name":"Xiaomi","kind":"maker","url":"https://platform.xiaomimimo.com/#/docs","inputPerMillion":0.14,"outputPerMillion":0.28,"cacheReadPerMillion":0.0028,"contextTokens":1048576,"maxOutputTokens":131072,"deprecated":true},{"host":"novita-ai","name":"Novita AI","kind":"host","url":"https://novita.ai/docs/guides/introduction","inputPerMillion":0.168,"outputPerMillion":0.336,"cacheReadPerMillion":0.0034,"contextTokens":1048576,"maxOutputTokens":131072,"deprecated":false},{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":0.4,"outputPerMillion":2,"cacheReadPerMillion":0.08,"contextTokens":1000000,"maxOutputTokens":65536,"deprecated":true}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"inkling","providerId":"thinking-machines-lab","slug":"inkling","name":"Inkling","provider":"Thinking Machines Lab","description":"Inkling is a reasoning model from Thinking Machines Lab. 45 benchmarks count toward its index, in 7 categories.","logoUrl":null,"releaseDate":"2026-07-15","contextWindowTokens":524288,"modalities":["text","image","audio"],"openWeights":true,"details":{"apiId":"thinkingmachines/inkling","maxOutputTokens":471859,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":false,"cacheReadPerMillion":0.16999999999999998,"weightsUrl":"https://huggingface.co/thinkingmachines/Inkling"},"reasoning":true,"overallScore":54.76069058696988,"overallScoreRange":{"low":51.85020930722511,"high":57.67117186671465},"badges":[],"capabilityScores":{"agentic":51.16054345041621,"coding":54.06993728036115,"reasoning":56.09341441514941,"multimodal":55.043416177242925,"knowledge":57.12656497175371,"multilingual":null,"instruction":66.0093466847406,"math":54.127136567955766},"capabilityCounts":{"math":6,"knowledge":6,"coding":12,"multimodal":3,"instruction":2,"reasoning":6,"agentic":10},"benchmarks":[{"benchmarkId":"aime2026","benchmarkName":"AIME 2026","category":"math","score":97.1,"unit":"percent","family":"aime2026","index":55.19720096214989,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"arc-prize:v1-public-eval:overall","benchmarkName":"ARC-AGI-1 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":90.625,"unit":"percent","family":"arc-prize:v1-public-eval:overall","index":null,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.21569280526315787,"latency":null},{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":88.88888888888889,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":60.634117163950165,"aboveFit":false,"run":"xhigh effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":3.9171949640122654,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-math:overall","benchmarkName":"LiveBench Mathematics","category":"math","domain":null,"eligible":true,"ranked":true,"score":88.36474999999999,"unit":"percent","family":"livebench:livebench-math:overall","index":62.93048119244728,"aboveFit":false,"run":"xhigh effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Mathematics","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.10632717678100262,"latency":null},{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":88.25757575757575,"unit":"percent","family":"gpqa-diamond","index":58.15653757736085,"aboveFit":false,"run":"xhigh effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.9355903133033585,"costPerTest":null,"latency":null},{"benchmarkId":"gpqa","benchmarkName":"Graduate-Level Google-Proof Q&A","category":"knowledge","score":87.9,"unit":"percent","family":"gpqa-diamond","index":57.82857886329864,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gpqaDiamond","benchmarkName":"GPQA Diamond","category":"knowledge","score":87.9,"unit":"percent","family":"gpqa-diamond","index":57.82857886329864,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":87.2,"unit":"percent","family":"gpqa-diamond","index":57.18655799085479,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:gpqa:overall","benchmarkName":"GPQA Diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":87.122,"unit":"percent","family":"gpqa-diamond","index":57.115018522211045,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","measuredAt":"2026-09-01","stderr":1.784,"costPerTest":0.16589,"latency":308.595},{"benchmarkId":"vals-ai:mmlu_pro:overall","benchmarkName":"MMLU Pro","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":86.295,"unit":"percent","family":"mmlu-pro","index":53.20209918096656,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","measuredAt":"2026-09-01","stderr":0.34,"costPerTest":0.039629,"latency":73.264},{"benchmarkId":"vals-ai:lcb:overall","benchmarkName":"LiveCodeBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":85.517,"unit":"percent","family":"livecodebench","index":60.30074718412602,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","measuredAt":"2026-09-01","stderr":1.003,"costPerTest":0.153251,"latency":257.712},{"benchmarkId":"vals-ai:medscribe:overall","benchmarkName":"MedScribe","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":85.405,"unit":"percent","family":"vals-ai:medscribe:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medscribe","measuredAt":"2026-10-29","stderr":1.844,"costPerTest":0.165561,"latency":244.954},{"benchmarkId":"vals-ai:legal_bench:overall","benchmarkName":"LegalBench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":82.949,"unit":"percent","family":"vals-ai:legal_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","measuredAt":"2026-10-13","stderr":0.496,"costPerTest":0.013327,"latency":23.142},{"benchmarkId":"charxiv","benchmarkName":"CharXiv Reasoning","category":"multimodal","score":82,"unit":"percent","family":"charxiv","index":57.541071927866,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"ifBench","benchmarkName":"Instruction Following Benchmark","category":"instruction","score":79.8,"unit":"percent","family":"ifbench-family","index":63.59757645545925,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"arc-prize:v1-semi-private:overall","benchmarkName":"ARC-AGI-1 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":79.5,"unit":"percent","family":"arc-prize:v1-semi-private:overall","index":63.536379046655306,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.30047416199999993,"latency":null},{"benchmarkId":"vals-ai:mmmu:overall","benchmarkName":"MMMU Pro","category":"multimodal","domain":null,"eligible":true,"ranked":true,"score":78.497,"unit":"percent","family":"mmmu-pro","index":57.309773602894786,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmmu","measuredAt":"2026-09-01","stderr":0.988,"costPerTest":0.031466,"latency":123.645},{"benchmarkId":"livebench:livebench-reasoning:overall","benchmarkName":"LiveBench Reasoning","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":78.34625,"unit":"percent","family":"livebench","index":63.11211705601656,"aboveFit":false,"run":"xhigh effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Reasoning","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.12088217821782178,"latency":null},{"benchmarkId":"charxivNoTools","benchmarkName":"CharXiv Reasoning without tools","category":"multimodal","score":78.1,"unit":"percent","family":"charxivNoTools","index":56.78256008976845,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"sweVerified","benchmarkName":"Software Engineering Benchmark Verified","category":"coding","score":77.6,"unit":"percent","family":"swe-bench-verified","index":58.46239556548613,"aboveFit":false,"run":null,"source":"Carlos E. Jimenez, John Yang, Alexander Wettig, Shunyu Yao, Kexin Pei, Ofir Press, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2310.06770","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:swebench:overall","benchmarkName":"SWE-bench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":77.6,"unit":"percent","family":"swe-bench-verified","index":58.46239556548613,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/swebench","measuredAt":"2026-09-01","stderr":1.866,"costPerTest":0.585966,"latency":394.193},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":77.3,"unit":"percent","family":"lcr","index":60.332074086942235,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"browseComp","benchmarkName":"BrowseComp","category":"agentic","score":77.1,"unit":"percent","family":"browseComp","index":60.819521093145156,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:tax_eval_v2:overall","benchmarkName":"TaxEval v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":75.306,"unit":"percent","family":"vals-ai:tax_eval_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","measuredAt":"2026-09-01","stderr":0.839,"costPerTest":0.103224,"latency":197.78},{"benchmarkId":"mcpAtlas","benchmarkName":"MCP Atlas","category":"agentic","score":74.1,"unit":"percent","family":"mcpAtlas","index":60.96089698052803,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmmuPro","benchmarkName":"Massive Multi-discipline Multimodal Understanding Pro","category":"multimodal","score":73.5,"unit":"percent","family":"mmmu-pro","index":50.806616514094316,"aboveFit":false,"run":null,"source":"MMMU-Pro authors","sourceUrl":"https://arxiv.org/abs/2409.02813","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaMmmuPro","benchmarkName":"Artificial Analysis MMMU-Pro","category":"multimodal","score":73.5,"unit":"percent","family":"mmmu-pro","index":50.806616514094316,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmmu-pro","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-language:overall","benchmarkName":"LiveBench Language","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":73.45866666666667,"unit":"percent","family":"livebench:livebench-language:overall","index":57.626508342779616,"aboveFit":false,"run":"xhigh effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Language","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.08477933333333333,"latency":null},{"benchmarkId":"livebench:livebench-data-analysis:overall","benchmarkName":"LiveBench Data Analysis","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":72.778,"unit":"percent","family":"livebench","index":55.8988167069735,"aboveFit":false,"run":"xhigh effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Data%20Analysis","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.16953933333333332,"latency":null},{"benchmarkId":"livebench:livebench-coding:overall","benchmarkName":"LiveBench Coding","category":"coding","domain":null,"eligible":true,"ranked":true,"score":71.0195,"unit":"percent","family":"livebench:livebench-coding:overall","index":54.70682137518749,"aboveFit":false,"run":"xhigh effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Coding","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.05318547008547008,"latency":null},{"benchmarkId":"livebench:livebench-instruction-following:overall","benchmarkName":"LiveBench Instruction Following","category":"instruction","domain":null,"eligible":true,"ranked":true,"score":70.09575000000001,"unit":"percent","family":"livebench:livebench-instruction-following:overall","index":68.42111691402195,"aboveFit":false,"run":"xhigh effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=IF","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.066624,"latency":null},{"benchmarkId":"vals-ai:corp_fin_v2:overall","benchmarkName":"CorpFin v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":68.57,"unit":"percent","family":"vals-ai:corp_fin_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","measuredAt":"2026-08-12","stderr":0.915,"costPerTest":0.084258,"latency":29.199},{"benchmarkId":"vals-ai:cyber:overall","benchmarkName":"CyberBench v1.1","category":null,"domain":"Other","eligible":false,"ranked":true,"score":66.786,"unit":"percent","family":"vals-ai:cyber:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/cyber","measuredAt":"2026-11-05","stderr":5.953,"costPerTest":1.463894,"latency":811.106},{"benchmarkId":"livebench:livebench-coding-2:overall","benchmarkName":"LiveBench Coding 2.0","category":"coding","domain":null,"eligible":true,"ranked":true,"score":64,"unit":"percent","family":"livebench:livebench-coding-2:overall","index":54.52872374278504,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Coding","measuredAt":"2026-10-14","stderr":null,"costPerTest":1.8766172413793103,"latency":null},{"benchmarkId":"vals-ai:mortgage_tax:overall","benchmarkName":"MortgageTax","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":63.99,"unit":"percent","family":"vals-ai:mortgage_tax:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mortgage_tax","measuredAt":"2026-09-01","stderr":0.944,"costPerTest":0.056492,"latency":69.108},{"benchmarkId":"terminalBench21","benchmarkName":"Terminal-Bench 2.1","category":"agentic","score":63.8,"unit":"percent","family":"terminal-bench-2-1","index":58.454893460730986,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-research:overall","benchmarkName":"LiveBench Research","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":59.9,"unit":"percent","family":"livebench-2-agentic","index":62.39620416810558,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Research","measuredAt":"2026-10-14","stderr":null,"costPerTest":1.6679954545454547,"latency":null},{"benchmarkId":"vals-ai:public-benefits-bench:overall","benchmarkName":"Public Benefits Bench v1.1","category":null,"domain":"Public benefits","eligible":false,"ranked":true,"score":58.457,"unit":"percent","family":"vals-ai:public-benefits-bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/public-benefits-bench","measuredAt":"2026-10-12","stderr":1.282,"costPerTest":0.367636,"latency":810.832},{"benchmarkId":"livebench:livebench-design:overall","benchmarkName":"LiveBench Design","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":58.3,"unit":"percent","family":"livebench-2-agentic","index":61.22607441623284,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Design","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.286128,"latency":null},{"benchmarkId":"swePro","benchmarkName":"SWE-bench Pro","category":"coding","score":54.3,"unit":"percent","family":"swePro","index":55.081805652421,"aboveFit":false,"run":null,"source":"Xiang Deng, Jeff Da, Edwin Pan, Yannis Yiming He, Charles Ide, Kanak Garg, Niklas Lauffer, Andrew Park, Nitin Pasari, Chetan Rane, Karmini Sampath, Maya Krishnan, Srivatsa Kundurthy, Sean Hendryx, Zifan Wang, Chen Bo Calvin Zhang, Noah Jacobson, Bing Liu, Brad Kenstler","sourceUrl":"https://arxiv.org/abs/2509.16941","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-agentic-coding:overall","benchmarkName":"LiveBench Agentic Coding","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":49.394000000000005,"unit":"percent","family":"livebench:livebench-agentic-coding:overall","index":60.101881299797455,"aboveFit":false,"run":"xhigh effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Agentic%20Coding","measuredAt":"2026-06-25","stderr":null,"costPerTest":2.231922222222222,"latency":null},{"benchmarkId":"vals-ai:terminal-bench-2-1:overall","benchmarkName":"Terminal-Bench 2.1","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":47.566,"unit":"percent","family":"terminal-bench-2-1","index":48.96035147387668,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2-1","measuredAt":"2026-10-01","stderr":0.375,"costPerTest":0.630691,"latency":771.551},{"benchmarkId":"aaSciCode","benchmarkName":"Artificial Analysis SciCode","category":"coding","score":47,"unit":"percent","family":"scicode-family","index":57.11328232400372,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/scicode","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:fabv2:overall","benchmarkName":"Finance Agent (v2)","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":46.597,"unit":"percent","family":"vals-ai:fabv2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/fabv2","measuredAt":"2026-10-29","stderr":0.925,"costPerTest":0.996264,"latency":1082.125},{"benchmarkId":"hle","benchmarkName":"Humanity's Last Exam","category":"knowledge","score":46,"unit":"percent","family":"hle","index":70.53811058468402,"aboveFit":false,"run":null,"source":"Center for AI Safety, Scale AI, and thousands of expert contributors","sourceUrl":"https://lastexam.ai/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:medcode:overall","benchmarkName":"MedCode","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":41.19,"unit":"percent","family":"vals-ai:medcode:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medcode","measuredAt":"2026-10-29","stderr":2.23,"costPerTest":0.127025,"latency":167.123},{"benchmarkId":"epoch-ai:epoch-simpleqa-verified:overall","benchmarkName":"SimpleQA Verified","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":40.300000000000004,"unit":"percent","family":"epoch-ai:epoch-simpleqa-verified:overall","index":57.492509923549186,"aboveFit":false,"run":"xhigh effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.5518757419066513,"costPerTest":null,"latency":null},{"benchmarkId":"omniscienceAccuracy","benchmarkName":"Artificial Analysis Omniscience Accuracy","category":"knowledge","score":39.3,"unit":"percent","family":"omniscienceAccuracy","index":60.21801064016436,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:emb:overall","benchmarkName":"EMB","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":38.362,"unit":"percent","family":"vals-ai:emb:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/emb","measuredAt":"2026-10-20","stderr":3.273,"costPerTest":1.671783,"latency":1581.835},{"benchmarkId":"arc-prize:v2-public-eval:overall","benchmarkName":"ARC-AGI-2 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":37.77777777777778,"unit":"percent","family":"arc-prize:v2-public-eval:overall","index":null,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.7133307604166669,"latency":null},{"benchmarkId":"cweBenchV1","benchmarkName":"CWE-bench v1","category":"agentic","score":37,"unit":"percent","family":"cweBenchV1","index":53.4368057470289,"aboveFit":false,"run":null,"source":"Collinear AI","sourceUrl":"https://cwe-bench.com/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"arc-prize:v2-semi-private:overall","benchmarkName":"ARC-AGI-2 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":36.52777777777778,"unit":"percent","family":"arc-prize:v2-semi-private:overall","index":55.959685729566026,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.6420766277310923,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tiers-1-3-v2-private:overall","benchmarkName":"FrontierMath-Tiers-1-3-v2-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":33.33333333333333,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tiers-1-3-v2-private:overall","index":48.4397533486134,"aboveFit":false,"run":"xhigh effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.7972711943222968,"costPerTest":null,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":31.9,"unit":"percent","family":"hle","index":56.39168287976391,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hleNoTools","benchmarkName":"Humanity's Last Exam without tools","category":"knowledge","score":30,"unit":"percent","family":"hle","index":54.48542666420731,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gdpvalAaNormalized","benchmarkName":"GDPval-AA normalized","category":"agentic","score":28.6,"unit":"percent","family":"gdpvalAaNormalized","index":54.48211030157918,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:legal_research:overall","benchmarkName":"Legal Research Bench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":28.365,"unit":"percent","family":"vals-ai:legal_research:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_research","measuredAt":"2026-11-09","stderr":3.133,"costPerTest":0.494044,"latency":869.377},{"benchmarkId":"vals-ai:skillsbench:overall","benchmarkName":"SkillsBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":26.091,"unit":"percent","family":"vals-ai:skillsbench:overall","index":44.563902374440055,"aboveFit":false,"run":"OpenHands","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/skillsbench","measuredAt":"2026-10-29","stderr":3.863,"costPerTest":0.826622,"latency":938.657},{"benchmarkId":"tau2-bench:banking-knowledge:overall","benchmarkName":"τ²-bench Banking","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":25,"unit":"percent","family":"tau2-bench","index":19.378767943379025,"aboveFit":false,"run":"max effort · Sierra","source":"Sierra Research","sourceUrl":"https://github.com/sierra-research/tau2-bench","measuredAt":"2026-10-20","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-chess-puzzles:overall","benchmarkName":"Chess Puzzles","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":21,"unit":"percent","family":"epoch-ai:epoch-chess-puzzles:overall","index":49.923994823290954,"aboveFit":false,"run":"xhigh effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":4.093601807403323,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:vibe-code:overall","benchmarkName":"Vibe Code Bench v1.1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":19.214,"unit":"percent","family":"vals-ai:vibe-code:overall","index":49.18223566554191,"aboveFit":false,"run":"OpenHands","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","measuredAt":"2026-11-09","stderr":3.659,"costPerTest":1.605949,"latency":1105.492},{"benchmarkId":"livebench:livebench-browser:overall","benchmarkName":"LiveBench Browser Use","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":16,"unit":"percent","family":"livebench-2-agentic","index":30.29076910109751,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Browser","measuredAt":"2026-10-14","stderr":null,"costPerTest":2.39616,"latency":null},{"benchmarkId":"vals-ai:tax_agent_bench:overall","benchmarkName":"Tax Agent Bench","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":15.291,"unit":"percent","family":"vals-ai:tax_agent_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench","measuredAt":"2026-11-09","stderr":2.423,"costPerTest":0.353757,"latency":810.557},{"benchmarkId":"vals-ai:ioi:overall","benchmarkName":"IOI","category":"coding","domain":null,"eligible":true,"ranked":true,"score":14.944,"unit":"percent","family":"vals-ai:ioi:overall","index":51.72694653081509,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/ioi","measuredAt":"2026-10-29","stderr":5.403,"costPerTest":1.70838,"latency":1812.242},{"benchmarkId":"livebench:livebench-long-horizon:overall","benchmarkName":"LiveBench Long Agentic","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":14.1,"unit":"percent","family":"livebench-2-agentic","index":28.901240020748645,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Long%20Agentic","measuredAt":"2026-10-14","stderr":null,"costPerTest":10.538786206896551,"latency":null},{"benchmarkId":"vals-ai:code-migration:overall","benchmarkName":"Code Migration","category":"coding","domain":null,"eligible":true,"ranked":true,"score":11.79,"unit":"percent","family":"vals-ai:code-migration:overall","index":51.452458149374486,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/code-migration","measuredAt":"2026-10-29","stderr":3.396,"costPerTest":3.006967,"latency":1773.502},{"benchmarkId":"vals-ai:vcb-1-100:overall","benchmarkName":"Vibe Code Bench 1-100","category":"coding","domain":null,"eligible":true,"ranked":true,"score":7.333,"unit":"percent","family":"vals-ai:vcb-1-100:overall","index":56.53011992403132,"aboveFit":false,"run":"OpenHands","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vcb-1-100","measuredAt":"2026-10-29","stderr":1.839,"costPerTest":4.997334,"latency":3001.539},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":5.4,"unit":"percent","family":"critpt","index":47.3028859229469,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tier-4-v2-private:overall","benchmarkName":"FrontierMath-Tier-4-v2-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":4.878048780487805,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tier-4-v2-private:overall","index":50.40464614989685,"aboveFit":false,"run":"xhigh effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":3.4059122057973035,"costPerTest":null,"latency":null},{"benchmarkId":"frontierSweV2","benchmarkName":"FrontierSWE v2","category":"coding","score":4.1,"unit":"percent","family":"frontierSweV2","index":55.189808876121575,"aboveFit":false,"run":null,"source":"Proximal","sourceUrl":"https://www.frontierswe.com/blog/v2","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:hlab:overall","benchmarkName":"HLAB-Vals-1.1","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":3.75,"unit":"percent","family":"vals-ai:hlab:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/hlab","measuredAt":"2026-11-09","stderr":1.469,"costPerTest":1.252023,"latency":529.644},{"benchmarkId":"vals-ai:terminal-bench-4:overall","benchmarkName":"Terminal-Bench 4.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":0.505,"unit":"percent","family":"terminal-bench-4","index":54.755806974722255,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-4","measuredAt":"2026-11-09","stderr":0.505,"costPerTest":2.495998,"latency":1378.754},{"benchmarkId":"lmarena:agent-bash-recovery:overall","benchmarkName":"Agent Arena command recovery","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":0.08143083339426926,"unit":"index","family":"agent-arena","index":66.59775022170696,"aboveFit":false,"run":null,"source":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset","measuredAt":"2026-10-08","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:programbench:overall","benchmarkName":"ProgramBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":0,"unit":"percent","family":"vals-ai:programbench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/programbench","measuredAt":"2026-11-09","stderr":0,"costPerTest":4.143489,"latency":2734.146},{"benchmarkId":"vals-ai:proof_bench:overall","benchmarkName":"ProofBench v1.1","category":"math","domain":null,"eligible":true,"ranked":true,"score":0,"unit":"percent","family":"vals-ai:proof_bench:overall","index":47.156620590676994,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/proof_bench","measuredAt":"2026-11-09","stderr":0,"costPerTest":0.198639,"latency":436.43},{"benchmarkId":"vals-ai:srebench:overall","benchmarkName":"SRE Bench","category":null,"domain":"Other","eligible":false,"ranked":true,"score":0,"unit":"percent","family":"vals-ai:srebench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/srebench","measuredAt":"2026-11-05","stderr":0,"costPerTest":2.225036,"latency":1718.686},{"benchmarkId":"lmarena:agent-steerability:overall","benchmarkName":"Agent Arena steerability","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":-22.29161397409308,"unit":"index","family":"agent-arena","index":48.2035999380131,"aboveFit":false,"run":null,"source":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset","measuredAt":"2026-11-11","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lmarena:agent-task-outcome:overall","benchmarkName":"Agent Arena task outcome","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":-26.803048100305215,"unit":"index","family":"agent-arena","index":44.49449382274891,"aboveFit":false,"run":null,"source":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset","measuredAt":"2026-11-11","stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":1,"outputPerMillion":4.05,"batch":null,"freeTier":true,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":85,"timeToFirstToken":0.318,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":{"elo":1443.546072497843,"eloUpper":1447.6119783676731,"eloLower":1439.4801666280123,"votes":50137,"rank":69,"rankUpper":null,"rankLower":null,"arenaName":"Inkling","organization":"thinky","updatedAt":"2026-11-09"},"quality":{"coveragePercent":95,"benchmarkCount":45,"categoryCount":7,"confidence":"high","state":"available"},"hosts":[{"host":"deepinfra","name":"Deep Infra","kind":"host","url":"https://deepinfra.com/models","inputPerMillion":0.95,"outputPerMillion":4.05,"cacheReadPerMillion":0.16,"contextTokens":524288,"maxOutputTokens":1048576,"deprecated":true},{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":1,"outputPerMillion":4.05,"cacheReadPerMillion":0.17,"contextTokens":524288,"maxOutputTokens":471859,"deprecated":false},{"host":"together-ai","name":"Together AI","kind":"host","url":"https://docs.together.ai/docs/serverless-models","inputPerMillion":1,"outputPerMillion":4.05,"cacheReadPerMillion":0.17,"contextTokens":524288,"maxOutputTokens":131072,"deprecated":false},{"host":"modal","name":"Modal","kind":"host","url":"https://modal.com/docs/guide/endpoints","inputPerMillion":1.2,"outputPerMillion":5,"cacheReadPerMillion":0.27,"contextTokens":1048576,"maxOutputTokens":262144,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:livebench","name":"LiveBench","url":"https://livebench.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:arc-prize-foundation","name":"ARC Prize Foundation","url":"https://arcprize.org","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:sierra-research","name":"Sierra Research","url":"https://github.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:lmarena","name":"LMArena","url":"https://huggingface.co","license":"CC BY 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"qwen3-6-plus","providerId":"alibaba","slug":"qwen3-6-plus","name":"Qwen3.6 Plus","provider":"Alibaba","description":"Qwen3.6 Plus is a reasoning model from Alibaba. 58 benchmarks count toward its index, in 8 categories.","logoUrl":null,"releaseDate":"2026-04-02","contextWindowTokens":1000000,"modalities":["text","image","video"],"openWeights":false,"details":{"apiId":"qwen/qwen3.6-plus","maxOutputTokens":65536,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":null,"weightsUrl":null},"reasoning":true,"overallScore":54.62263835578915,"overallScoreRange":{"low":52.49941333644743,"high":56.74586337513087},"badges":[],"capabilityScores":{"agentic":55.009723068758205,"coding":56.23077900736392,"reasoning":51.24297489525935,"multimodal":56.11023362584744,"knowledge":54.104144116530186,"multilingual":54.0526835036468,"instruction":52.437199930731325,"math":54.871473795990056},"capabilityCounts":{"agentic":15,"multimodal":6,"math":10,"knowledge":9,"instruction":3,"coding":8,"multilingual":1,"reasoning":6},"benchmarks":[{"benchmarkId":"tau2Bench","benchmarkName":"τ²-Bench Tool-Agent-User Evaluation","category":"agentic","score":97.7,"unit":"percent","family":"tau2-bench","index":67.62053318007978,"aboveFit":false,"run":null,"source":"Victor Barres, Honghua Dong, Soham Ray, Xujie Si, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2506.07982","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vStar","benchmarkName":"V*","category":"multimodal","score":96.9,"unit":"percent","family":"vStar","index":57.60734703162754,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hmmtFeb2025","benchmarkName":"Harvard-MIT Mathematics Tournament February 2025","category":"math","score":96.7,"unit":"percent","family":"hmmtFeb2025","index":54.45686585540949,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aime2026","benchmarkName":"AIME 2026","category":"math","score":95.3,"unit":"percent","family":"aime2026","index":53.901308033618,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hmmtNov2025","benchmarkName":"Harvard-MIT Mathematics Tournament November 2025","category":"math","score":94.6,"unit":"percent","family":"hmmtNov2025","index":56.387494199973816,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:aime:overall","benchmarkName":"AIME","category":"math","domain":null,"eligible":true,"ranked":true,"score":94.583,"unit":"percent","family":"vals-ai:aime:overall","index":56.83966514939485,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/aime","measuredAt":"2026-04-16","stderr":0.579,"costPerTest":0.09352,"latency":516.49},{"benchmarkId":"mmluRedux","benchmarkName":"MMLU-Redux","category":"knowledge","score":94.5,"unit":"percent","family":"mmluRedux","index":50.46926829204164,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"ifeval","benchmarkName":"Instruction-Following Eval","category":"instruction","score":94.3,"unit":"percent","family":"ifeval","index":53.09454352454052,"aboveFit":false,"run":null,"source":"Jeffrey Zhou, Tianjian Lu, Swaroop Mishra, Siddhartha Brahma, Sujoy Basu, Yi Luan, Denny Zhou, Le Hou","sourceUrl":"https://arxiv.org/abs/2311.07911","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":93.33333333333333,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":63.219868478319434,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":3.760507165451774,"costPerTest":null,"latency":null},{"benchmarkId":"cEval","benchmarkName":"C-Eval","category":"knowledge","score":93.3,"unit":"percent","family":"cEval","index":50.22416587200141,"aboveFit":false,"run":null,"source":"C-Eval authors","sourceUrl":"https://arxiv.org/abs/2305.08322","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gpqa","benchmarkName":"Graduate-Level Google-Proof Q&A","category":"knowledge","score":90.4,"unit":"percent","family":"gpqa-diamond","index":60.1215105505981,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmluPro","benchmarkName":"Massive Multitask Language Understanding Professional","category":"knowledge","score":88.5,"unit":"percent","family":"mmlu-pro","index":55.98013524925106,"aboveFit":false,"run":null,"source":"Yubo Wang, Xueguang Ma, Ge Zhang, Yuansheng Ni, Abhranil Chandra, Shiguang Guo, Weiming Ren, Aaran Arulraj, Xuan He, Ziyan Jiang, Tianle Li, Max Ku, Kai Wang, Alex Zhuang, Rongqi Fan, Xiang Yue, Wenhu Chen","sourceUrl":"https://arxiv.org/abs/2406.01574","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":88.38383838383838,"unit":"percent","family":"gpqa-diamond","index":58.27234220803255,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.2828881775249354,"costPerTest":null,"latency":null},{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":88.2,"unit":"percent","family":"gpqa-diamond","index":58.10373066577457,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mathVision","benchmarkName":"MathVision","category":"multimodal","score":88,"unit":"percent","family":"mathVision","index":58.98418173621045,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hmmtFeb2026","benchmarkName":"Harvard-MIT Mathematics Tournament February 2026","category":"math","score":87.8,"unit":"percent","family":"hmmtFeb2026","index":54.84479830498993,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:mmlu_pro:overall","benchmarkName":"MMLU Pro","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":87.668,"unit":"percent","family":"mmlu-pro","index":54.931914837091114,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","measuredAt":"2026-09-01","stderr":0.323,"costPerTest":0.008813,"latency":46.184},{"benchmarkId":"vals-ai:gpqa:overall","benchmarkName":"GPQA Diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":87.374,"unit":"percent","family":"gpqa-diamond","index":57.346146036290826,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","measuredAt":"2026-09-01","stderr":1.668,"costPerTest":0.022964,"latency":122.081},{"benchmarkId":"liveCodeBenchV6","benchmarkName":"LiveCodeBench v6","category":"coding","score":87.1,"unit":"percent","family":"liveCodeBenchV6","index":55.93114861816006,"aboveFit":false,"run":null,"source":"LiveCodeBench maintainers","sourceUrl":"https://github.com/LiveCodeBench/LiveCodeBench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmmu","benchmarkName":"Massive Multi-discipline Multimodal Understanding","category":"multimodal","score":86,"unit":"percent","family":"mmmu","index":55.194651505606686,"aboveFit":false,"run":null,"source":"MMMU authors","sourceUrl":"https://arxiv.org/abs/2401.05508","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:lcb:overall","benchmarkName":"LiveCodeBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":85.952,"unit":"percent","family":"livecodebench","index":60.661684618742214,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","measuredAt":"2026-09-01","stderr":0.977,"costPerTest":0.077628,"latency":475.705},{"benchmarkId":"mmluProX","benchmarkName":"MMLU-ProX","category":"multilingual","score":84.7,"unit":"percent","family":"mmluProX","index":54.0526835036468,"aboveFit":false,"run":null,"source":"MMLU-ProX authors","sourceUrl":"https://arxiv.org/abs/2503.10497","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:legal_bench:overall","benchmarkName":"LegalBench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":84.233,"unit":"percent","family":"vals-ai:legal_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","measuredAt":"2026-10-13","stderr":0.422,"costPerTest":0.003042,"latency":19.073},{"benchmarkId":"vals-ai:mmmu:overall","benchmarkName":"MMMU Pro","category":"multimodal","domain":null,"eligible":true,"ranked":true,"score":84.162,"unit":"percent","family":"mmmu-pro","index":64.68227408479487,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmmu","measuredAt":"2026-09-01","stderr":0.878,"costPerTest":0.017413,"latency":111.943},{"benchmarkId":"videoMmmu","benchmarkName":"VideoMMMU","category":"multimodal","score":84,"unit":"percent","family":"videoMmmu","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmAnswerBench","benchmarkName":"MMAnswerBench","category":"math","score":83.8,"unit":"percent","family":"mmAnswerBench","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-math:overall","benchmarkName":"LiveBench Mathematics","category":"math","domain":null,"eligible":true,"ranked":true,"score":83.72475,"unit":"percent","family":"livebench:livebench-math:overall","index":56.46362367063702,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Mathematics","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.10636226912928759,"latency":null},{"benchmarkId":"charxiv","benchmarkName":"CharXiv Reasoning","category":"multimodal","score":81.5,"unit":"percent","family":"charxiv","index":56.971822715082595,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"sweVerified","benchmarkName":"Software Engineering Benchmark Verified","category":"coding","score":78.8,"unit":"percent","family":"swe-bench-verified","index":59.34746465882657,"aboveFit":false,"run":null,"source":"Carlos E. Jimenez, John Yang, Alexander Wettig, Shunyu Yao, Kexin Pei, Ofir Press, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2310.06770","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmmuPro","benchmarkName":"Massive Multi-discipline Multimodal Understanding Pro","category":"multimodal","score":78.8,"unit":"percent","family":"mmmu-pro","index":57.70410151922589,"aboveFit":false,"run":null,"source":"MMMU-Pro authors","sourceUrl":"https://arxiv.org/abs/2409.02813","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":78.3,"unit":"percent","family":"lcr","index":61.018154612527894,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-coding:overall","benchmarkName":"LiveBench Coding","category":"coding","domain":null,"eligible":true,"ranked":true,"score":78.1845,"unit":"percent","family":"livebench:livebench-coding:overall","index":64.78358851077434,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Coding","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.017412820512820514,"latency":null},{"benchmarkId":"aaMmmuPro","benchmarkName":"Artificial Analysis MMMU-Pro","category":"multimodal","score":78,"unit":"percent","family":"mmmu-pro","index":56.66297170713056,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmmu-pro","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:medscribe:overall","benchmarkName":"MedScribe","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":76.963,"unit":"percent","family":"vals-ai:medscribe:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medscribe","measuredAt":"2026-10-29","stderr":1.917,"costPerTest":0.029294,"latency":173.545},{"benchmarkId":"livebench:livebench-reasoning:overall","benchmarkName":"LiveBench Reasoning","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":75.827,"unit":"percent","family":"livebench","index":59.84859496122434,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Reasoning","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.09352227722772277,"latency":null},{"benchmarkId":"ifBench","benchmarkName":"Instruction Following Benchmark","category":"instruction","score":75.8,"unit":"percent","family":"ifbench-family","index":59.85953399614123,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaIfBench","benchmarkName":"Artificial Analysis IFBench","category":"instruction","score":75.2,"unit":"percent","family":"ifbench-family","index":59.29882762724353,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-language:overall","benchmarkName":"LiveBench Language","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":74.98933333333333,"unit":"percent","family":"livebench:livebench-language:overall","index":59.3574009613186,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Language","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.032214,"latency":null},{"benchmarkId":"vals-ai:tax_eval_v2:overall","benchmarkName":"TaxEval v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":74.734,"unit":"percent","family":"vals-ai:tax_eval_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","measuredAt":"2026-09-01","stderr":0.852,"costPerTest":0.01145,"latency":70.634},{"benchmarkId":"wideResearch","benchmarkName":"WideResearch","category":"agentic","score":74.3,"unit":"percent","family":"wideResearch","index":57.066543440189164,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mcpTasks","benchmarkName":"MCP-Tasks","category":"agentic","score":74.1,"unit":"percent","family":"mcpTasks","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:swebench:overall","benchmarkName":"SWE-bench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":73.4,"unit":"percent","family":"swe-bench-verified","index":55.36465373879464,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/swebench","measuredAt":"2026-09-01","stderr":1.978,"costPerTest":0.869211,"latency":424.172},{"benchmarkId":"superGpqa","benchmarkName":"SuperGPQA: Scaling LLM Evaluation Across 285 Graduate Disciplines","category":"knowledge","score":71.6,"unit":"percent","family":"superGpqa","index":55.806841649661536,"aboveFit":false,"run":null,"source":"Xiaoxuan Du, Yao Yao, Kexin Ma, Bowen Wang, Tianyu Zheng, Kaiyan Zhu, Yiming Zhang, Yutao Zhu, Jiawei Zhou, Jingren Zhou","sourceUrl":"https://arxiv.org/abs/2502.14739","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"tau3Bench","benchmarkName":"τ³-Bench Tool-Agent-User Evaluation","category":"agentic","score":70.7,"unit":"percent","family":"tau3Bench","index":60.79881676390667,"aboveFit":false,"run":null,"source":"Sierra Research","sourceUrl":"https://github.com/sierra-research/tau2-bench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-data-analysis:overall","benchmarkName":"LiveBench Data Analysis","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":69.91166666666668,"unit":"percent","family":"livebench","index":52.18567106268542,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Data%20Analysis","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.07666133333333333,"latency":null},{"benchmarkId":"aiNeedle","benchmarkName":"AI-Needle","category":"reasoning","score":68.3,"unit":"percent","family":"aiNeedle","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"screenSpotPro","benchmarkName":"ScreenSpot Pro","category":"multimodal","score":68.2,"unit":"percent","family":"screenSpotPro","index":50.19929724733148,"aboveFit":false,"run":null,"source":"Kaixin Li, Ziyang Meng, Hongzhan Lin, Ziyang Luo, Yuchen Tian, Jing Ma, Zhiyong Huang, Tat-Seng Chua","sourceUrl":"https://arxiv.org/abs/2504.07981","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:mortgage_tax:overall","benchmarkName":"MortgageTax","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":67.965,"unit":"percent","family":"vals-ai:mortgage_tax:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mortgage_tax","measuredAt":"2026-09-01","stderr":0.908,"costPerTest":0.013476,"latency":78.18},{"benchmarkId":"longBenchV2","benchmarkName":"LongBench v2","category":"reasoning","score":62,"unit":"percent","family":"longBenchV2","index":53.85648830580091,"aboveFit":false,"run":null,"source":"LongBench v2 authors","sourceUrl":"https://arxiv.org/abs/2412.15204","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:corp_fin_v2:overall","benchmarkName":"CorpFin v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":61.927,"unit":"percent","family":"vals-ai:corp_fin_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","measuredAt":"2026-08-12","stderr":0.957,"costPerTest":0.047203,"latency":35.839},{"benchmarkId":"terminalBench2","benchmarkName":"Terminal-Bench 2.0","category":"agentic","score":61.6,"unit":"percent","family":"terminalBench2","index":57.71652110623645,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"clawEval","benchmarkName":"Claw-Eval","category":"agentic","score":58.8,"unit":"percent","family":"clawEval","index":50.14163073153125,"aboveFit":false,"run":null,"source":"Bowen Ye, Rang Li, Qibin Yang, Yuanxin Liu, Linli Yao, Hanglong Lv, Zhihui Xie, Chenxin An, Lei Li, Lingpeng Kong, Qi Liu, Zhifang Sui, Tong Yang","sourceUrl":"https://arxiv.org/abs/2604.06132","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-instruction-following:overall","benchmarkName":"LiveBench Instruction Following","category":"instruction","domain":null,"eligible":true,"ranked":true,"score":58.342,"unit":"percent","family":"livebench:livebench-instruction-following:overall","index":44.63787545596108,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=IF","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.050054499999999995,"latency":null},{"benchmarkId":"nova63","benchmarkName":"NOVA-63","category":"multilingual","score":57.9,"unit":"percent","family":"nova63","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-swe-bench-verified:overall","benchmarkName":"SWE-Bench verified","category":"coding","domain":null,"eligible":true,"ranked":true,"score":57.85123966942148,"unit":"percent","family":"swe-bench-verified","index":43.896547731833984,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.246855652298021,"costPerTest":null,"latency":null},{"benchmarkId":"qwenClawBench","benchmarkName":"QwenClawBench","category":"agentic","score":57.2,"unit":"percent","family":"qwenClawBench","index":54.96606802679662,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"swePro","benchmarkName":"SWE-bench Pro","category":"coding","score":56.6,"unit":"percent","family":"swePro","index":57.04050860915336,"aboveFit":false,"run":null,"source":"Xiang Deng, Jeff Da, Edwin Pan, Yannis Yiming He, Charles Ide, Kanak Garg, Niklas Lauffer, Andrew Park, Nitin Pasari, Chetan Rane, Karmini Sampath, Maya Krishnan, Srivatsa Kundurthy, Sean Hendryx, Zifan Wang, Chen Bo Calvin Zhang, Noah Jacobson, Bing Liu, Brad Kenstler","sourceUrl":"https://arxiv.org/abs/2509.16941","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:finance_agent:overall","benchmarkName":"Finance Agent v1.1","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":54.627,"unit":"percent","family":"vals-ai:finance_agent:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/finance_agent","measuredAt":"2026-06-04","stderr":2.889,"costPerTest":0.151866,"latency":327.762},{"benchmarkId":"vals-ai:terminal-bench-2-1:overall","benchmarkName":"Terminal-Bench 2.1","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":53.184,"unit":"percent","family":"terminal-bench-2-1","index":52.24606891148586,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2-1","measuredAt":"2026-10-01","stderr":1.633,"costPerTest":0.201902,"latency":778.109},{"benchmarkId":"vals-ai:case_law_v2:overall","benchmarkName":"CaseLaw v2","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":51.447,"unit":"percent","family":"vals-ai:case_law_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/case_law_v2","measuredAt":"2026-05-04","stderr":1.159,"costPerTest":0.047281,"latency":43.909},{"benchmarkId":"gertLabs","benchmarkName":"Gert Labs Composite Game Benchmark","category":"agentic","score":50.6,"unit":"percent","family":"gertLabs","index":56.51842628670877,"aboveFit":false,"run":null,"source":"Gert Labs","sourceUrl":"https://gertlabs.com/rankings","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-2025-02-28-public:overall","benchmarkName":"FrontierMath-2025-02-28-Public","category":"math","domain":null,"eligible":false,"ranked":true,"score":50,"unit":"percent","family":"epoch-ai:epoch-frontiermath-2025-02-28-public:overall","index":null,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":16.666666666666664,"costPerTest":null,"latency":null},{"benchmarkId":"mcpAtlas","benchmarkName":"MCP Atlas","category":"agentic","score":48.2,"unit":"percent","family":"mcpAtlas","index":43.51995104226888,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:terminal-bench-2:overall","benchmarkName":"Terminal-Bench 2.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":44.944,"unit":"percent","family":"vals-ai:terminal-bench-2:overall","index":53.61255278791465,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2","measuredAt":"2026-06-04","stderr":5.303,"costPerTest":0.14597,"latency":840.666},{"benchmarkId":"vitaBench","benchmarkName":"VITA-Bench","category":"agentic","score":44.3,"unit":"percent","family":"vitaBench","index":57.9931488663871,"aboveFit":false,"run":null,"source":"Meituan LongCat Team","sourceUrl":"https://vitabench.github.io/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-simpleqa-verified:overall","benchmarkName":"SimpleQA Verified","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":44.14414414414414,"unit":"percent","family":"epoch-ai:epoch-simpleqa-verified:overall","index":60.69502135022083,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.5718302238794237,"costPerTest":null,"latency":null},{"benchmarkId":"terminalBenchHard","benchmarkName":"Terminal-Bench Hard","category":"agentic","score":43.9,"unit":"percent","family":"terminalBenchHard","index":55.11285551926768,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/terminalbench-hard","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"deepPlanning","benchmarkName":"DeepPlanning","category":"agentic","score":41.5,"unit":"percent","family":"deepPlanning","index":null,"aboveFit":false,"run":null,"source":"DeepPlanning authors","sourceUrl":"https://arxiv.org/abs/2601.18137","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-agentic-coding:overall","benchmarkName":"LiveBench Agentic Coding","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":41.36366666666667,"unit":"percent","family":"livebench:livebench-agentic-coding:overall","index":51.820335303202945,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Agentic%20Coding","measuredAt":"2026-06-25","stderr":null,"costPerTest":1.5460749999999999,"latency":null},{"benchmarkId":"vals-ai:fabv2:overall","benchmarkName":"Finance Agent (v2)","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":40.846,"unit":"percent","family":"vals-ai:fabv2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/fabv2","measuredAt":"2026-10-29","stderr":0.131,"costPerTest":0.814379,"latency":611.907},{"benchmarkId":"toolathlon","benchmarkName":"Toolathlon","category":"agentic","score":39.8,"unit":"percent","family":"toolathlon","index":54.46168848675458,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"nl2Repo","benchmarkName":"NL2Repo","category":"coding","score":37.9,"unit":"percent","family":"nl2Repo","index":53.50390675114332,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:medcode:overall","benchmarkName":"MedCode","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":36.894,"unit":"percent","family":"vals-ai:medcode:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medcode","measuredAt":"2026-10-29","stderr":2.017,"costPerTest":0.015673,"latency":56.222},{"benchmarkId":"vals-ai:emb:overall","benchmarkName":"EMB","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":32.867,"unit":"percent","family":"vals-ai:emb:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/emb","measuredAt":"2026-10-20","stderr":2.814,"costPerTest":2.866775,"latency":1873.595},{"benchmarkId":"epoch-ai:epoch-frontiermath-tiers-1-3-v2-private:overall","benchmarkName":"FrontierMath-Tiers-1-3-v2-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":32.280701754385966,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tiers-1-3-v2-private:overall","index":47.88569891566738,"aboveFit":false,"run":"none effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.7743965653274283,"costPerTest":null,"latency":null},{"benchmarkId":"hle","benchmarkName":"Humanity's Last Exam","category":"knowledge","score":28.8,"unit":"percent","family":"hle","index":53.281475370171556,"aboveFit":false,"run":null,"source":"Center for AI Safety, Scale AI, and thousands of expert contributors","sourceUrl":"https://lastexam.ai/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":27.8,"unit":"percent","family":"hle","index":52.27818262514176,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"omniscienceAccuracy","benchmarkName":"Artificial Analysis Omniscience Accuracy","category":"knowledge","score":26.4,"unit":"percent","family":"omniscienceAccuracy","index":43.96070844579634,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-2025-02-28-private:overall","benchmarkName":"FrontierMath-2025-02-28-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":26.20689655172414,"unit":"percent","family":"epoch-ai:epoch-frontiermath-2025-02-28-private:overall","index":53.04903528962972,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.586823296864886,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:vibe-code:overall","benchmarkName":"Vibe Code Bench v1.1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":25.565,"unit":"percent","family":"vals-ai:vibe-code:overall","index":51.473776247235485,"aboveFit":false,"run":"OpenHands","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","measuredAt":"2026-11-09","stderr":3.906,"costPerTest":5.447636,"latency":2289.977},{"benchmarkId":"gdpvalAaNormalized","benchmarkName":"GDPval-AA normalized","category":"agentic","score":24.5,"unit":"percent","family":"gdpvalAaNormalized","index":51.55070557864267,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"researchClawBench","benchmarkName":"ResearchClawBench","category":"agentic","score":18,"unit":"percent","family":"researchClawBench","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-chess-puzzles:overall","benchmarkName":"Chess Puzzles","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":16,"unit":"percent","family":"epoch-ai:epoch-chess-puzzles:overall","index":46.12891884754047,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":3.6845294917747067,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:legal_research:overall","benchmarkName":"Legal Research Bench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":14.904,"unit":"percent","family":"vals-ai:legal_research:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_research","measuredAt":"2026-11-09","stderr":2.475,"costPerTest":0.697997,"latency":658.954},{"benchmarkId":"epoch-ai:epoch-mystery-game-puzzles:overall","benchmarkName":"Mystery Game Puzzles","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":12,"unit":"percent","family":"epoch-ai:epoch-mystery-game-puzzles:overall","index":47.973976962532255,"aboveFit":false,"run":"none effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":3.2659863237109037,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:code-migration:overall","benchmarkName":"Code Migration","category":"coding","domain":null,"eligible":true,"ranked":true,"score":11.102,"unit":"percent","family":"vals-ai:code-migration:overall","index":51.086964964907914,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/code-migration","measuredAt":"2026-10-29","stderr":1.312,"costPerTest":4.781669,"latency":1789.961},{"benchmarkId":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-private:overall","benchmarkName":"FrontierMath-Tier-4-2025-07-01-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":8.333333333333332,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-private:overall","index":51.666380062260956,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":4.0314953903908055,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:tax_agent_bench:overall","benchmarkName":"Tax Agent Bench","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":7.679,"unit":"percent","family":"vals-ai:tax_agent_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench","measuredAt":"2026-11-09","stderr":1.81,"costPerTest":0.298017,"latency":378.159},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":2.9,"unit":"percent","family":"critpt","index":42.46317763119968,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:programbench:overall","benchmarkName":"ProgramBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":0,"unit":"percent","family":"vals-ai:programbench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/programbench","measuredAt":"2026-11-09","stderr":0,"costPerTest":null,"latency":1981.11},{"benchmarkId":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-public:overall","benchmarkName":"FrontierMath-Tier-4-2025-07-01-Public","category":"math","domain":null,"eligible":false,"ranked":true,"score":0,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-public:overall","index":null,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":0.325,"outputPerMillion":1.95,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":44,"timeToFirstToken":1.054,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":{"elo":1436.2087562810798,"eloUpper":1440.3297446739489,"eloLower":1432.0877678882107,"votes":47601,"rank":86,"rankUpper":null,"rankLower":null,"arenaName":"Qwen3.6-Plus","organization":"alibaba","updatedAt":"2026-11-09"},"quality":{"coveragePercent":100,"benchmarkCount":58,"categoryCount":8,"confidence":"high","state":"available"},"hosts":[{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":0.325,"outputPerMillion":1.95,"cacheReadPerMillion":null,"contextTokens":1000000,"maxOutputTokens":65536,"deprecated":false},{"host":"together-ai","name":"Together AI","kind":"host","url":"https://docs.together.ai/docs/serverless-models","inputPerMillion":0.5,"outputPerMillion":3,"cacheReadPerMillion":null,"contextTokens":1000000,"maxOutputTokens":65536,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:livebench","name":"LiveBench","url":"https://livebench.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gpt-5-4-mini","providerId":"openai","slug":"gpt-5-4-mini","name":"GPT-5.4 mini","provider":"OpenAI","description":"GPT-5.4 mini is a reasoning model from OpenAI in the GPT-5.4 family. 45 benchmarks count toward its index, in 8 categories.","logoUrl":null,"releaseDate":"2026-03-17","contextWindowTokens":400000,"modalities":["file","image","text"],"openWeights":false,"details":{"apiId":"openai/gpt-5.4-mini","maxOutputTokens":128000,"knowledgeCutoff":"2025-08-31","retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.075,"weightsUrl":null},"reasoning":true,"overallScore":54.536973978746886,"overallScoreRange":{"low":52.126492699002114,"high":56.94745525849166},"badges":[],"capabilityScores":{"agentic":56.06465597731324,"coding":55.14452071464614,"reasoning":52.159337073526466,"multimodal":54.84099453596372,"knowledge":53.668885275540994,"multilingual":57.613920075112844,"instruction":52.55971562317141,"math":53.918825756957276},"capabilityCounts":{"math":7,"agentic":13,"knowledge":6,"coding":8,"multimodal":1,"reasoning":7,"instruction":2,"multilingual":1},"benchmarks":[{"benchmarkId":"vals-ai:aime:overall","benchmarkName":"AIME","category":"math","domain":null,"eligible":true,"ranked":true,"score":95.625,"unit":"percent","family":"vals-ai:aime:overall","index":57.29506683176784,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/aime","measuredAt":"2026-04-16","stderr":0.409,"costPerTest":0.128636,"latency":300.051},{"benchmarkId":"tau2Bench","benchmarkName":"τ²-Bench Tool-Agent-User Evaluation","category":"agentic","score":93.4,"unit":"percent","family":"tau2-bench","index":64.76716879881687,"aboveFit":false,"run":null,"source":"Victor Barres, Honghua Dong, Soham Ray, Xujie Si, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2506.07982","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaGlobalMmluLite","benchmarkName":"Artificial Analysis Global-MMLU-Lite","category":"multilingual","score":89.4,"unit":"percent","family":"aaGlobalMmluLite","index":null,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/global-mmlu-lite","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":88.88888888888889,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":60.634117163950165,"aboveFit":false,"run":"xhigh effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":4.737793696791343,"costPerTest":null,"latency":null},{"benchmarkId":"gpqa","benchmarkName":"Graduate-Level Google-Proof Q&A","category":"knowledge","score":88,"unit":"percent","family":"gpqa-diamond","index":57.92029613079061,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":87.5,"unit":"percent","family":"gpqa-diamond","index":57.46170979333072,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":86.86868686868688,"unit":"percent","family":"gpqa-diamond","index":56.88268663997228,"aboveFit":false,"run":"xhigh effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.406315641682249,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:mmlu_pro:overall","benchmarkName":"MMLU Pro","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":84.554,"unit":"percent","family":"mmlu-pro","index":51.00864757330973,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","measuredAt":"2026-09-01","stderr":0.359,"costPerTest":0.00368,"latency":20.001},{"benchmarkId":"vals-ai:gpqa:overall","benchmarkName":"GPQA Diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":83.08,"unit":"percent","family":"gpqa-diamond","index":53.40780657018527,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","measuredAt":"2026-09-01","stderr":2.457,"costPerTest":0.012743,"latency":63.11},{"benchmarkId":"vals-ai:lcb:overall","benchmarkName":"LiveCodeBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":81.465,"unit":"percent","family":"livecodebench","index":56.938635724206925,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","measuredAt":"2026-09-01","stderr":1.094,"costPerTest":0.032935,"latency":188.757},{"benchmarkId":"vals-ai:mmmu:overall","benchmarkName":"MMMU Pro","category":"multimodal","domain":null,"eligible":true,"ranked":true,"score":79.249,"unit":"percent","family":"mmmu-pro","index":58.28843562626439,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmmu","measuredAt":"2026-09-01","stderr":0.975,"costPerTest":0.007061,"latency":50.813},{"benchmarkId":"livebench:livebench-math:overall","benchmarkName":"LiveBench Mathematics","category":"math","domain":null,"eligible":true,"ranked":true,"score":78.462,"unit":"percent","family":"livebench:livebench-math:overall","index":49.1288274588036,"aboveFit":false,"run":"xhigh effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Mathematics","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.14675729442970822,"latency":null},{"benchmarkId":"mmmuProPython","benchmarkName":"MMMU-Pro with Python","category":"multimodal","score":78,"unit":"percent","family":"mmmuProPython","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":77,"unit":"percent","family":"lcr","index":60.126249929266535,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmmuPro","benchmarkName":"Massive Multi-discipline Multimodal Understanding Pro","category":"multimodal","score":76.6,"unit":"percent","family":"mmmu-pro","index":54.84099453596372,"aboveFit":false,"run":null,"source":"MMMU-Pro authors","sourceUrl":"https://arxiv.org/abs/2409.02813","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"arc-prize:v1-public-eval:overall","benchmarkName":"ARC-AGI-1 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":75.12,"unit":"percent","family":"arc-prize:v1-public-eval:overall","index":null,"aboveFit":false,"run":"xhigh effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.35,"latency":null},{"benchmarkId":"aaMmmuPro","benchmarkName":"Artificial Analysis MMMU-Pro","category":"multimodal","score":73.3,"unit":"percent","family":"mmmu-pro","index":50.54633406107048,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmmu-pro","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaIfBench","benchmarkName":"Artificial Analysis IFBench","category":"instruction","score":73.3,"unit":"percent","family":"ifbench-family","index":57.52325745906747,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:swebench:overall","benchmarkName":"SWE-bench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":73,"unit":"percent","family":"swe-bench-verified","index":55.069630707681156,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/swebench","measuredAt":"2026-09-01","stderr":1.987,"costPerTest":0.511052,"latency":326.46},{"benchmarkId":"osWorldVerified","benchmarkName":"OSWorld-Verified","category":"agentic","score":72.1,"unit":"percent","family":"osWorldVerified","index":58.68424471194735,"aboveFit":false,"run":null,"source":"Tianbao Xie, Danyang Zhang, Jixuan Chen, Xiaochuan Li, Siheng Zhao, Ruisheng Cao, Toh Jing Hua, Zhoujun Cheng, Dongchan Shin, Fangyu Lei, Yitao Liu, Yiheng Xu, Shuyan Zhou, Silvio Savarese, Caiming Xiong, Victor Zhong, Tao Yu","sourceUrl":"https://os-world.github.io/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-coding:overall","benchmarkName":"LiveBench Coding","category":"coding","domain":null,"eligible":true,"ranked":true,"score":71.624,"unit":"percent","family":"livebench:livebench-coding:overall","index":55.556982677834,"aboveFit":false,"run":"xhigh effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Coding","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.22178727272727272,"latency":null},{"benchmarkId":"livebench:livebench-reasoning:overall","benchmarkName":"LiveBench Reasoning","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":71.32375,"unit":"percent","family":"livebench","index":54.01493181807737,"aboveFit":false,"run":"xhigh effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Reasoning","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.157035175879397,"latency":null},{"benchmarkId":"vals-ai:tax_eval_v2:overall","benchmarkName":"TaxEval v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":71.218,"unit":"percent","family":"vals-ai:tax_eval_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","measuredAt":"2026-09-01","stderr":0.9,"costPerTest":0.00677,"latency":44.578},{"benchmarkId":"livebench:livebench-language:overall","benchmarkName":"LiveBench Language","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":70.95166666666667,"unit":"percent","family":"livebench:livebench-language:overall","index":54.79156858144864,"aboveFit":false,"run":"xhigh effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Language","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.1302046979865772,"latency":null},{"benchmarkId":"livebench:livebench-data-analysis:overall","benchmarkName":"LiveBench Data Analysis","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":70.78566666666667,"unit":"percent","family":"livebench","index":53.31788037531751,"aboveFit":false,"run":"xhigh effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Data%20Analysis","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.19652266666666665,"latency":null},{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":68.93375,"unit":"percent","family":"euroeval","index":65.74747300451459,"aboveFit":false,"run":"high effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":0.5646970980091893,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-french:overall","benchmarkName":"EuroEval French","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":67.365,"unit":"percent","family":"euroeval","index":63.61904021360597,"aboveFit":false,"run":"high effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/french","measuredAt":null,"stderr":0.38508116027663575,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-french:overall","benchmarkName":"EuroEval French","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":67.29875,"unit":"percent","family":"euroeval","index":63.52915420729669,"aboveFit":false,"run":"medium effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/french","measuredAt":null,"stderr":0.42641492410561804,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":67.26750000000001,"unit":"percent","family":"euroeval","index":63.486755147716856,"aboveFit":false,"run":"medium effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":0.5557076007218185,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-french:overall","benchmarkName":"EuroEval French","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":67.10249999999999,"unit":"percent","family":"euroeval","index":63.262888113135205,"aboveFit":false,"run":"low effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/french","measuredAt":null,"stderr":0.4048244218176566,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-italian:overall","benchmarkName":"EuroEval Italian","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":66.18249999999999,"unit":"percent","family":"euroeval","index":62.014659799104344,"aboveFit":false,"run":"high effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/italian","measuredAt":null,"stderr":0.4676320000812605,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-italian:overall","benchmarkName":"EuroEval Italian","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":65.835,"unit":"percent","family":"euroeval","index":61.54318225657639,"aboveFit":false,"run":"medium effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/italian","measuredAt":null,"stderr":0.43338060639581,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":65.0375,"unit":"percent","family":"euroeval","index":60.46115825609854,"aboveFit":false,"run":"low effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":0.6631423772463949,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-portuguese:overall","benchmarkName":"EuroEval Portuguese","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":64.74,"unit":"percent","family":"euroeval","index":60.05751920889835,"aboveFit":false,"run":"high effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/portuguese","measuredAt":null,"stderr":0.4371516470288085,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-portuguese:overall","benchmarkName":"EuroEval Portuguese","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":63.6925,"unit":"percent","family":"euroeval","index":58.6363027317817,"aboveFit":false,"run":"medium effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/portuguese","measuredAt":null,"stderr":0.4777976820789318,"costPerTest":null,"latency":null},{"benchmarkId":"arc-prize:v1-semi-private:overall","benchmarkName":"ARC-AGI-1 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":63.67,"unit":"percent","family":"arc-prize:v1-semi-private:overall","index":56.258756536570615,"aboveFit":false,"run":"xhigh effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.51,"latency":null},{"benchmarkId":"vals-ai:mortgage_tax:overall","benchmarkName":"MortgageTax","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":63.514,"unit":"percent","family":"vals-ai:mortgage_tax:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mortgage_tax","measuredAt":"2026-09-01","stderr":0.906,"costPerTest":0.044769,"latency":126.454},{"benchmarkId":"euroeval:euroeval-dutch:overall","benchmarkName":"EuroEval Dutch","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":63.089999999999996,"unit":"percent","family":"euroeval","index":57.818848863082124,"aboveFit":false,"run":"high effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/dutch","measuredAt":null,"stderr":0.420578672489567,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-dutch:overall","benchmarkName":"EuroEval Dutch","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":63.04666666666666,"unit":"percent","family":"euroeval","index":57.76005550046473,"aboveFit":false,"run":"medium effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/dutch","measuredAt":null,"stderr":0.489166587797331,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-spanish:overall","benchmarkName":"EuroEval Spanish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":62.83125,"unit":"percent","family":"euroeval","index":57.467784649760944,"aboveFit":false,"run":"high effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/spanish","measuredAt":null,"stderr":0.42268302248848366,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":62.52142857142858,"unit":"percent","family":"euroeval","index":57.04742825906928,"aboveFit":false,"run":"high effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.4144728641323106,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-italian:overall","benchmarkName":"EuroEval Italian","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":62.462500000000006,"unit":"percent","family":"euroeval","index":56.9674757467187,"aboveFit":false,"run":"low effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/italian","measuredAt":null,"stderr":0.47128812843100554,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-dutch:overall","benchmarkName":"EuroEval Dutch","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":62.08222222222222,"unit":"percent","family":"euroeval","index":56.45152630169808,"aboveFit":false,"run":"low effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/dutch","measuredAt":null,"stderr":0.44399699698684153,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:corp_fin_v2:overall","benchmarkName":"CorpFin v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":60.917,"unit":"percent","family":"vals-ai:corp_fin_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","measuredAt":"2026-08-12","stderr":0.962,"costPerTest":0.076368,"latency":52.297},{"benchmarkId":"euroeval:euroeval-spanish:overall","benchmarkName":"EuroEval Spanish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":60.79875,"unit":"percent","family":"euroeval","index":54.71014981468733,"aboveFit":false,"run":"medium effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/spanish","measuredAt":null,"stderr":0.4281774602895393,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-portuguese:overall","benchmarkName":"EuroEval Portuguese","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":60.56875,"unit":"percent","family":"euroeval","index":54.39809273617962,"aboveFit":false,"run":"low effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/portuguese","measuredAt":null,"stderr":0.43502155119028296,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":60.22142857142858,"unit":"percent","family":"euroeval","index":53.92685747399212,"aboveFit":false,"run":"medium effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.44758079877109497,"costPerTest":null,"latency":null},{"benchmarkId":"terminalBench2","benchmarkName":"Terminal-Bench 2.0","category":"agentic","score":60,"unit":"percent","family":"terminalBench2","index":56.57836830564313,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-instruction-following:overall","benchmarkName":"LiveBench Instruction Following","category":"instruction","domain":null,"eligible":true,"ranked":true,"score":59.804,"unit":"percent","family":"livebench:livebench-instruction-following:overall","index":47.59617378727535,"aboveFit":false,"run":"xhigh effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=IF","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.1716585,"latency":null},{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":58.77428571428571,"unit":"percent","family":"euroeval","index":51.96341759493424,"aboveFit":false,"run":"low effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.5321864795159604,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-spanish:overall","benchmarkName":"EuroEval Spanish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":57.94625,"unit":"percent","family":"euroeval","index":50.839963656238375,"aboveFit":false,"run":"low effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/spanish","measuredAt":null,"stderr":0.47121186848380636,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":57.75625000000001,"unit":"percent","family":"euroeval","index":50.582177373992884,"aboveFit":false,"run":"high effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.46698668342898175,"costPerTest":null,"latency":null},{"benchmarkId":"mcpAtlas","benchmarkName":"MCP Atlas","category":"agentic","score":57.7,"unit":"percent","family":"mcpAtlas","index":49.91720920495081,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":56.324999999999996,"unit":"percent","family":"euroeval","index":48.64030044523561,"aboveFit":false,"run":"medium effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.46773222841707196,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:terminal-bench-2-1:overall","benchmarkName":"Terminal-Bench 2.1","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":54.682,"unit":"percent","family":"terminal-bench-2-1","index":53.122182247466384,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2-1","measuredAt":"2026-10-01","stderr":4.556,"costPerTest":0.286739,"latency":673.036},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":54.61500000000001,"unit":"percent","family":"euroeval","index":46.32022390502609,"aboveFit":false,"run":"low effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.5548620324008483,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:finance_agent:overall","benchmarkName":"Finance Agent v1.1","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":53.405,"unit":"percent","family":"vals-ai:finance_agent:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/finance_agent","measuredAt":"2026-06-04","stderr":2.837,"costPerTest":0.491653,"latency":876.254},{"benchmarkId":"terminalBenchHard","benchmarkName":"Terminal-Bench Hard","category":"agentic","score":52.3,"unit":"percent","family":"terminalBenchHard","index":62.01370029642923,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/terminalbench-hard","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaSciCode","benchmarkName":"Artificial Analysis SciCode","category":"coding","score":52.1,"unit":"percent","family":"scicode-family","index":63.76210717071886,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/scicode","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:case_law_v2:overall","benchmarkName":"CaseLaw v2","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":51.661,"unit":"percent","family":"vals-ai:case_law_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/case_law_v2","measuredAt":"2026-05-04","stderr":0.934,"costPerTest":0.040877,"latency":57.683},{"benchmarkId":"epoch-ai:epoch-frontiermath-tiers-1-3-v2-private:overall","benchmarkName":"FrontierMath-Tiers-1-3-v2-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":51.2280701754386,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tiers-1-3-v2-private:overall","index":57.85867870869576,"aboveFit":false,"run":"xhigh effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.966059084324674,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-2025-02-28-public:overall","benchmarkName":"FrontierMath-2025-02-28-Public","category":"math","domain":null,"eligible":false,"ranked":true,"score":50,"unit":"percent","family":"epoch-ai:epoch-frontiermath-2025-02-28-public:overall","index":null,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":16.666666666666664,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:vibe-code:overall","benchmarkName":"Vibe Code Bench v1.1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":47.969,"unit":"percent","family":"vals-ai:vibe-code:overall","index":59.55749144047479,"aboveFit":false,"run":"OpenHands","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","measuredAt":"2026-11-09","stderr":5.606,"costPerTest":1.191299,"latency":2055.117},{"benchmarkId":"vals-ai:emb:overall","benchmarkName":"EMB","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":45.426,"unit":"percent","family":"vals-ai:emb:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/emb","measuredAt":"2026-10-20","stderr":3.645,"costPerTest":2.146984,"latency":2601.215},{"benchmarkId":"vals-ai:fabv2:overall","benchmarkName":"Finance Agent (v2)","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":45.36,"unit":"percent","family":"vals-ai:fabv2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/fabv2","measuredAt":"2026-10-29","stderr":0.447,"costPerTest":1.201815,"latency":2153.503},{"benchmarkId":"vals-ai:terminal-bench-2:overall","benchmarkName":"Terminal-Bench 2.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":44.944,"unit":"percent","family":"vals-ai:terminal-bench-2:overall","index":53.61255278791465,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2","measuredAt":"2026-06-04","stderr":4.492,"costPerTest":0.304702,"latency":747.185},{"benchmarkId":"toolathlon","benchmarkName":"Toolathlon","category":"agentic","score":42.9,"unit":"percent","family":"toolathlon","index":56.95626644803549,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-agentic-coding:overall","benchmarkName":"LiveBench Agentic Coding","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":41.666666666666664,"unit":"percent","family":"livebench:livebench-agentic-coding:overall","index":52.132814042603734,"aboveFit":false,"run":"xhigh effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Agentic%20Coding","measuredAt":"2026-06-25","stderr":null,"costPerTest":1.1763569444444444,"latency":null},{"benchmarkId":"hle","benchmarkName":"Humanity's Last Exam","category":"knowledge","score":41.5,"unit":"percent","family":"hle","index":66.02329323204994,"aboveFit":false,"run":null,"source":"Center for AI Safety, Scale AI, and thousands of expert contributors","sourceUrl":"https://lastexam.ai/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"omniscienceAccuracy","benchmarkName":"Artificial Analysis Omniscience Accuracy","category":"knowledge","score":37.5,"unit":"percent","family":"omniscienceAccuracy","index":57.9495498688572,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaItbench","benchmarkName":"Artificial Analysis ITBench-AA","category":"agentic","score":35.2,"unit":"percent","family":"aaItbench","index":51.54871956785933,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/itbench-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-simpleqa-verified:overall","benchmarkName":"SimpleQA Verified","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":29.4,"unit":"percent","family":"epoch-ai:epoch-simpleqa-verified:overall","index":48.4118476898252,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.4414290540008265,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-2025-02-28-private:overall","benchmarkName":"FrontierMath-2025-02-28-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":28.28,"unit":"percent","family":"epoch-ai:epoch-frontiermath-2025-02-28-private:overall","index":54.879124630935024,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.0650000000000004,"costPerTest":null,"latency":null},{"benchmarkId":"apexAgentsAa","benchmarkName":"APEX-Agents-AA","category":"agentic","score":28.2,"unit":"percent","family":"apexAgentsAa","index":61.45514026671307,"aboveFit":false,"run":null,"source":"Artificial Analysis / Mercor","sourceUrl":"https://artificialanalysis.ai/evaluations/apex-agents-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hleNoTools","benchmarkName":"Humanity's Last Exam without tools","category":"knowledge","score":28.2,"unit":"percent","family":"hle","index":52.679499723153675,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":28.1,"unit":"percent","family":"hle","index":52.5791704486507,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"frontierCode","benchmarkName":"FrontierCode 1.1 Main","category":"coding","score":27,"unit":"percent","family":"frontierCode","index":54.876848114175765,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gdpvalAaNormalized","benchmarkName":"GDPval-AA normalized","category":"agentic","score":25.6,"unit":"percent","family":"gdpvalAaNormalized","index":52.33718001650369,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"arc-prize:v2-semi-private:overall","benchmarkName":"ARC-AGI-2 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":18.9,"unit":"percent","family":"arc-prize:v2-semi-private:overall","index":47.866161250691896,"aboveFit":false,"run":"xhigh effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.75,"latency":null},{"benchmarkId":"arc-prize:v2-public-eval:overall","benchmarkName":"ARC-AGI-2 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":17.78,"unit":"percent","family":"arc-prize:v2-public-eval:overall","index":null,"aboveFit":false,"run":"xhigh effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.81,"latency":null},{"benchmarkId":"epoch-ai:epoch-chess-puzzles:overall","benchmarkName":"Chess Puzzles","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":17,"unit":"percent","family":"epoch-ai:epoch-chess-puzzles:overall","index":46.88793404269057,"aboveFit":false,"run":"xhigh effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":3.7752516806863685,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:code-migration:overall","benchmarkName":"Code Migration","category":"coding","domain":null,"eligible":true,"ranked":true,"score":12.936,"unit":"percent","family":"vals-ai:code-migration:overall","index":52.0612592967563,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/code-migration","measuredAt":"2026-10-29","stderr":3.641,"costPerTest":1.953198,"latency":1733.471},{"benchmarkId":"vals-ai:legal_research:overall","benchmarkName":"Legal Research Bench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":12.5,"unit":"percent","family":"vals-ai:legal_research:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_research","measuredAt":"2026-11-09","stderr":2.299,"costPerTest":1.864673,"latency":2256.339},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":10,"unit":"percent","family":"critpt","index":56.20794917976178,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tier-4-v2-private:overall","benchmarkName":"FrontierMath-Tier-4-v2-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":9.75609756097561,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tier-4-v2-private:overall","index":52.32098780254311,"aboveFit":false,"run":"xhigh effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":4.691557088212523,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:tax_agent_bench:overall","benchmarkName":"Tax Agent Bench","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":8.808,"unit":"percent","family":"vals-ai:tax_agent_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench","measuredAt":"2026-11-09","stderr":2.045,"costPerTest":0.806171,"latency":1006.185},{"benchmarkId":"epoch-ai:epoch-mystery-game-puzzles:overall","benchmarkName":"Mystery Game Puzzles","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":7.000000000000001,"unit":"percent","family":"epoch-ai:epoch-mystery-game-puzzles:overall","index":44.1019024790064,"aboveFit":false,"run":"medium effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.564323999762428,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:ioi-v1:overall","benchmarkName":"IOI v1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":6.417,"unit":"percent","family":"vals-ai:ioi-v1:overall","index":43.33321058532131,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/ioi-v1","measuredAt":"2026-08-09","stderr":3.607,"costPerTest":0.35057,"latency":1896.804},{"benchmarkId":"vals-ai:terminal-bench-4:overall","benchmarkName":"Terminal-Bench 4.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":2.525,"unit":"percent","family":"terminal-bench-4","index":55.714981010188346,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-4","measuredAt":"2026-11-09","stderr":1.01,"costPerTest":1.438224,"latency":1770.424},{"benchmarkId":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-private:overall","benchmarkName":"FrontierMath-Tier-4-2025-07-01-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":2.08,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-private:overall","index":45.314977702005415,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.08,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:programbench:overall","benchmarkName":"ProgramBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":0,"unit":"percent","family":"vals-ai:programbench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/programbench","measuredAt":"2026-11-09","stderr":0,"costPerTest":null,"latency":3225.344}],"price":{"inputPerMillion":0.75,"outputPerMillion":4.5,"batch":{"inputPerMillion":0.375,"outputPerMillion":2.25},"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":95,"timeToFirstToken":1.264,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":{"elo":1411.8315752456379,"eloUpper":1415.5578050222043,"eloLower":1408.1053454690714,"votes":62795,"rank":140,"rankUpper":null,"rankLower":null,"arenaName":"GPT-5.4 mini-high","organization":"openai","updatedAt":"2026-11-09"},"quality":{"coveragePercent":100,"benchmarkCount":45,"categoryCount":8,"confidence":"high","state":"available"},"hosts":[{"host":"databricks","name":"Databricks","kind":"cloud","url":"https://docs.databricks.com/aws/en/machine-learning/foundation-models/","inputPerMillion":0.75,"outputPerMillion":4.5,"cacheReadPerMillion":0.075,"contextTokens":400000,"maxOutputTokens":128000,"deprecated":false},{"host":"digitalocean","name":"DigitalOcean","kind":"cloud","url":"https://docs.digitalocean.com/products/gradient-ai-platform/details/models/","inputPerMillion":0.75,"outputPerMillion":4.5,"cacheReadPerMillion":0.075,"contextTokens":400000,"maxOutputTokens":128000,"deprecated":false},{"host":"azure","name":"Microsoft Azure","kind":"cloud","url":"https://learn.microsoft.com/en-us/azure/ai-services/openai/concepts/models","inputPerMillion":0.75,"outputPerMillion":4.5,"cacheReadPerMillion":0.075,"contextTokens":400000,"maxOutputTokens":128000,"deprecated":false},{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":0.75,"outputPerMillion":4.5,"cacheReadPerMillion":0.075,"contextTokens":400000,"maxOutputTokens":128000,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:livebench","name":"LiveBench","url":"https://livebench.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:arc-prize-foundation","name":"ARC Prize Foundation","url":"https://arcprize.org","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"qwen3-5-397b","providerId":"alibaba","slug":"qwen3-5-397b","name":"Qwen3.5 397B","provider":"Alibaba","description":"Qwen3.5 397B is a standard model from Alibaba. 46 benchmarks count toward its index, in 8 categories.","logoUrl":null,"releaseDate":"2026-02-16","contextWindowTokens":128000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":53.81534845775686,"overallScoreRange":{"low":51.431212026732176,"high":56.19948488878155},"badges":[],"capabilityScores":{"agentic":53.27593142768724,"coding":53.28079444387869,"reasoning":51.91677321755898,"multimodal":55.594768631605646,"knowledge":53.82338966709293,"multilingual":55.63608766251952,"instruction":55.629647029316885,"math":53.60173315709446},"capabilityCounts":{"multimodal":11,"agentic":13,"knowledge":6,"math":4,"instruction":2,"multilingual":2,"coding":5,"reasoning":3},"benchmarks":[{"benchmarkId":"countBench","benchmarkName":"CountBench","category":"multimodal","score":97.2,"unit":"percent","family":"countBench","index":53.18394478845735,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vStar","benchmarkName":"V*","category":"multimodal","score":95.8,"unit":"percent","family":"vStar","index":56.302892776677126,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"tau2Bench","benchmarkName":"τ²-Bench Tool-Agent-User Evaluation","category":"agentic","score":95.6,"unit":"percent","family":"tau2-bench","index":66.22702964504438,"aboveFit":false,"run":null,"source":"Victor Barres, Honghua Dong, Soham Ray, Xujie Si, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2506.07982","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmluRedux","benchmarkName":"MMLU-Redux","category":"knowledge","score":94.9,"unit":"percent","family":"mmluRedux","index":51.14780933849774,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hmmtFeb2025","benchmarkName":"Harvard-MIT Mathematics Tournament February 2025","category":"math","score":94.8,"unit":"percent","family":"hmmtFeb2025","index":52.73028783335306,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"ai2dTest","benchmarkName":"AI2D test split","category":"multimodal","score":93.9,"unit":"percent","family":"ai2dTest","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aime2026","benchmarkName":"AIME 2026","category":"math","score":93.3,"unit":"percent","family":"aime2026","index":52.461427001915894,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"cEval","benchmarkName":"C-Eval","category":"knowledge","score":93,"unit":"percent","family":"cEval","index":49.82190344288738,"aboveFit":false,"run":null,"source":"C-Eval authors","sourceUrl":"https://arxiv.org/abs/2305.08322","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hmmtNov2025","benchmarkName":"Harvard-MIT Mathematics Tournament November 2025","category":"math","score":92.7,"unit":"percent","family":"hmmtNov2025","index":54.30043752675107,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"ifeval","benchmarkName":"Instruction-Following Eval","category":"instruction","score":92.6,"unit":"percent","family":"ifeval","index":49.67091542505794,"aboveFit":false,"run":null,"source":"Jeffrey Zhou, Tianjian Lu, Swaroop Mishra, Siddhartha Brahma, Sujoy Basu, Yi Luan, Denny Zhou, Le Hou","sourceUrl":"https://arxiv.org/abs/2311.07911","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"refcocoAvg","benchmarkName":"RefCOCO average","category":"multimodal","score":92.3,"unit":"percent","family":"refcocoAvg","index":51.871708724230714,"aboveFit":false,"run":null,"source":"RefCOCO dataset authors","sourceUrl":"https://github.com/lichengunc/refer","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"omniDocBench15","benchmarkName":"OmniDocBench 1.5","category":"multimodal","score":90.8,"unit":"percent","family":"omniDocBench15","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":89.3,"unit":"percent","family":"gpqa-diamond","index":59.11262060818633,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mathVision","benchmarkName":"MathVision","category":"multimodal","score":88.6,"unit":"percent","family":"mathVision","index":59.55578412592234,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gpqa","benchmarkName":"Graduate-Level Google-Proof Q&A","category":"knowledge","score":88.4,"unit":"percent","family":"gpqa-diamond","index":58.28716520075854,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"maxife","benchmarkName":"MAXIFE","category":"multilingual","score":88.2,"unit":"percent","family":"maxife","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"weMath","benchmarkName":"We-Math","category":"multimodal","score":87.9,"unit":"percent","family":"weMath","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hmmtFeb2026","benchmarkName":"Harvard-MIT Mathematics Tournament February 2026","category":"math","score":87.9,"unit":"percent","family":"hmmtFeb2026","index":54.91478026635781,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmluPro","benchmarkName":"Massive Multitask Language Understanding Professional","category":"knowledge","score":87.8,"unit":"percent","family":"mmlu-pro","index":55.09821903709725,"aboveFit":false,"run":null,"source":"Yubo Wang, Xueguang Ma, Ge Zhang, Yuansheng Ni, Abhranil Chandra, Shiguang Guo, Weiming Ren, Aaran Arulraj, Xuan He, Ziyan Jiang, Tianle Li, Max Ku, Kai Wang, Alex Zhuang, Rongqi Fan, Xiang Yue, Wenhu Chen","sourceUrl":"https://arxiv.org/abs/2406.01574","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"videoMmeWithSub","benchmarkName":"Video-MME with subtitle","category":"multimodal","score":87.5,"unit":"percent","family":"videoMmeWithSub","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mlvuAvg","benchmarkName":"MLVU mean average","category":"multimodal","score":86.7,"unit":"percent","family":"mlvuAvg","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"dynaMath","benchmarkName":"DynaMath","category":"multimodal","score":86.3,"unit":"percent","family":"dynaMath","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"include","benchmarkName":"INCLUDE","category":"multilingual","score":85.6,"unit":"percent","family":"include","index":57.21949182139225,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmmu","benchmarkName":"Massive Multi-discipline Multimodal Understanding","category":"multimodal","score":85,"unit":"percent","family":"mmmu","index":54.10637994105606,"aboveFit":false,"run":null,"source":"MMMU authors","sourceUrl":"https://arxiv.org/abs/2401.05508","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"videoMmmu","benchmarkName":"VideoMMMU","category":"multimodal","score":84.7,"unit":"percent","family":"videoMmmu","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmluProX","benchmarkName":"MMLU-ProX","category":"multilingual","score":84.7,"unit":"percent","family":"mmluProX","index":54.0526835036468,"aboveFit":false,"run":null,"source":"MMLU-ProX authors","sourceUrl":"https://arxiv.org/abs/2503.10497","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"realWorldQa","benchmarkName":"RealWorldQA","category":"multimodal","score":83.9,"unit":"percent","family":"realWorldQa","index":57.230696483280155,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"videoMmeNoSub","benchmarkName":"Video-MME without subtitle","category":"multimodal","score":83.7,"unit":"percent","family":"videoMmeNoSub","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"liveCodeBenchV6","benchmarkName":"LiveCodeBench v6","category":"coding","score":83.6,"unit":"percent","family":"liveCodeBenchV6","index":53.05744860347197,"aboveFit":false,"run":null,"source":"LiveCodeBench maintainers","sourceUrl":"https://github.com/LiveCodeBench/LiveCodeBench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"ccOcr","benchmarkName":"CC-OCR","category":"multimodal","score":82,"unit":"percent","family":"ccOcr","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmAnswerBench","benchmarkName":"MMAnswerBench","category":"math","score":80.9,"unit":"percent","family":"mmAnswerBench","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"charxiv","benchmarkName":"CharXiv Reasoning","category":"multimodal","score":80.8,"unit":"percent","family":"charxiv","index":56.17487381718582,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmmuPro","benchmarkName":"Massive Multi-discipline Multimodal Understanding Pro","category":"multimodal","score":79,"unit":"percent","family":"mmmu-pro","index":57.96438397224972,"aboveFit":false,"run":null,"source":"MMMU-Pro authors","sourceUrl":"https://arxiv.org/abs/2409.02813","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaIfBench","benchmarkName":"Artificial Analysis IFBench","category":"instruction","score":78.8,"unit":"percent","family":"ifbench-family","index":62.66306584062974,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":77.3,"unit":"percent","family":"lcr","index":60.332074086942235,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaMmmuPro","benchmarkName":"Artificial Analysis MMMU-Pro","category":"multimodal","score":77.3,"unit":"percent","family":"mmmu-pro","index":55.75198312154714,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmmu-pro","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"ifBench","benchmarkName":"Instruction Following Benchmark","category":"instruction","score":76.5,"unit":"percent","family":"ifbench-family","index":60.51369142652189,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"sweVerified","benchmarkName":"Software Engineering Benchmark Verified","category":"coding","score":76.2,"unit":"percent","family":"swe-bench-verified","index":57.42981495658897,"aboveFit":false,"run":null,"source":"Carlos E. Jimenez, John Yang, Alexander Wettig, Shunyu Yao, Kexin Pei, Ofir Press, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2310.06770","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mcpTasks","benchmarkName":"MCP-Tasks","category":"agentic","score":74.2,"unit":"percent","family":"mcpTasks","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"wideResearch","benchmarkName":"WideResearch","category":"agentic","score":74,"unit":"percent","family":"wideResearch","index":56.78869729543727,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"polyMath","benchmarkName":"PolyMath","category":"multilingual","score":73.3,"unit":"percent","family":"polyMath","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"superGpqa","benchmarkName":"SuperGPQA: Scaling LLM Evaluation Across 285 Graduate Disciplines","category":"knowledge","score":70.4,"unit":"percent","family":"superGpqa","index":54.84087327217975,"aboveFit":false,"run":null,"source":"Xiaoxuan Du, Yao Yao, Kexin Ma, Bowen Wang, Tianyu Zheng, Kaiyan Zhu, Yiming Zhang, Yutao Zhu, Jiawei Zhou, Jingren Zhou","sourceUrl":"https://arxiv.org/abs/2502.14739","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"browseComp","benchmarkName":"BrowseComp","category":"agentic","score":69,"unit":"percent","family":"browseComp","index":54.42036066455064,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aiNeedle","benchmarkName":"AI-Needle","category":"reasoning","score":68.7,"unit":"percent","family":"aiNeedle","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"tau3Bench","benchmarkName":"τ³-Bench Tool-Agent-User Evaluation","category":"agentic","score":68.4,"unit":"percent","family":"tau3Bench","index":58.153169074630725,"aboveFit":false,"run":null,"source":"Sierra Research","sourceUrl":"https://github.com/sierra-research/tau2-bench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"erqa","benchmarkName":"ERQA","category":"multimodal","score":67.5,"unit":"percent","family":"erqa","index":60.67143162312397,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"simpleVqa","benchmarkName":"SimpleVQA","category":"multimodal","score":67.1,"unit":"percent","family":"simpleVqa","index":57.65632694841929,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"screenSpotPro","benchmarkName":"ScreenSpot Pro","category":"multimodal","score":65.6,"unit":"percent","family":"screenSpotPro","index":47.93023217241084,"aboveFit":false,"run":null,"source":"Kaixin Li, Ziyang Meng, Hongzhan Lin, Ziyang Luo, Yuchen Tian, Jing Ma, Zhiyong Huang, Tat-Seng Chua","sourceUrl":"https://arxiv.org/abs/2504.07981","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"longBenchV2","benchmarkName":"LongBench v2","category":"reasoning","score":63.2,"unit":"percent","family":"longBenchV2","index":55.27812791457371,"aboveFit":false,"run":null,"source":"LongBench v2 authors","sourceUrl":"https://arxiv.org/abs/2412.15204","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"osWorldVerified","benchmarkName":"OSWorld-Verified","category":"agentic","score":62.2,"unit":"percent","family":"osWorldVerified","index":50.692399821149706,"aboveFit":false,"run":null,"source":"Tianbao Xie, Danyang Zhang, Jixuan Chen, Xiaochuan Li, Siheng Zhao, Ruisheng Cao, Toh Jing Hua, Zhoujun Cheng, Dongchan Shin, Fangyu Lei, Yitao Liu, Yiheng Xu, Shuyan Zhou, Silvio Savarese, Caiming Xiong, Victor Zhong, Tao Yu","sourceUrl":"https://os-world.github.io/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmLongBenchDoc","benchmarkName":"MMLongBench-Doc","category":"multimodal","score":61.5,"unit":"percent","family":"mmLongBenchDoc","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"nova63","benchmarkName":"NOVA-63","category":"multilingual","score":59.1,"unit":"percent","family":"nova63","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"clawEval","benchmarkName":"Claw-Eval","category":"agentic","score":56.8,"unit":"percent","family":"clawEval","index":47.24739616158037,"aboveFit":false,"run":null,"source":"Bowen Ye, Rang Li, Qibin Yang, Yuanxin Liu, Linli Yao, Hanglong Lv, Zhihui Xie, Chenxin An, Lei Li, Lingpeng Kong, Qi Liu, Zhifang Sui, Tong Yang","sourceUrl":"https://arxiv.org/abs/2604.06132","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"terminalBench2","benchmarkName":"Terminal-Bench 2.0","category":"agentic","score":52.5,"unit":"percent","family":"terminalBench2","index":51.24327705286194,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"qwenClawBench","benchmarkName":"QwenClawBench","category":"agentic","score":51.8,"unit":"percent","family":"qwenClawBench","index":50.53605272313763,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"swePro","benchmarkName":"SWE-bench Pro","category":"coding","score":50.9,"unit":"percent","family":"swePro","index":52.18633171638186,"aboveFit":false,"run":null,"source":"Xiang Deng, Jeff Da, Edwin Pan, Yannis Yiming He, Charles Ide, Kanak Garg, Niklas Lauffer, Andrew Park, Nitin Pasari, Chetan Rane, Karmini Sampath, Maya Krishnan, Srivatsa Kundurthy, Sean Hendryx, Zifan Wang, Chen Bo Calvin Zhang, Noah Jacobson, Bing Liu, Brad Kenstler","sourceUrl":"https://arxiv.org/abs/2509.16941","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"odinw13","benchmarkName":"ODINW13","category":"multimodal","score":47,"unit":"percent","family":"odinw13","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gertLabs","benchmarkName":"Gert Labs Composite Game Benchmark","category":"agentic","score":46.76,"unit":"percent","family":"gertLabs","index":53.40911382673282,"aboveFit":false,"run":null,"source":"Gert Labs","sourceUrl":"https://gertlabs.com/rankings","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mcpAtlas","benchmarkName":"MCP Atlas","category":"agentic","score":46.1,"unit":"percent","family":"mcpAtlas","index":42.10582029051813,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaSciCode","benchmarkName":"Artificial Analysis SciCode","category":"coding","score":44.8,"unit":"percent","family":"scicode-family","index":54.24516180189131,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/scicode","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vitaBench","benchmarkName":"VITA-Bench","category":"agentic","score":43.7,"unit":"percent","family":"vitaBench","index":57.528439246414656,"aboveFit":false,"run":null,"source":"Meituan LongCat Team","sourceUrl":"https://vitabench.github.io/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"deepPlanning","benchmarkName":"DeepPlanning","category":"agentic","score":37.6,"unit":"percent","family":"deepPlanning","index":null,"aboveFit":false,"run":null,"source":"DeepPlanning authors","sourceUrl":"https://arxiv.org/abs/2601.18137","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"toolathlon","benchmarkName":"Toolathlon","category":"agentic","score":36.3,"unit":"percent","family":"toolathlon","index":51.64522949821161,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"nl2Repo","benchmarkName":"NL2Repo","category":"coding","score":32.2,"unit":"percent","family":"nl2Repo","index":49.485215141059356,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":29,"unit":"percent","family":"hle","index":53.482133919177514,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hle","benchmarkName":"Humanity's Last Exam","category":"knowledge","score":28.7,"unit":"percent","family":"hle","index":53.18114609566857,"aboveFit":false,"run":null,"source":"Center for AI Safety, Scale AI, and thousands of expert contributors","sourceUrl":"https://lastexam.ai/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"apexAgentsAa","benchmarkName":"APEX-Agents-AA","category":"agentic","score":15.3,"unit":"percent","family":"apexAgentsAa","index":52.590123259664246,"aboveFit":false,"run":null,"source":"Artificial Analysis / Mercor","sourceUrl":"https://artificialanalysis.ai/evaluations/apex-agents-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"researchClawBench","benchmarkName":"ResearchClawBench","category":"agentic","score":14.2,"unit":"percent","family":"researchClawBench","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":1.7,"unit":"percent","family":"critpt","index":40.14011765116101,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":0.6,"outputPerMillion":3.6,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":59,"timeToFirstToken":56.62,"source":"BenchLM","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":100,"benchmarkCount":46,"categoryCount":8,"confidence":"high","state":"available"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gpt-5-1-codex-max","providerId":"openai","slug":"gpt-5-1-codex-max","name":"GPT-5.1-Codex-Max","provider":"OpenAI","description":"GPT-5.1-Codex-Max is a reasoning model from OpenAI. 12 benchmarks count toward its index, in 6 categories.","logoUrl":null,"releaseDate":"2025-11-19","contextWindowTokens":400000,"modalities":["text","image"],"openWeights":false,"details":{"apiId":"openai/gpt-5.1-codex-max","maxOutputTokens":128000,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.125,"weightsUrl":null},"reasoning":true,"overallScore":53.79137138409565,"overallScoreRange":{"low":47.623494457697525,"high":59.95924831049377},"badges":[],"capabilityScores":{"agentic":57.490938796632285,"coding":53.23981304732745,"reasoning":51.36354040010676,"multimodal":49.505204248975154,"knowledge":55.743794290710106,"multilingual":null,"instruction":54.43937243013011,"math":null},"capabilityCounts":{"knowledge":3,"coding":3,"agentic":2,"multimodal":1,"instruction":1,"reasoning":2},"benchmarks":[{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":86,"unit":"percent","family":"gpqa-diamond","index":56.085950780951045,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:lcb:overall","benchmarkName":"LiveCodeBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":83.558,"unit":"percent","family":"livecodebench","index":58.67528411650966,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","measuredAt":"2026-09-01","stderr":1.055,"costPerTest":null,"latency":66.201},{"benchmarkId":"tau2Bench","benchmarkName":"τ²-Bench Tool-Agent-User Evaluation","category":"agentic","score":83,"unit":"percent","family":"tau2-bench","index":57.86600843483216,"aboveFit":false,"run":null,"source":"Victor Barres, Honghua Dong, Soham Ray, Xujie Si, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2506.07982","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaMmmuPro","benchmarkName":"Artificial Analysis MMMU-Pro","category":"multimodal","score":72.5,"unit":"percent","family":"mmmu-pro","index":49.505204248975154,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmmu-pro","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaIfBench","benchmarkName":"Artificial Analysis IFBench","category":"instruction","score":70,"unit":"percent","family":"ifbench-family","index":54.43937243013011,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":69.3,"unit":"percent","family":"lcr","index":54.843429882256956,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:terminal-bench:overall","benchmarkName":"Terminal-Bench 1.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":53.75,"unit":"percent","family":"vals-ai:terminal-bench:overall","index":57.1158691584324,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench","measuredAt":"2026-01-12","stderr":5.574,"costPerTest":1.018534,"latency":711.822},{"benchmarkId":"omniscienceAccuracy","benchmarkName":"Artificial Analysis Omniscience Accuracy","category":"knowledge","score":39.9,"unit":"percent","family":"omniscienceAccuracy","index":60.97416423060008,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":25.7,"unit":"percent","family":"hle","index":50.17126786057919,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:vibe-code:overall","benchmarkName":"Vibe Code Bench v1.1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":22.168,"unit":"percent","family":"vals-ai:vibe-code:overall","index":50.248085276362694,"aboveFit":false,"run":"OpenHands","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","measuredAt":"2026-11-09","stderr":3.84,"costPerTest":7.324105,"latency":31182.938},{"benchmarkId":"vals-ai:ioi-v1:overall","benchmarkName":"IOI v1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":21.416,"unit":"percent","family":"vals-ai:ioi-v1:overall","index":50.796069749109996,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/ioi-v1","measuredAt":"2026-08-09","stderr":6.995,"costPerTest":0.422645,"latency":956.692},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":5.7,"unit":"percent","family":"critpt","index":47.883650917956565,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":1.25,"outputPerMillion":10,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":55,"timeToFirstToken":3.153,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":85,"benchmarkCount":12,"categoryCount":6,"confidence":"low","state":"available"},"hosts":[{"host":"azure","name":"Microsoft Azure","kind":"cloud","url":"https://learn.microsoft.com/en-us/azure/ai-services/openai/concepts/models","inputPerMillion":1.25,"outputPerMillion":10,"cacheReadPerMillion":0.125,"contextTokens":400000,"maxOutputTokens":128000,"deprecated":false},{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":1.25,"outputPerMillion":10,"cacheReadPerMillion":0.125,"contextTokens":400000,"maxOutputTokens":128000,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"claude-sonnet-4-5-thinking","providerId":"anthropic","slug":"claude-sonnet-4-5-thinking","name":"Claude Sonnet 4.5 Thinking","provider":"Anthropic","description":"Claude Sonnet 4.5 Thinking is a reasoning model from Anthropic in the Claude Sonnet 4.5 family. 11 benchmarks count toward its index, in 6 categories.","logoUrl":null,"releaseDate":"2025-09-29","contextWindowTokens":200000,"modalities":["text"],"openWeights":false,"details":null,"reasoning":true,"overallScore":53.78051354330713,"overallScoreRange":{"low":46.90507510148469,"high":60.65595198512957},"badges":[],"capabilityScores":{"agentic":57.06316467782892,"coding":50.61066218629025,"reasoning":null,"multimodal":58.362616125376185,"knowledge":53.31037488620527,"multilingual":60.390775817195994,"instruction":null,"math":54.04563160293556},"capabilityCounts":{"math":1,"knowledge":2,"multimodal":1,"coding":4,"multilingual":1,"agentic":2},"benchmarks":[{"benchmarkId":"vals-ai:medqa:overall","benchmarkName":"MedQA","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":94.708,"unit":"percent","family":"vals-ai:medqa:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medqa","measuredAt":"2026-04-16","stderr":0.204,"costPerTest":0.031308,"latency":35.776},{"benchmarkId":"vals-ai:mgsm:overall","benchmarkName":"MGSM","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":94.327,"unit":"percent","family":"vals-ai:mgsm:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mgsm","measuredAt":"2026-01-09","stderr":1.416,"costPerTest":0.011286,"latency":10.272},{"benchmarkId":"vals-ai:aime:overall","benchmarkName":"AIME","category":"math","domain":null,"eligible":true,"ranked":true,"score":88.19,"unit":"percent","family":"vals-ai:aime:overall","index":54.04563160293556,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/aime","measuredAt":"2026-04-16","stderr":0.686,"costPerTest":0.266927,"latency":217.504},{"benchmarkId":"vals-ai:mmlu_pro:overall","benchmarkName":"MMLU Pro","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":87.357,"unit":"percent","family":"mmlu-pro","index":54.5400920628342,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","measuredAt":"2026-09-01","stderr":0.391,"costPerTest":0.040152,"latency":38.2},{"benchmarkId":"vals-ai:medscribe:overall","benchmarkName":"MedScribe","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":84.101,"unit":"percent","family":"vals-ai:medscribe:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medscribe","measuredAt":"2026-10-29","stderr":1.873,"costPerTest":0.082281,"latency":67.924},{"benchmarkId":"vals-ai:legal_bench:overall","benchmarkName":"LegalBench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":84.084,"unit":"percent","family":"vals-ai:legal_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","measuredAt":"2026-10-13","stderr":0.447,"costPerTest":null,"latency":13.286},{"benchmarkId":"vals-ai:gpqa:overall","benchmarkName":"GPQA Diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":81.633,"unit":"percent","family":"gpqa-diamond","index":52.080657709576336,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","measuredAt":"2026-09-01","stderr":2.254,"costPerTest":0.114262,"latency":117.251},{"benchmarkId":"vals-ai:mmmu:overall","benchmarkName":"MMMU Pro","category":"multimodal","domain":null,"eligible":true,"ranked":true,"score":79.306,"unit":"percent","family":"mmmu-pro","index":58.362616125376185,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmmu","measuredAt":"2026-09-01","stderr":0.974,"costPerTest":0.058232,"latency":56.361},{"benchmarkId":"vals-ai:tax_eval_v2:overall","benchmarkName":"TaxEval v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":73.303,"unit":"percent","family":"vals-ai:tax_eval_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","measuredAt":"2026-09-01","stderr":0.861,"costPerTest":0.040814,"latency":48.391},{"benchmarkId":"vals-ai:lcb:overall","benchmarkName":"LiveCodeBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":72.996,"unit":"percent","family":"livecodebench","index":49.91155725578272,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","measuredAt":"2026-09-01","stderr":1.125,"costPerTest":0.135104,"latency":109.547},{"benchmarkId":"vals-ai:swebench:overall","benchmarkName":"SWE-bench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":70,"unit":"percent","family":"swe-bench-verified","index":52.85695797433008,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/swebench","measuredAt":"2026-09-01","stderr":2.051,"costPerTest":0.807156,"latency":360.558},{"benchmarkId":"euroeval:euroeval-french:overall","benchmarkName":"EuroEval French","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":66.91875,"unit":"percent","family":"euroeval","index":63.01358164280569,"aboveFit":false,"run":"thinking effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/french","measuredAt":null,"stderr":0.4251268193139548,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":65.985,"unit":"percent","family":"euroeval","index":61.74669774255969,"aboveFit":false,"run":"thinking effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":0.6293995154113164,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-dutch:overall","benchmarkName":"EuroEval Dutch","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":65.31444444444445,"unit":"percent","family":"euroeval","index":60.83690814410845,"aboveFit":false,"run":"thinking effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/dutch","measuredAt":null,"stderr":0.37732652787999854,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-italian:overall","benchmarkName":"EuroEval Italian","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":65.125,"unit":"percent","family":"euroeval","index":60.57987562292214,"aboveFit":false,"run":"thinking effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/italian","measuredAt":null,"stderr":0.3269389277831565,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-portuguese:overall","benchmarkName":"EuroEval Portuguese","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":64.84625000000001,"unit":"percent","family":"euroeval","index":60.20167601146987,"aboveFit":false,"run":"thinking effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/portuguese","measuredAt":null,"stderr":0.4244389679329644,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:mortgage_tax:overall","benchmarkName":"MortgageTax","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":63.99,"unit":"percent","family":"vals-ai:mortgage_tax:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mortgage_tax","measuredAt":"2026-09-01","stderr":0.955,"costPerTest":0.052096,"latency":47.622},{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":63.82714285714286,"unit":"percent","family":"euroeval","index":58.81898210848575,"aboveFit":false,"run":"thinking effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.32377871001847003,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:case_law_v2:overall","benchmarkName":"CaseLaw v2","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":62.165,"unit":"percent","family":"vals-ai:case_law_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/case_law_v2","measuredAt":"2026-05-04","stderr":1.546,"costPerTest":0.255566,"latency":25.416},{"benchmarkId":"vals-ai:corp_fin_v2:overall","benchmarkName":"CorpFin v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":61.966,"unit":"percent","family":"vals-ai:corp_fin_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","measuredAt":"2026-08-12","stderr":0.956,"costPerTest":0.255071,"latency":20.544},{"benchmarkId":"vals-ai:terminal-bench:overall","benchmarkName":"Terminal-Bench 1.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":61.25,"unit":"percent","family":"vals-ai:terminal-bench:overall","index":62.543493610401654,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench","measuredAt":"2026-01-12","stderr":5.447,"costPerTest":0.660783,"latency":525.778},{"benchmarkId":"euroeval:euroeval-spanish:overall","benchmarkName":"EuroEval Spanish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":60.3975,"unit":"percent","family":"euroeval","index":54.16574588968202,"aboveFit":false,"run":"thinking effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/spanish","measuredAt":null,"stderr":0.43272393046837615,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":57.07625,"unit":"percent","family":"euroeval","index":49.65957383753528,"aboveFit":false,"run":"thinking effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.489457607970292,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:finance_agent:overall","benchmarkName":"Finance Agent v1.1","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":54.5,"unit":"percent","family":"vals-ai:finance_agent:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/finance_agent","measuredAt":"2026-06-04","stderr":2.86,"costPerTest":1.102422,"latency":202.068},{"benchmarkId":"vals-ai:medcode:overall","benchmarkName":"MedCode","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":44.134,"unit":"percent","family":"vals-ai:medcode:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medcode","measuredAt":"2026-10-29","stderr":1.998,"costPerTest":0.101495,"latency":74.331},{"benchmarkId":"vals-ai:terminal-bench-2:overall","benchmarkName":"Terminal-Bench 2.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":41.573,"unit":"percent","family":"vals-ai:terminal-bench-2:overall","index":51.5828357452562,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2","measuredAt":"2026-06-04","stderr":5.254,"costPerTest":0.751119,"latency":736.918},{"benchmarkId":"vals-ai:vibe-code:overall","benchmarkName":"Vibe Code Bench v1.1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":22.621,"unit":"percent","family":"vals-ai:vibe-code:overall","index":50.41153479352648,"aboveFit":false,"run":"OpenHands","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","measuredAt":"2026-11-09","stderr":3.728,"costPerTest":6.656984,"latency":2962.133},{"benchmarkId":"vals-ai:ioi-v1:overall","benchmarkName":"IOI v1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":18.334,"unit":"percent","family":"vals-ai:ioi-v1:overall","index":49.262598721521705,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/ioi-v1","measuredAt":"2026-08-09","stderr":5.918,"costPerTest":3.07299,"latency":3315.166}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":80,"benchmarkCount":11,"categoryCount":6,"confidence":"low","state":"available"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gpt-5-high","providerId":"openai","slug":"gpt-5-high","name":"GPT-5 (high)","provider":"OpenAI","description":"GPT-5 (high) is a reasoning model from OpenAI in the GPT-5 family. 13 benchmarks count toward its index, in 7 categories.","logoUrl":null,"releaseDate":"2025-08-07","contextWindowTokens":400000,"modalities":["text","image","file"],"openWeights":false,"details":{"apiId":"openai/gpt-5","maxOutputTokens":128000,"knowledgeCutoff":"2024-09-30","retiresAt":"2026-12-11","tools":true,"structuredOutput":true,"cacheReadPerMillion":0.125,"weightsUrl":null},"reasoning":true,"overallScore":53.56436514125734,"overallScoreRange":{"low":48.579614054776364,"high":58.54911622773832},"badges":[],"capabilityScores":{"agentic":49.749584296650376,"coding":59.53987447713051,"reasoning":54.416598738962946,"multimodal":51.71760509967773,"knowledge":56.05267604295728,"multilingual":null,"instruction":57.33635533610157,"math":49.16854957190736},"capabilityCounts":{"math":1,"knowledge":4,"agentic":3,"coding":1,"reasoning":2,"multimodal":1,"instruction":1},"benchmarks":[{"benchmarkId":"aaMath500","benchmarkName":"Artificial Analysis MATH-500","category":"math","score":99.4,"unit":"percent","family":"math-500","index":49.16854957190736,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/math-500","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaAime2025","benchmarkName":"Artificial Analysis AIME 2025","category":"math","score":94.3,"unit":"percent","family":"aaAime2025","index":null,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/aime-2025","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaGlobalMmluLite","benchmarkName":"Artificial Analysis Global-MMLU-Lite","category":"multilingual","score":90.7,"unit":"percent","family":"aaGlobalMmluLite","index":null,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/global-mmlu-lite","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaMmluPro","benchmarkName":"Artificial Analysis MMLU-Pro","category":"knowledge","score":87.1,"unit":"percent","family":"mmlu-pro","index":54.21630282494343,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmlu-pro","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":85.4,"unit":"percent","family":"gpqa-diamond","index":55.53564717599917,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"tau2Bench","benchmarkName":"τ²-Bench Tool-Agent-User Evaluation","category":"agentic","score":84.8,"unit":"percent","family":"tau2-bench","index":59.06044003629105,"aboveFit":false,"run":null,"source":"Victor Barres, Honghua Dong, Soham Ray, Xujie Si, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2506.07982","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaLiveCodeBench","benchmarkName":"Artificial Analysis LiveCodeBench","category":"coding","score":84.6,"unit":"percent","family":"livecodebench","index":59.53987447713051,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/livecodebench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":78.2,"unit":"percent","family":"lcr","index":60.949546559969335,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaMmmuPro","benchmarkName":"Artificial Analysis MMMU-Pro","category":"multimodal","score":74.2,"unit":"percent","family":"mmmu-pro","index":51.71760509967773,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmmu-pro","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaIfBench","benchmarkName":"Artificial Analysis IFBench","category":"instruction","score":73.1,"unit":"percent","family":"ifbench-family","index":57.33635533610157,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"omniscienceAccuracy","benchmarkName":"Artificial Analysis Omniscience Accuracy","category":"knowledge","score":40.3,"unit":"percent","family":"omniscienceAccuracy","index":61.47826662422389,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":28.5,"unit":"percent","family":"hle","index":52.980487546662616,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gdpvalAaNormalized","benchmarkName":"GDPval-AA normalized","category":"agentic","score":20.9,"unit":"percent","family":"gdpvalAaNormalized","index":48.97678923655207,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"jobBench","benchmarkName":"JobBench","category":"agentic","score":8.53,"unit":"percent","family":"jobBench","index":41.211523617108,"aboveFit":false,"run":null,"source":"Yuetai Li, Yichen Feng, Zhangchen Xu, Zixian Ma, Kaiyuan Zheng, Fengqing Jiang, Xinghua Sun, Rulin Shao, Zichen Chen, Yue Huang, Xinyang Han, Brian Lee, Kayla Xu, Shenglai Zeng, Hang Hua, Xiangliang Zhang, Basel Alomair, Ranjay Krishna, Luke Zettlemoyer, Pang Wei Koh, Bhaskar Ramasubramanian, Luyao Niu, Xiang Yue, Radha Poovendran","sourceUrl":"https://arxiv.org/abs/2605.26329","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":5.7,"unit":"percent","family":"critpt","index":47.883650917956565,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":1.25,"outputPerMillion":10,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":76,"timeToFirstToken":4.5795,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":{"elo":1406.360510180139,"eloUpper":1410.7879860163412,"eloLower":1401.9330343439365,"votes":31638,"rank":149,"rankUpper":null,"rankLower":null,"arenaName":"GPT-5-high","organization":"openai","updatedAt":"2026-11-09"},"quality":{"coveragePercent":95,"benchmarkCount":13,"categoryCount":7,"confidence":"medium","state":"available"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"mai-thinking-1","providerId":"microsoft","slug":"mai-thinking-1","name":"MAI-Thinking-1","provider":"Microsoft","description":"MAI-Thinking-1 is a reasoning model from Microsoft in the MAI-Thinking family. 10 benchmarks count toward its index, in 5 categories.","logoUrl":null,"releaseDate":"2026-06-02","contextWindowTokens":256000,"modalities":["text"],"openWeights":false,"details":null,"reasoning":true,"overallScore":53.54391829305899,"overallScoreRange":{"low":45.43051531656671,"high":61.65732126955126},"badges":[],"capabilityScores":{"agentic":46.61953130045158,"coding":55.222194361201446,"reasoning":null,"multimodal":null,"knowledge":53.00279707728872,"multilingual":null,"instruction":68.45703165257267,"math":52.59420099687438},"capabilityCounts":{"math":3,"coding":3,"knowledge":2,"instruction":1,"agentic":1},"benchmarks":[{"benchmarkId":"aime2025","benchmarkName":"American Invitational Mathematics Examination 2025","category":"math","score":97,"unit":"percent","family":"aime2025","index":51.6419259443643,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aime2026","benchmarkName":"AIME 2026","category":"math","score":94.5,"unit":"percent","family":"aime2026","index":53.32535562093716,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"graphwalksBfs128k","benchmarkName":"Graphwalks BFS 0K-128K","category":"reasoning","score":90,"unit":"percent","family":"graphwalksBfs128k","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"liveCodeBenchV6","benchmarkName":"LiveCodeBench v6","category":"coding","score":87.7,"unit":"percent","family":"liveCodeBenchV6","index":56.42378290639231,"aboveFit":false,"run":null,"source":"LiveCodeBench maintainers","sourceUrl":"https://github.com/LiveCodeBench/LiveCodeBench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmluPro","benchmarkName":"Massive Multitask Language Understanding Professional","category":"knowledge","score":85,"unit":"percent","family":"mmlu-pro","index":51.57055418848201,"aboveFit":false,"run":null,"source":"Yubo Wang, Xueguang Ma, Ge Zhang, Yuansheng Ni, Abhranil Chandra, Shiguang Guo, Weiming Ren, Aaran Arulraj, Xuan He, Ziyan Jiang, Tianle Li, Max Ku, Kai Wang, Alex Zhuang, Rongqi Fan, Xiang Yue, Wenhu Chen","sourceUrl":"https://arxiv.org/abs/2406.01574","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"ifBench","benchmarkName":"Instruction Following Benchmark","category":"instruction","score":85,"unit":"percent","family":"ifbench-family","index":68.45703165257267,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hmmtFeb2026","benchmarkName":"Harvard-MIT Mathematics Tournament February 2026","category":"math","score":84.9,"unit":"percent","family":"hmmtFeb2026","index":52.815321425321656,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gpqa","benchmarkName":"Graduate-Level Google-Proof Q&A","category":"knowledge","score":84.2,"unit":"percent","family":"gpqa-diamond","index":54.43503996609543,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gpqaDiamond","benchmarkName":"GPQA Diamond","category":"knowledge","score":84.2,"unit":"percent","family":"gpqa-diamond","index":54.43503996609543,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"sweVerified","benchmarkName":"Software Engineering Benchmark Verified","category":"coding","score":73.5,"unit":"percent","family":"swe-bench-verified","index":55.438409496573,"aboveFit":false,"run":null,"source":"Carlos E. Jimenez, John Yang, Alexander Wettig, Shunyu Yao, Kexin Pei, Ofir Press, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2310.06770","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"swePro","benchmarkName":"SWE-bench Pro","category":"coding","score":52.8,"unit":"percent","family":"swePro","index":53.804390680639024,"aboveFit":false,"run":null,"source":"Xiang Deng, Jeff Da, Edwin Pan, Yannis Yiming He, Charles Ide, Kanak Garg, Niklas Lauffer, Andrew Park, Nitin Pasari, Chetan Rane, Karmini Sampath, Maya Krishnan, Srivatsa Kundurthy, Sean Hendryx, Zifan Wang, Chen Bo Calvin Zhang, Noah Jacobson, Bing Liu, Brad Kenstler","sourceUrl":"https://arxiv.org/abs/2509.16941","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"terminalBench2","benchmarkName":"Terminal-Bench 2.0","category":"agentic","score":46,"unit":"percent","family":"terminalBench2","index":46.61953130045158,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":70,"benchmarkCount":10,"categoryCount":5,"confidence":"low","state":"available"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"glm5thinking","providerId":"z-ai","slug":"z-ai-glm-5-thinking","name":"GLM-5 Thinking","provider":"Z.AI","description":"GLM-5 Thinking is a model from Z.AI. 8 benchmarks count toward its index, in 4 categories.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":53.35823613657599,"overallScoreRange":{"low":44.14127781068275,"high":62.57519446246923},"badges":[],"capabilityScores":{"agentic":56.318440770622175,"coding":53.231754978802215,"reasoning":null,"multimodal":null,"knowledge":53.25404107879558,"multilingual":null,"instruction":null,"math":55.565239711967145},"capabilityCounts":{"math":1,"knowledge":2,"coding":4,"agentic":1},"benchmarks":[{"benchmarkId":"vals-ai:medqa:overall","benchmarkName":"MedQA","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":94.267,"unit":"percent","family":"vals-ai:medqa:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medqa","measuredAt":"2026-04-16","stderr":0.212,"costPerTest":0.008007,"latency":81.08},{"benchmarkId":"vals-ai:aime:overall","benchmarkName":"AIME","category":"math","domain":null,"eligible":true,"ranked":true,"score":91.667,"unit":"percent","family":"vals-ai:aime:overall","index":55.565239711967145,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/aime","measuredAt":"2026-04-16","stderr":0.772,"costPerTest":0.069817,"latency":458.694},{"benchmarkId":"vals-ai:mmlu_pro:overall","benchmarkName":"MMLU Pro","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":86.03,"unit":"percent","family":"mmlu-pro","index":52.86823090065119,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","measuredAt":"2026-09-01","stderr":0.343,"costPerTest":0.011542,"latency":86.889},{"benchmarkId":"vals-ai:legal_bench:overall","benchmarkName":"LegalBench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":84.059,"unit":"percent","family":"vals-ai:legal_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","measuredAt":"2026-10-13","stderr":0.411,"costPerTest":0.002311,"latency":16.056},{"benchmarkId":"vals-ai:gpqa:overall","benchmarkName":"GPQA Diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":83.333,"unit":"percent","family":"gpqa-diamond","index":53.63985125693998,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","measuredAt":"2026-09-01","stderr":1.873,"costPerTest":0.034091,"latency":242.523},{"benchmarkId":"vals-ai:lcb:overall","benchmarkName":"LiveCodeBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":81.868,"unit":"percent","family":"livecodebench","index":57.273021439495025,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","measuredAt":"2026-09-01","stderr":1.057,"costPerTest":0.043918,"latency":289.752},{"benchmarkId":"vals-ai:swebench:overall","benchmarkName":"SWE-bench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":71.4,"unit":"percent","family":"swe-bench-verified","index":53.88953858322726,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/swebench","measuredAt":"2026-09-01","stderr":2.023,"costPerTest":0.448911,"latency":513.842},{"benchmarkId":"vals-ai:tax_eval_v2:overall","benchmarkName":"TaxEval v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":70.033,"unit":"percent","family":"vals-ai:tax_eval_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","measuredAt":"2026-09-01","stderr":0.9,"costPerTest":0.021913,"latency":149.298},{"benchmarkId":"vals-ai:corp_fin_v2:overall","benchmarkName":"CorpFin v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":62.898,"unit":"percent","family":"vals-ai:corp_fin_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","measuredAt":"2026-08-12","stderr":0.95,"costPerTest":0.067926,"latency":68.577},{"benchmarkId":"vals-ai:finance_agent:overall","benchmarkName":"Finance Agent v1.1","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":53.182,"unit":"percent","family":"vals-ai:finance_agent:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/finance_agent","measuredAt":"2026-06-04","stderr":2.798,"costPerTest":0.501332,"latency":564.093},{"benchmarkId":"vals-ai:case_law_v2:overall","benchmarkName":"CaseLaw v2","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":52.519,"unit":"percent","family":"vals-ai:case_law_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/case_law_v2","measuredAt":"2026-05-04","stderr":1.725,"costPerTest":0.0344,"latency":117.983},{"benchmarkId":"vals-ai:terminal-bench-2:overall","benchmarkName":"Terminal-Bench 2.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":49.438,"unit":"percent","family":"vals-ai:terminal-bench-2:overall","index":56.318440770622175,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2","measuredAt":"2026-06-04","stderr":5.33,"costPerTest":0.247234,"latency":875.602},{"benchmarkId":"vals-ai:vibe-code:overall","benchmarkName":"Vibe Code Bench v1.1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":23.359,"unit":"percent","family":"vals-ai:vibe-code:overall","index":50.67781678837609,"aboveFit":false,"run":"OpenHands","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","measuredAt":"2026-11-09","stderr":4.034,"costPerTest":40.270258,"latency":13455.794},{"benchmarkId":"vals-ai:ioi-v1:overall","benchmarkName":"IOI v1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":22,"unit":"percent","family":"vals-ai:ioi-v1:overall","index":51.0866431041105,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/ioi-v1","measuredAt":"2026-08-09","stderr":7.832,"costPerTest":3.273674,"latency":6806.938}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":65,"benchmarkCount":8,"categoryCount":4,"confidence":"low","state":"available"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"grok-4-3","providerId":"xai","slug":"grok-4-3","name":"Grok 4.3","provider":"xAI","description":"Grok 4.3 is a reasoning model from xAI. 27 benchmarks count toward its index, in 7 categories.","logoUrl":null,"releaseDate":"2026-04-30","contextWindowTokens":1000000,"modalities":["text","image","file"],"openWeights":false,"details":{"apiId":"x-ai/grok-4.3","maxOutputTokens":900000,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.19999999999999998,"weightsUrl":null},"reasoning":true,"overallScore":53.34814667352096,"overallScoreRange":{"low":49.736228722588876,"high":56.96006462445304},"badges":[],"capabilityScores":{"agentic":50.57343344346424,"coding":53.235734309829404,"reasoning":49.12214953384423,"multimodal":56.79311293364247,"knowledge":57.02734516000497,"multilingual":null,"instruction":59.27831447238402,"math":57.31971574666546},"capabilityCounts":{"agentic":7,"knowledge":5,"coding":8,"math":1,"multimodal":1,"instruction":2,"reasoning":3},"benchmarks":[{"benchmarkId":"tau2Bench","benchmarkName":"τ²-Bench Tool-Agent-User Evaluation","category":"agentic","score":97.7,"unit":"percent","family":"tau2-bench","index":67.62053318007978,"aboveFit":false,"run":null,"source":"Victor Barres, Honghua Dong, Soham Ray, Xujie Si, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2506.07982","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:gpqa:overall","benchmarkName":"GPQA Diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":91.414,"unit":"percent","family":"gpqa-diamond","index":61.05152364296676,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","measuredAt":"2026-09-01","stderr":1.406,"costPerTest":0.025366,"latency":739.636},{"benchmarkId":"gpqa","benchmarkName":"Graduate-Level Google-Proof Q&A","category":"knowledge","score":90.1,"unit":"percent","family":"gpqa-diamond","index":59.84635874812216,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":90.1,"unit":"percent","family":"gpqa-diamond","index":59.84635874812216,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:mmlu_pro:overall","benchmarkName":"MMLU Pro","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":85.838,"unit":"percent","family":"mmlu-pro","index":52.62633388246042,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","measuredAt":"2026-09-01","stderr":0.33,"costPerTest":0.009489,"latency":637.088},{"benchmarkId":"vals-ai:lcb:overall","benchmarkName":"LiveCodeBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":84.494,"unit":"percent","family":"livecodebench","index":59.45192190685622,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","measuredAt":"2026-09-01","stderr":1.024,"costPerTest":0.020464,"latency":3248.962},{"benchmarkId":"vals-ai:legal_bench:overall","benchmarkName":"LegalBench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":84.458,"unit":"percent","family":"vals-ai:legal_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","measuredAt":"2026-10-13","stderr":0.424,"costPerTest":0.003055,"latency":348.006},{"benchmarkId":"livebench:livebench-math:overall","benchmarkName":"LiveBench Mathematics","category":"math","domain":null,"eligible":true,"ranked":true,"score":84.339,"unit":"percent","family":"livebench:livebench-math:overall","index":57.31971574666546,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Mathematics","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.03414854881266491,"latency":null},{"benchmarkId":"vals-ai:mmmu:overall","benchmarkName":"MMMU Pro","category":"multimodal","domain":null,"eligible":true,"ranked":true,"score":83.064,"unit":"percent","family":"mmmu-pro","index":63.253323417694006,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmmu","measuredAt":"2026-09-01","stderr":0.902,"costPerTest":0.02182,"latency":150.948},{"benchmarkId":"ifBench","benchmarkName":"Instruction Following Benchmark","category":"instruction","score":81.3,"unit":"percent","family":"ifbench-family","index":64.9993423777035,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaIfBench","benchmarkName":"Artificial Analysis IFBench","category":"instruction","score":81.3,"unit":"percent","family":"ifbench-family","index":64.9993423777035,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:case_law_v2:overall","benchmarkName":"CaseLaw v2","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":79.314,"unit":"percent","family":"vals-ai:case_law_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/case_law_v2","measuredAt":"2026-05-04","stderr":1.368,"costPerTest":0.040307,"latency":42.813},{"benchmarkId":"mmmuPro","benchmarkName":"Massive Multi-discipline Multimodal Understanding Pro","category":"multimodal","score":78.1,"unit":"percent","family":"mmmu-pro","index":56.79311293364247,"aboveFit":false,"run":null,"source":"MMMU-Pro authors","sourceUrl":"https://arxiv.org/abs/2409.02813","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaMmmuPro","benchmarkName":"Artificial Analysis MMMU-Pro","category":"multimodal","score":78.1,"unit":"percent","family":"mmmu-pro","index":56.79311293364247,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmmu-pro","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:medscribe:overall","benchmarkName":"MedScribe","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":74.399,"unit":"percent","family":"vals-ai:medscribe:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medscribe","measuredAt":"2026-10-29","stderr":2.019,"costPerTest":0.015293,"latency":100.457},{"benchmarkId":"livebench:livebench-language:overall","benchmarkName":"LiveBench Language","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":73.58,"unit":"percent","family":"livebench:livebench-language:overall","index":57.76371324546867,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Language","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.022544666666666668,"latency":null},{"benchmarkId":"vals-ai:swebench:overall","benchmarkName":"SWE-bench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":71.4,"unit":"percent","family":"swe-bench-verified","index":53.88953858322726,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/swebench","measuredAt":"2026-09-01","stderr":2.023,"costPerTest":0.22259,"latency":117.767},{"benchmarkId":"livebench:livebench-reasoning:overall","benchmarkName":"LiveBench Reasoning","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":70.822,"unit":"percent","family":"livebench","index":53.364947810510834,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Reasoning","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.0411509900990099,"latency":null},{"benchmarkId":"vals-ai:tax_eval_v2:overall","benchmarkName":"TaxEval v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":70.81,"unit":"percent","family":"vals-ai:tax_eval_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","measuredAt":"2026-09-01","stderr":0.896,"costPerTest":0.013656,"latency":391.972},{"benchmarkId":"livebench:livebench-coding:overall","benchmarkName":"LiveBench Coding","category":"coding","domain":null,"eligible":true,"ranked":true,"score":69.9325,"unit":"percent","family":"livebench:livebench-coding:overall","index":53.17807805678096,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Coding","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.010757264957264957,"latency":null},{"benchmarkId":"vals-ai:corp_fin_v2:overall","benchmarkName":"CorpFin v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":68.532,"unit":"percent","family":"vals-ai:corp_fin_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","measuredAt":"2026-08-12","stderr":0.915,"costPerTest":0.134161,"latency":36.444},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":64.3,"unit":"percent","family":"lcr","index":51.413027254328654,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-instruction-following:overall","benchmarkName":"LiveBench Instruction Following","category":"instruction","domain":null,"eligible":true,"ranked":true,"score":62.75,"unit":"percent","family":"livebench:livebench-instruction-following:overall","index":53.55728656706454,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=IF","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.026667500000000004,"latency":null},{"benchmarkId":"livebench:livebench-data-analysis:overall","benchmarkName":"LiveBench Data Analysis","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":55.77266666666666,"unit":"percent","family":"livebench","index":33.86952979116924,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Data%20Analysis","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.035357999999999994,"latency":null},{"benchmarkId":"vals-ai:finance_agent:overall","benchmarkName":"Finance Agent v1.1","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":53.812,"unit":"percent","family":"vals-ai:finance_agent:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/finance_agent","measuredAt":"2026-06-04","stderr":2.828,"costPerTest":0.442142,"latency":795.917},{"benchmarkId":"vals-ai:public-benefits-bench:overall","benchmarkName":"Public Benefits Bench v1.1","category":null,"domain":"Public benefits","eligible":false,"ranked":true,"score":51.691,"unit":"percent","family":"vals-ai:public-benefits-bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/public-benefits-bench","measuredAt":"2026-10-12","stderr":1.3,"costPerTest":0.264337,"latency":205.01},{"benchmarkId":"vals-ai:public-benefits-bench-v1:overall","benchmarkName":"Public Benefits Bench v1","category":null,"domain":"Public benefits","eligible":false,"ranked":true,"score":50.068,"unit":"percent","family":"vals-ai:public-benefits-bench-v1:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/public-benefits-bench-v1","measuredAt":"2026-06-09","stderr":1.301,"costPerTest":1.207438,"latency":418.342},{"benchmarkId":"aaSciCode","benchmarkName":"Artificial Analysis SciCode","category":"coding","score":48.3,"unit":"percent","family":"scicode-family","index":58.80808081434287,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/scicode","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:mortgage_tax:overall","benchmarkName":"MortgageTax","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":48.252,"unit":"percent","family":"vals-ai:mortgage_tax:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mortgage_tax","measuredAt":"2026-09-01","stderr":0.973,"costPerTest":0.012184,"latency":443.266},{"benchmarkId":"sciCode","benchmarkName":"Scientific Code Benchmark","category":"coding","score":47.3,"unit":"percent","family":"scicode-family","index":57.50438966792814,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gertLabs","benchmarkName":"Gert Labs Composite Game Benchmark","category":"agentic","score":43.86,"unit":"percent","family":"gertLabs","index":51.06093514602182,"aboveFit":false,"run":null,"source":"Gert Labs","sourceUrl":"https://gertlabs.com/rankings","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:terminal-bench-2:overall","benchmarkName":"Terminal-Bench 2.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":43.446,"unit":"percent","family":"vals-ai:terminal-bench-2:overall","index":52.71059012701214,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2","measuredAt":"2026-06-04","stderr":4.406,"costPerTest":0.373255,"latency":654.058},{"benchmarkId":"vals-ai:terminal-bench-2-1:overall","benchmarkName":"Terminal-Bench 2.1","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":41.948,"unit":"percent","family":"terminal-bench-2-1","index":45.67463403626749,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2-1","measuredAt":"2026-10-01","stderr":2.622,"costPerTest":0.982203,"latency":470.713},{"benchmarkId":"vals-ai:skillsbench:overall","benchmarkName":"SkillsBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":40.638,"unit":"percent","family":"vals-ai:skillsbench:overall","index":55.39082959700369,"aboveFit":false,"run":"OpenHands","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/skillsbench","measuredAt":"2026-10-29","stderr":4.615,"costPerTest":0.419712,"latency":277.173},{"benchmarkId":"vals-ai:medcode:overall","benchmarkName":"MedCode","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":38.068,"unit":"percent","family":"vals-ai:medcode:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medcode","measuredAt":"2026-10-29","stderr":2.081,"costPerTest":0.022202,"latency":43.924},{"benchmarkId":"vals-ai:fabv2:overall","benchmarkName":"Finance Agent (v2)","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":37.728,"unit":"percent","family":"vals-ai:fabv2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/fabv2","measuredAt":"2026-10-29","stderr":0.373,"costPerTest":0.848345,"latency":615.499},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":37.2,"unit":"percent","family":"hle","index":61.709134428421834,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hle","benchmarkName":"Humanity's Last Exam","category":"knowledge","score":35,"unit":"percent","family":"hle","index":59.50189038935628,"aboveFit":false,"run":null,"source":"Center for AI Safety, Scale AI, and thousands of expert contributors","sourceUrl":"https://lastexam.ai/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"omniscienceAccuracy","benchmarkName":"Artificial Analysis Omniscience Accuracy","category":"knowledge","score":34.6,"unit":"percent","family":"omniscienceAccuracy","index":54.29480751508454,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gdpvalAaNormalized","benchmarkName":"GDPval-AA normalized","category":"agentic","score":29.2,"unit":"percent","family":"gdpvalAaNormalized","index":54.91109635859428,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:vibe-code:overall","benchmarkName":"Vibe Code Bench v1.1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":19.403,"unit":"percent","family":"vals-ai:vibe-code:overall","index":49.250429834954616,"aboveFit":false,"run":"OpenHands","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","measuredAt":"2026-11-09","stderr":4.243,"costPerTest":1.28472,"latency":589.407},{"benchmarkId":"livebench:livebench-agentic-coding:overall","benchmarkName":"LiveBench Agentic Coding","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":18.535333333333334,"unit":"percent","family":"livebench:livebench-agentic-coding:overall","index":28.277863941417447,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Agentic%20Coding","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.16674305555555557,"latency":null},{"benchmarkId":"vals-ai:emb:overall","benchmarkName":"EMB","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":18.394,"unit":"percent","family":"vals-ai:emb:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/emb","measuredAt":"2026-10-20","stderr":1.667,"costPerTest":0.634124,"latency":271.547},{"benchmarkId":"apexAgentsAa","benchmarkName":"APEX-Agents-AA","category":"agentic","score":17,"unit":"percent","family":"apexAgentsAa","index":53.75838131485673,"aboveFit":false,"run":null,"source":"Artificial Analysis / Mercor","sourceUrl":"https://artificialanalysis.ai/evaluations/apex-agents-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:legal_research:overall","benchmarkName":"Legal Research Bench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":15.385,"unit":"percent","family":"vals-ai:legal_research:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_research","measuredAt":"2026-11-09","stderr":2.508,"costPerTest":0.203143,"latency":84.583},{"benchmarkId":"vals-ai:ioi-v1:overall","benchmarkName":"IOI v1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":15.334,"unit":"percent","family":"vals-ai:ioi-v1:overall","index":47.76992737734103,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/ioi-v1","measuredAt":"2026-08-09","stderr":6.313,"costPerTest":0.351754,"latency":628.816},{"benchmarkId":"researchClawBench","benchmarkName":"ResearchClawBench","category":"agentic","score":12.4,"unit":"percent","family":"researchClawBench","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":8,"unit":"percent","family":"critpt","index":52.33618254636401,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:code-migration:overall","benchmarkName":"Code Migration","category":"coding","domain":null,"eligible":true,"ranked":true,"score":6.795,"unit":"percent","family":"vals-ai:code-migration:overall","index":48.79891388133595,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/code-migration","measuredAt":"2026-10-29","stderr":1.195,"costPerTest":0.416921,"latency":180.478},{"benchmarkId":"vals-ai:tax_agent_bench:overall","benchmarkName":"Tax Agent Bench","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":6.179,"unit":"percent","family":"vals-ai:tax_agent_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench","measuredAt":"2026-11-09","stderr":1.732,"costPerTest":0.116616,"latency":97.519},{"benchmarkId":"vals-ai:programbench:overall","benchmarkName":"ProgramBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":0,"unit":"percent","family":"vals-ai:programbench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/programbench","measuredAt":"2026-11-09","stderr":0,"costPerTest":0.355823,"latency":236.134}],"price":{"inputPerMillion":1.25,"outputPerMillion":2.5,"batch":{"inputPerMillion":1,"outputPerMillion":2},"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":61,"timeToFirstToken":1.346,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":{"elo":1396.9857922491929,"eloUpper":1400.6111057929133,"eloLower":1393.3604787054724,"votes":70242,"rank":162,"rankUpper":null,"rankLower":null,"arenaName":"Grok 4.3","organization":"xai","updatedAt":"2026-11-09"},"quality":{"coveragePercent":95,"benchmarkCount":27,"categoryCount":7,"confidence":"medium","state":"available"},"hosts":[{"host":"amazon-bedrock","name":"Amazon Bedrock","kind":"cloud","url":"https://docs.aws.amazon.com/bedrock/latest/userguide/models-supported.html","inputPerMillion":1.25,"outputPerMillion":2.5,"cacheReadPerMillion":0.2,"contextTokens":1000000,"maxOutputTokens":131072,"deprecated":false},{"host":"google-vertex","name":"Google Vertex AI","kind":"cloud","url":"https://cloud.google.com/vertex-ai/generative-ai/docs/models","inputPerMillion":1.25,"outputPerMillion":2.5,"cacheReadPerMillion":0.2,"contextTokens":200000,"maxOutputTokens":30000,"deprecated":false},{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":1.25,"outputPerMillion":2.5,"cacheReadPerMillion":0.2,"contextTokens":1000000,"maxOutputTokens":900000,"deprecated":false},{"host":"oracle-cloud","name":"Oracle Cloud","kind":"cloud","url":"https://docs.oracle.com/en-us/iaas/Content/generative-ai/pretrained-models.htm","inputPerMillion":1.25,"outputPerMillion":2.5,"cacheReadPerMillion":0.2,"contextTokens":1000000,"maxOutputTokens":30000,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:livebench","name":"LiveBench","url":"https://livebench.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"solar-open-2","providerId":"upstage","slug":"solar-open-2","name":"Solar Open 2","provider":"Upstage","description":"Solar Open 2 is a reasoning model from Upstage in the Solar Open family. 12 benchmarks count toward its index, in 6 categories.","logoUrl":null,"releaseDate":"2026-07-22","contextWindowTokens":1000000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":true,"overallScore":53.300528899497365,"overallScoreRange":{"low":47.13265197309924,"high":59.46840582589549},"badges":[],"capabilityScores":{"agentic":47.91012888305673,"coding":56.71736625149423,"reasoning":50.040866203157336,"multimodal":null,"knowledge":54.2416628352576,"multilingual":null,"instruction":63.78447857842515,"math":56.65149109419427},"capabilityCounts":{"math":2,"coding":2,"knowledge":3,"instruction":1,"reasoning":1,"agentic":3},"benchmarks":[{"benchmarkId":"aime2026","benchmarkName":"AIME 2026","category":"math","score":95.7,"unit":"percent","family":"aime2026","index":54.189284239958425,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hmmtFeb2026","benchmarkName":"Harvard-MIT Mathematics Tournament February 2026","category":"math","score":93.9,"unit":"percent","family":"hmmtFeb2026","index":59.11369794843011,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"liveCodeBenchV6","benchmarkName":"LiveCodeBench v6","category":"coding","score":92.4,"unit":"percent","family":"liveCodeBenchV6","index":60.282751497544886,"aboveFit":false,"run":null,"source":"LiveCodeBench maintainers","sourceUrl":"https://github.com/LiveCodeBench/LiveCodeBench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gpqaDiamond","benchmarkName":"GPQA Diamond","category":"knowledge","score":86.3,"unit":"percent","family":"gpqa-diamond","index":56.36110258342698,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmluPro","benchmarkName":"Massive Multitask Language Understanding Professional","category":"knowledge","score":86.2,"unit":"percent","family":"mmlu-pro","index":53.08241055217426,"aboveFit":false,"run":null,"source":"Yubo Wang, Xueguang Ma, Ge Zhang, Yuansheng Ni, Abhranil Chandra, Shiguang Guo, Weiming Ren, Aaran Arulraj, Xuan He, Ziyan Jiang, Tianle Li, Max Ku, Kai Wang, Alex Zhuang, Rongqi Fan, Xiang Yue, Wenhu Chen","sourceUrl":"https://arxiv.org/abs/2406.01574","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"ifBench","benchmarkName":"Instruction Following Benchmark","category":"instruction","score":80,"unit":"percent","family":"ifbench-family","index":63.78447857842515,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"sweVerified","benchmarkName":"Software Engineering Benchmark Verified","category":"coding","score":70.4,"unit":"percent","family":"swe-bench-verified","index":53.15198100544356,"aboveFit":false,"run":null,"source":"Carlos E. Jimenez, John Yang, Alexander Wettig, Shunyu Yao, Kexin Pei, Ofir Press, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2310.06770","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":62.3,"unit":"percent","family":"lcr","index":50.040866203157336,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mcpAtlas","benchmarkName":"MCP Atlas","category":"agentic","score":58.2,"unit":"percent","family":"mcpAtlas","index":50.2539070029867,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hleNoTools","benchmarkName":"Humanity's Last Exam without tools","category":"knowledge","score":28.8,"unit":"percent","family":"hle","index":53.281475370171556,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"terminalBenchHard","benchmarkName":"Terminal-Bench Hard","category":"agentic","score":28.3,"unit":"percent","family":"terminalBenchHard","index":42.29700093311053,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/terminalbench-hard","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"apexAgents","benchmarkName":"APEX-Agents","category":"agentic","score":16.6,"unit":"percent","family":"apexAgents","index":51.179478713072946,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":85,"benchmarkCount":12,"categoryCount":6,"confidence":"low","state":"available"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"muse-glimmer-30b","providerId":"meta","slug":"muse-glimmer-30b","name":"Muse Glimmer 30B","provider":"Meta","description":"Muse Glimmer 30B is a reasoning model from Meta in the Muse Glimmer family. 18 benchmarks count toward its index, in 7 categories.","logoUrl":null,"releaseDate":"2026-08-10","contextWindowTokens":131072,"modalities":["text","image"],"openWeights":true,"details":{"apiId":"meta/muse-glimmer-30b","maxOutputTokens":16384,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.04,"weightsUrl":"https://huggingface.co/meta-models/Muse-Glimmer-30B"},"reasoning":true,"overallScore":53.03565744222922,"overallScoreRange":{"low":48.72435189163373,"high":57.346962992824714},"badges":[],"capabilityScores":{"agentic":53.25416684063597,"coding":54.41741660771123,"reasoning":53.165484938323104,"multimodal":54.011091174220866,"knowledge":48.74307393930405,"multilingual":null,"instruction":60.980946733936634,"math":53.469343724107375},"capabilityCounts":{"math":1,"knowledge":3,"reasoning":2,"multimodal":3,"instruction":1,"coding":3,"agentic":5},"benchmarks":[{"benchmarkId":"aime2026","benchmarkName":"AIME 2026","category":"math","score":94.7,"unit":"percent","family":"aime2026","index":53.469343724107375,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":83.5,"unit":"percent","family":"gpqa-diamond","index":53.79301909365158,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":83.3,"unit":"percent","family":"lcr","index":64.4485572404562,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"charxiv","benchmarkName":"CharXiv Reasoning","category":"multimodal","score":78.8,"unit":"percent","family":"charxiv","index":53.897876966052195,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"ifBench","benchmarkName":"Instruction Following Benchmark","category":"instruction","score":77,"unit":"percent","family":"ifbench-family","index":60.980946733936634,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"sweVerified","benchmarkName":"Software Engineering Benchmark Verified","category":"coding","score":76,"unit":"percent","family":"swe-bench-verified","index":57.282303441032234,"aboveFit":false,"run":null,"source":"Carlos E. Jimenez, John Yang, Alexander Wettig, Shunyu Yao, Kexin Pei, Ofir Press, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2310.06770","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"omniDocBench15","benchmarkName":"OmniDocBench 1.5","category":"multimodal","score":75.8,"unit":"percent","family":"omniDocBench15","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mcpAtlas","benchmarkName":"MCP Atlas","category":"agentic","score":75.5,"unit":"percent","family":"mcpAtlas","index":61.90365081502853,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"screenSpotPro","benchmarkName":"ScreenSpot Pro","category":"multimodal","score":75.4,"unit":"percent","family":"screenSpotPro","index":56.48286207018863,"aboveFit":false,"run":null,"source":"Kaixin Li, Ziyang Meng, Hongzhan Lin, Ziyang Luo, Yuchen Tian, Jing Ma, Zhiyong Huang, Tat-Seng Chua","sourceUrl":"https://arxiv.org/abs/2504.07981","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"deepSearchQa","benchmarkName":"DeepSearchQA","category":"agentic","score":74.6,"unit":"percent","family":"deepSearchQa","index":55.123343962373674,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaMmmuPro","benchmarkName":"Artificial Analysis MMMU-Pro","category":"multimodal","score":74.3,"unit":"percent","family":"mmmu-pro","index":51.84774632618964,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmmu-pro","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmmuPro","benchmarkName":"Massive Multi-discipline Multimodal Understanding Pro","category":"multimodal","score":74,"unit":"percent","family":"mmmu-pro","index":51.4573226466539,"aboveFit":false,"run":null,"source":"MMMU-Pro authors","sourceUrl":"https://arxiv.org/abs/2409.02813","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"osWorldVerified","benchmarkName":"OSWorld-Verified","category":"agentic","score":65.9,"unit":"percent","family":"osWorldVerified","index":53.67925094195287,"aboveFit":false,"run":null,"source":"Tianbao Xie, Danyang Zhang, Jixuan Chen, Xiaochuan Li, Siheng Zhao, Ruisheng Cao, Toh Jing Hua, Zhoujun Cheng, Dongchan Shin, Fangyu Lei, Yitao Liu, Yiheng Xu, Shuyan Zhou, Silvio Savarese, Caiming Xiong, Victor Zhong, Tao Yu","sourceUrl":"https://os-world.github.io/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"terminalBench21","benchmarkName":"Terminal-Bench 2.1","category":"agentic","score":51.7,"unit":"percent","family":"terminal-bench-2-1","index":51.378143550607966,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"swePro","benchmarkName":"SWE-bench Pro","category":"coding","score":51.2,"unit":"percent","family":"swePro","index":52.44181471073826,"aboveFit":false,"run":null,"source":"Xiang Deng, Jeff Da, Edwin Pan, Yannis Yiming He, Charles Ide, Kanak Garg, Niklas Lauffer, Andrew Park, Nitin Pasari, Chetan Rane, Karmini Sampath, Maya Krishnan, Srivatsa Kundurthy, Sean Hendryx, Zifan Wang, Chen Bo Calvin Zhang, Noah Jacobson, Bing Liu, Brad Kenstler","sourceUrl":"https://arxiv.org/abs/2509.16941","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaSciCode","benchmarkName":"Artificial Analysis SciCode","category":"coding","score":44.9,"unit":"percent","family":"scicode-family","index":54.37553091653278,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/scicode","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"sciCode","benchmarkName":"Scientific Code Benchmark","category":"coding","score":43.6,"unit":"percent","family":"scicode-family","index":52.680732426193636,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"omniscienceAccuracy","benchmarkName":"Artificial Analysis Omniscience Accuracy","category":"knowledge","score":28,"unit":"percent","family":"omniscienceAccuracy","index":45.9771180202916,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":22,"unit":"percent","family":"hle","index":46.459084703968955,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gdpvalAaNormalized","benchmarkName":"GDPval-AA normalized","category":"agentic","score":14.2,"unit":"percent","family":"gdpvalAaNormalized","index":44.1864449332168,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":2.6,"unit":"percent","family":"critpt","index":41.88241263619001,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":0.3,"outputPerMillion":1.2,"batch":{"inputPerMillion":0.24,"outputPerMillion":0.96},"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":90,"timeToFirstToken":0.778,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":95,"benchmarkCount":18,"categoryCount":7,"confidence":"medium","state":"available"},"hosts":[{"host":"vultr","name":"Vultr","kind":"cloud","url":"https://api.vultrinference.com/","inputPerMillion":0.25,"outputPerMillion":1,"cacheReadPerMillion":null,"contextTokens":131072,"maxOutputTokens":131072,"deprecated":false},{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":0.3,"outputPerMillion":1.2,"cacheReadPerMillion":0.04,"contextTokens":131072,"maxOutputTokens":16384,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gpt-5-medium","providerId":"openai","slug":"gpt-5-medium","name":"GPT-5 (medium)","provider":"OpenAI","description":"GPT-5 (medium) is a reasoning model from OpenAI in the GPT-5 family. 8 benchmarks count toward its index, in 5 categories.","logoUrl":null,"releaseDate":"2025-08-07","contextWindowTokens":400000,"modalities":["text","image","file"],"openWeights":false,"details":{"apiId":"openai/gpt-5","maxOutputTokens":128000,"knowledgeCutoff":"2024-09-30","retiresAt":"2026-12-11","tools":true,"structuredOutput":true,"cacheReadPerMillion":0.125,"weightsUrl":null},"reasoning":true,"overallScore":53.0336058410918,"overallScoreRange":{"low":43.81664751519856,"high":62.250564166985036},"badges":[],"capabilityScores":{"agentic":60.18851432655778,"coding":null,"reasoning":48.14464270822689,"multimodal":51.84774632618964,"knowledge":54.92512728004732,"multilingual":null,"instruction":55.00007879902781,"math":null},"capabilityCounts":{"agentic":1,"knowledge":3,"reasoning":2,"multimodal":1,"instruction":1},"benchmarks":[{"benchmarkId":"tau2Bench","benchmarkName":"τ²-Bench Tool-Agent-User Evaluation","category":"agentic","score":86.5,"unit":"percent","family":"tau2-bench","index":60.18851432655778,"aboveFit":false,"run":null,"source":"Victor Barres, Honghua Dong, Soham Ray, Xujie Si, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2506.07982","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":84.2,"unit":"percent","family":"gpqa-diamond","index":54.43503996609543,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":76,"unit":"percent","family":"lcr","index":59.44016940368088,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaMmmuPro","benchmarkName":"Artificial Analysis MMMU-Pro","category":"multimodal","score":74.3,"unit":"percent","family":"mmmu-pro","index":51.84774632618964,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmmu-pro","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaIfBench","benchmarkName":"Artificial Analysis IFBench","category":"instruction","score":70.6,"unit":"percent","family":"ifbench-family","index":55.00007879902781,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"omniscienceAccuracy","benchmarkName":"Artificial Analysis Omniscience Accuracy","category":"knowledge","score":39.5,"unit":"percent","family":"omniscienceAccuracy","index":60.47006183697627,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":25.4,"unit":"percent","family":"hle","index":49.87028003707025,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":0,"unit":"percent","family":"critpt","index":36.8491160127729,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":1.25,"outputPerMillion":10,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":76,"timeToFirstToken":4.5795,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":65,"benchmarkCount":8,"categoryCount":5,"confidence":"low","state":"available"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"kimi-k2-5-reasoning","providerId":"moonshot-ai","slug":"kimi-k2-5-reasoning","name":"Kimi K2.5 (Reasoning)","provider":"Moonshot AI","description":"Kimi K2.5 (Reasoning) is a reasoning model from Moonshot AI in the Kimi K2.5 family. 21 benchmarks count toward its index, in 7 categories.","logoUrl":null,"releaseDate":"2026-01-27","contextWindowTokens":128000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":true,"overallScore":52.998739761754265,"overallScoreRange":{"low":48.970156476638266,"high":57.027323046870265},"badges":[],"capabilityScores":{"agentic":50.559871531942534,"coding":56.039639056303194,"reasoning":52.111491127071744,"multimodal":53.85715892078539,"knowledge":55.53649961956313,"multilingual":null,"instruction":54.62627455309602,"math":52.214542853935896},"capabilityCounts":{"math":2,"agentic":7,"knowledge":4,"coding":2,"multimodal":2,"reasoning":3,"instruction":1},"benchmarks":[{"benchmarkId":"aime2025","benchmarkName":"American Invitational Mathematics Examination 2025","category":"math","score":96.1,"unit":"percent","family":"aime2025","index":51.1535627096588,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"tau2Bench","benchmarkName":"τ²-Bench Tool-Agent-User Evaluation","category":"agentic","score":95.9,"unit":"percent","family":"tau2-bench","index":66.42610157862087,"aboveFit":false,"run":null,"source":"Victor Barres, Honghua Dong, Soham Ray, Xujie Si, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2506.07982","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hmmtFeb2025","benchmarkName":"Harvard-MIT Mathematics Tournament February 2025","category":"math","score":95.4,"unit":"percent","family":"hmmtFeb2025","index":53.27552299821299,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":87.9,"unit":"percent","family":"gpqa-diamond","index":57.82857886329864,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gpqa","benchmarkName":"Graduate-Level Google-Proof Q&A","category":"knowledge","score":87.6,"unit":"percent","family":"gpqa-diamond","index":57.55342706082269,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmluPro","benchmarkName":"Massive Multitask Language Understanding Professional","category":"knowledge","score":87.1,"unit":"percent","family":"mmlu-pro","index":54.21630282494343,"aboveFit":false,"run":null,"source":"Yubo Wang, Xueguang Ma, Ge Zhang, Yuansheng Ni, Abhranil Chandra, Shiguang Guo, Weiming Ren, Aaran Arulraj, Xuan He, Ziyan Jiang, Tianle Li, Max Ku, Kai Wang, Alex Zhuang, Rongqi Fan, Xiang Yue, Wenhu Chen","sourceUrl":"https://arxiv.org/abs/2406.01574","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"liveCodeBenchV6","benchmarkName":"LiveCodeBench v6","category":"coding","score":85,"unit":"percent","family":"liveCodeBenchV6","index":54.20692860934721,"aboveFit":false,"run":null,"source":"LiveCodeBench maintainers","sourceUrl":"https://github.com/LiveCodeBench/LiveCodeBench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmmuPro","benchmarkName":"Massive Multi-discipline Multimodal Understanding Pro","category":"multimodal","score":78.5,"unit":"percent","family":"mmmu-pro","index":57.31367783969014,"aboveFit":false,"run":null,"source":"MMMU-Pro authors","sourceUrl":"https://arxiv.org/abs/2409.02813","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":78,"unit":"percent","family":"lcr","index":60.8123304548522,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"charxiv","benchmarkName":"CharXiv Reasoning","category":"multimodal","score":77.5,"unit":"percent","family":"charxiv","index":52.41782901281534,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"sweVerified","benchmarkName":"Software Engineering Benchmark Verified","category":"coding","score":76.8,"unit":"percent","family":"swe-bench-verified","index":57.87234950325919,"aboveFit":false,"run":null,"source":"Carlos E. Jimenez, John Yang, Alexander Wettig, Shunyu Yao, Kexin Pei, Ofir Press, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2310.06770","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaMmmuPro","benchmarkName":"Artificial Analysis MMMU-Pro","category":"multimodal","score":75.4,"unit":"percent","family":"mmmu-pro","index":53.279299817820736,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmmu-pro","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaIfBench","benchmarkName":"Artificial Analysis IFBench","category":"instruction","score":70.2,"unit":"percent","family":"ifbench-family","index":54.62627455309602,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"osWorldVerified","benchmarkName":"OSWorld-Verified","category":"agentic","score":63.3,"unit":"percent","family":"osWorldVerified","index":51.58038258679388,"aboveFit":false,"run":null,"source":"Tianbao Xie, Danyang Zhang, Jixuan Chen, Xiaochuan Li, Siheng Zhao, Ruisheng Cao, Toh Jing Hua, Zhoujun Cheng, Dongchan Shin, Fangyu Lei, Yitao Liu, Yiheng Xu, Shuyan Zhou, Silvio Savarese, Caiming Xiong, Victor Zhong, Tao Yu","sourceUrl":"https://os-world.github.io/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"longBenchV2","benchmarkName":"LongBench v2","category":"reasoning","score":61,"unit":"percent","family":"longBenchV2","index":52.67178863182358,"aboveFit":false,"run":null,"source":"LongBench v2 authors","sourceUrl":"https://arxiv.org/abs/2412.15204","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"browseComp","benchmarkName":"BrowseComp","category":"agentic","score":60.6,"unit":"percent","family":"browseComp","index":47.784194294156336,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"terminalBench2","benchmarkName":"Terminal-Bench 2.0","category":"agentic","score":50.8,"unit":"percent","family":"terminalBench2","index":50.03398970223154,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"omniscienceAccuracy","benchmarkName":"Artificial Analysis Omniscience Accuracy","category":"knowledge","score":35.2,"unit":"percent","family":"omniscienceAccuracy","index":55.050961105520265,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gertLabs","benchmarkName":"Gert Labs Composite Game Benchmark","category":"agentic","score":32.58,"unit":"percent","family":"gertLabs","index":41.92732979484249,"aboveFit":false,"run":null,"source":"Gert Labs","sourceUrl":"https://gertlabs.com/rankings","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":30.7,"unit":"percent","family":"hle","index":55.18773158572816,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gdpvalAaNormalized","benchmarkName":"GDPval-AA normalized","category":"agentic","score":17,"unit":"percent","family":"gdpvalAaNormalized","index":46.18837986595393,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"apexAgentsAa","benchmarkName":"APEX-Agents-AA","category":"agentic","score":11.5,"unit":"percent","family":"apexAgentsAa","index":49.9787229009987,"aboveFit":false,"run":null,"source":"Artificial Analysis / Mercor","sourceUrl":"https://artificialanalysis.ai/evaluations/apex-agents-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":3.1,"unit":"percent","family":"critpt","index":42.85035429453946,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":0.6,"outputPerMillion":3,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":95,"benchmarkCount":21,"categoryCount":7,"confidence":"medium","state":"available"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gpt-5-1-codex","providerId":"openai","slug":"gpt-5-1-codex","name":"GPT-5.1-Codex","provider":"OpenAI","description":"GPT-5.1-Codex is a reasoning model from OpenAI. 15 benchmarks count toward its index, in 6 categories.","logoUrl":null,"releaseDate":"2025-11-13","contextWindowTokens":400000,"modalities":["text","image"],"openWeights":false,"details":{"apiId":"openai/gpt-5.1-codex","maxOutputTokens":128000,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.13,"weightsUrl":null},"reasoning":true,"overallScore":52.91288290550544,"overallScoreRange":{"low":47.237806858293844,"high":58.58795895271704},"badges":[],"capabilityScores":{"agentic":56.11349109694736,"coding":49.795226978412835,"reasoning":51.36354040010676,"multimodal":49.505204248975154,"knowledge":55.743794290710106,"multilingual":null,"instruction":54.43937243013011,"math":null},"capabilityCounts":{"knowledge":3,"coding":4,"agentic":4,"multimodal":1,"instruction":1,"reasoning":2},"benchmarks":[{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":86,"unit":"percent","family":"gpqa-diamond","index":56.085950780951045,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:lcb:overall","benchmarkName":"LiveCodeBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":85.55,"unit":"percent","family":"livecodebench","index":60.32812864468311,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","measuredAt":"2026-09-01","stderr":1.003,"costPerTest":0.137502,"latency":233.557},{"benchmarkId":"tau2Bench","benchmarkName":"τ²-Bench Tool-Agent-User Evaluation","category":"agentic","score":83,"unit":"percent","family":"tau2-bench","index":57.86600843483216,"aboveFit":false,"run":null,"source":"Victor Barres, Honghua Dong, Soham Ray, Xujie Si, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2506.07982","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaMmmuPro","benchmarkName":"Artificial Analysis MMMU-Pro","category":"multimodal","score":72.5,"unit":"percent","family":"mmmu-pro","index":49.505204248975154,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmmu-pro","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaIfBench","benchmarkName":"Artificial Analysis IFBench","category":"instruction","score":70,"unit":"percent","family":"ifbench-family","index":54.43937243013011,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":69.3,"unit":"percent","family":"lcr","index":54.843429882256956,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"swebench-experiments:swe-bench-verified:overall","benchmarkName":"SWE-bench Verified","category":"coding","domain":null,"eligible":true,"ranked":true,"score":66,"unit":"percent","family":"swe-bench-verified","index":49.90672766319531,"aboveFit":false,"run":"medium effort · mini-SWE-agent","source":"SWE-bench team","sourceUrl":"https://github.com/SWE-bench/experiments/tree/main/evaluation/verified","measuredAt":"2026-09-01","stderr":null,"costPerTest":0.5888823500000001,"latency":null},{"benchmarkId":"vals-ai:terminal-bench:overall","benchmarkName":"Terminal-Bench 1.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":57.5,"unit":"percent","family":"vals-ai:terminal-bench:overall","index":59.82968138441703,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench","measuredAt":"2026-01-12","stderr":5.527,"costPerTest":0.121214,"latency":946.453},{"benchmarkId":"gertLabs","benchmarkName":"Gert Labs Composite Game Benchmark","category":"agentic","score":49.68,"unit":"percent","family":"gertLabs","index":55.77348684317286,"aboveFit":false,"run":null,"source":"Gert Labs","sourceUrl":"https://gertlabs.com/rankings","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"omniscienceAccuracy","benchmarkName":"Artificial Analysis Omniscience Accuracy","category":"knowledge","score":39.9,"unit":"percent","family":"omniscienceAccuracy","index":60.97416423060008,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"jobBench","benchmarkName":"JobBench","category":"agentic","score":26.2,"unit":"percent","family":"jobBench","index":50.98478772536739,"aboveFit":false,"run":null,"source":"Yuetai Li, Yichen Feng, Zhangchen Xu, Zixian Ma, Kaiyuan Zheng, Fengqing Jiang, Xinghua Sun, Rulin Shao, Zichen Chen, Yue Huang, Xinyang Han, Brian Lee, Kayla Xu, Shenglai Zeng, Hang Hua, Xiangliang Zhang, Basel Alomair, Ranjay Krishna, Luke Zettlemoyer, Pang Wei Koh, Bhaskar Ramasubramanian, Luyao Niu, Xiang Yue, Radha Poovendran","sourceUrl":"https://arxiv.org/abs/2605.26329","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":25.7,"unit":"percent","family":"hle","index":50.17126786057919,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:vibe-code:overall","benchmarkName":"Vibe Code Bench v1.1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":13.115,"unit":"percent","family":"vals-ai:vibe-code:overall","index":46.98162064306528,"aboveFit":false,"run":"OpenHands","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","measuredAt":"2026-11-09","stderr":3.161,"costPerTest":3.797914,"latency":3026.069},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":5.7,"unit":"percent","family":"critpt","index":47.883650917956565,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:ioi-v1:overall","benchmarkName":"IOI v1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":3.666,"unit":"percent","family":"vals-ai:ioi-v1:overall","index":41.964430962707624,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/ioi-v1","measuredAt":"2026-08-09","stderr":3.347,"costPerTest":0.176857,"latency":2227.442}],"price":{"inputPerMillion":1.25,"outputPerMillion":10,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":55,"timeToFirstToken":4.0435,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":85,"benchmarkCount":15,"categoryCount":6,"confidence":"medium","state":"available"},"hosts":[{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":1.25,"outputPerMillion":10,"cacheReadPerMillion":0.13,"contextTokens":400000,"maxOutputTokens":128000,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:swe-bench-team","name":"SWE-bench team","url":"https://github.com","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"grok-4-20-beta","providerId":"xai","slug":"grok-4-20-beta","name":"Grok 4.20","provider":"xAI","description":"Grok 4.20 is a reasoning model from xAI. 18 benchmarks count toward its index, in 6 categories.","logoUrl":null,"releaseDate":"2026-03-10","contextWindowTokens":2000000,"modalities":["text","image","file"],"openWeights":false,"details":{"apiId":"x-ai/grok-4.20","maxOutputTokens":1800000,"knowledgeCutoff":"2025-09-01","retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.19999999999999998,"weightsUrl":null},"reasoning":true,"overallScore":52.77000598192065,"overallScoreRange":{"low":47.958700431325155,"high":57.58131153251614},"badges":[],"capabilityScores":{"agentic":50.70643568065979,"coding":55.37568722246593,"reasoning":55.99611638449172,"multimodal":45.84127092876427,"knowledge":55.384376470595456,"multilingual":null,"instruction":null,"math":57.43442768075309},"capabilityCounts":{"math":3,"knowledge":4,"coding":2,"multimodal":4,"agentic":4,"reasoning":1},"benchmarks":[{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":92.22222222222223,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":62.57343064972712,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":3.2829739672413996,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":89.33080808080808,"unit":"percent","family":"gpqa-diamond","index":59.14087693807023,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.8633389514966319,"costPerTest":null,"latency":null},{"benchmarkId":"gpqaDiamond","benchmarkName":"GPQA Diamond","category":"knowledge","score":88.5,"unit":"percent","family":"gpqa-diamond","index":58.37888246825051,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"sweVerified","benchmarkName":"Software Engineering Benchmark Verified","category":"coding","score":76.7,"unit":"percent","family":"swe-bench-verified","index":57.79859374548082,"aboveFit":false,"run":null,"source":"Carlos E. Jimenez, John Yang, Alexander Wettig, Shunyu Yao, Kexin Pei, Ofir Press, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2310.06770","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmmuPro","benchmarkName":"Massive Multi-discipline Multimodal Understanding Pro","category":"multimodal","score":75.2,"unit":"percent","family":"mmmu-pro","index":53.0190173647969,"aboveFit":false,"run":null,"source":"MMMU-Pro authors","sourceUrl":"https://arxiv.org/abs/2409.02813","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"liveCodeBenchPro","benchmarkName":"LiveCodeBench Pro","category":"coding","score":74.2,"unit":"percent","family":"liveCodeBenchPro","index":null,"aboveFit":false,"run":null,"source":"LiveCodeBench Pro authors","sourceUrl":"https://arxiv.org/abs/2506.11928","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"medXpertQaMm","benchmarkName":"MedXpertQA Multimodal","category":"multimodal","score":65.8,"unit":"percent","family":"medXpertQaMm","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"deepSearchQa","benchmarkName":"DeepSearchQA","category":"agentic","score":62.8,"unit":"percent","family":"deepSearchQa","index":46.83757120289639,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"charxiv","benchmarkName":"CharXiv Reasoning","category":"multimodal","score":60.9,"unit":"percent","family":"charxiv","index":33.51875514840623,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"simpleVqa","benchmarkName":"SimpleVQA","category":"multimodal","score":57.4,"unit":"percent","family":"simpleVqa","index":48.56099148828024,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"erqa","benchmarkName":"ERQA","category":"multimodal","score":54.1,"unit":"percent","family":"erqa","index":48.26631971357371,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"swePro","benchmarkName":"SWE-bench Pro","category":"coding","score":51.8,"unit":"percent","family":"swePro","index":52.95278069945104,"aboveFit":false,"run":null,"source":"Xiang Deng, Jeff Da, Edwin Pan, Yannis Yiming He, Charles Ide, Kanak Garg, Niklas Lauffer, Andrew Park, Nitin Pasari, Chetan Rane, Karmini Sampath, Maya Krishnan, Srivatsa Kundurthy, Sean Hendryx, Zifan Wang, Chen Bo Calvin Zhang, Noah Jacobson, Bing Liu, Brad Kenstler","sourceUrl":"https://arxiv.org/abs/2509.16941","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"medXpertQaText","benchmarkName":"MedXpertQA Text","category":"knowledge","score":50.2,"unit":"percent","family":"medXpertQaText","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"terminalBench2","benchmarkName":"Terminal-Bench 2.0","category":"agentic","score":47.1,"unit":"percent","family":"terminalBench2","index":47.40201135085949,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tiers-1-3-v2-private:overall","benchmarkName":"FrontierMath-Tiers-1-3-v2-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":44.91228070175438,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tiers-1-3-v2-private:overall","index":54.53435211101963,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.9515543245521676,"costPerTest":null,"latency":null},{"benchmarkId":"gertLabs","benchmarkName":"Gert Labs Composite Game Benchmark","category":"agentic","score":38.36,"unit":"percent","family":"gertLabs","index":46.60749282053545,"aboveFit":false,"run":null,"source":"Gert Labs","sourceUrl":"https://gertlabs.com/rankings","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hleNoTools","benchmarkName":"Humanity's Last Exam without tools","category":"knowledge","score":31.6,"unit":"percent","family":"hle","index":56.09069505625498,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-simpleqa-verified:overall","benchmarkName":"SimpleQA Verified","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":30.2,"unit":"percent","family":"epoch-ai:epoch-simpleqa-verified:overall","index":49.078318312483844,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.4526080235459493,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-chess-puzzles:overall","benchmarkName":"Chess Puzzles","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":28.999999999999996,"unit":"percent","family":"epoch-ai:epoch-chess-puzzles:overall","index":55.99611638449172,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":4.560480215720687,"costPerTest":null,"latency":null},{"benchmarkId":"healthBenchHard","benchmarkName":"HealthBench Hard","category":"knowledge","score":20.3,"unit":"percent","family":"healthBenchHard","index":57.60861281048265,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tier-4-v2-private:overall","benchmarkName":"FrontierMath-Tier-4-v2-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":17.073170731707318,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tier-4-v2-private:overall","index":55.1955002815125,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":5.949419959829496,"costPerTest":null,"latency":null},{"benchmarkId":"mazeBench","benchmarkName":"MazeBench: visual spatial reasoning in a 3D open world","category":"agentic","score":0,"unit":"percent","family":"mazeBench","index":61.97866734834783,"aboveFit":false,"run":null,"source":"Jonathan Pappas and David Pappas","sourceUrl":"https://mazebench.com/blog?post=maze-bench-results","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":1.25,"outputPerMillion":2.5,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":72,"timeToFirstToken":0.6795,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":90,"benchmarkCount":18,"categoryCount":6,"confidence":"medium","state":"available"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"grok-build-0-1","providerId":"xai","slug":"grok-build-0-1","name":"Grok Build 0.1","provider":"xAI","description":"Grok Build 0.1 is a standard model from xAI in the Grok Build family. 8 benchmarks count toward its index, in 6 categories.","logoUrl":null,"releaseDate":"2026-05-20","contextWindowTokens":256000,"modalities":["text","image","file"],"openWeights":false,"details":{"apiId":"x-ai/grok-build-0.1","maxOutputTokens":230400,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.19999999999999998,"weightsUrl":null},"reasoning":false,"overallScore":52.68705400155648,"overallScoreRange":{"low":45.470095675663245,"high":59.90401232744972},"badges":[],"capabilityScores":{"agentic":55.87401876087861,"coding":46.9246425853736,"reasoning":56.94050756912483,"multimodal":null,"knowledge":56.49720645141574,"multilingual":null,"instruction":58.55675004318953,"math":49.082486292294945},"capabilityCounts":{"math":1,"reasoning":1,"knowledge":1,"coding":2,"instruction":1,"agentic":2},"benchmarks":[{"benchmarkId":"livebench:livebench-math:overall","benchmarkName":"LiveBench Mathematics","category":"math","domain":null,"eligible":true,"ranked":true,"score":78.42875000000001,"unit":"percent","family":"livebench:livebench-math:overall","index":49.082486292294945,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Mathematics","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.0015730870712401058,"latency":null},{"benchmarkId":"livebench:livebench-reasoning:overall","benchmarkName":"LiveBench Reasoning","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":76.37025,"unit":"percent","family":"livebench","index":60.55233947963096,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Reasoning","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.00206980198019802,"latency":null},{"benchmarkId":"livebench:livebench-language:overall","benchmarkName":"LiveBench Language","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":72.46,"unit":"percent","family":"livebench:livebench-language:overall","index":56.49720645141574,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Language","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.001232,"latency":null},{"benchmarkId":"livebench:livebench-data-analysis:overall","benchmarkName":"LiveBench Data Analysis","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":70.794,"unit":"percent","family":"livebench","index":53.32867565861872,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Data%20Analysis","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.004236,"latency":null},{"benchmarkId":"livebench:livebench-coding:overall","benchmarkName":"LiveBench Coding","category":"coding","domain":null,"eligible":true,"ranked":true,"score":65.38550000000001,"unit":"percent","family":"livebench:livebench-coding:overall","index":46.78323365126619,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Coding","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.0008427350427350427,"latency":null},{"benchmarkId":"livebench:livebench-instruction-following:overall","benchmarkName":"LiveBench Instruction Following","category":"instruction","domain":null,"eligible":true,"ranked":true,"score":65.22075000000001,"unit":"percent","family":"livebench:livebench-instruction-following:overall","index":58.55675004318953,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=IF","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.0008984999999999999,"latency":null},{"benchmarkId":"gertLabs","benchmarkName":"Gert Labs Composite Game Benchmark","category":"agentic","score":49.15,"unit":"percent","family":"gertLabs","index":55.344336946353266,"aboveFit":false,"run":null,"source":"Gert Labs","sourceUrl":"https://gertlabs.com/rankings","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-agentic-coding:overall","benchmarkName":"LiveBench Agentic Coding","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":45.808,"unit":"percent","family":"livebench:livebench-agentic-coding:overall","index":56.40370057540396,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Agentic%20Coding","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.2579388888888889,"latency":null},{"benchmarkId":"vals-ai:vibe-code:overall","benchmarkName":"Vibe Code Bench v1.1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":13.349,"unit":"percent","family":"vals-ai:vibe-code:overall","index":47.06605151948101,"aboveFit":false,"run":"Grok Build","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","measuredAt":"2026-11-09","stderr":3.819,"costPerTest":null,"latency":394.638}],"price":{"inputPerMillion":1,"outputPerMillion":2,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":147,"timeToFirstToken":1.463,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":85,"benchmarkCount":8,"categoryCount":6,"confidence":"low","state":"available"},"hosts":[{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":1,"outputPerMillion":2,"cacheReadPerMillion":0.2,"contextTokens":256000,"maxOutputTokens":230400,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:livebench","name":"LiveBench","url":"https://livebench.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"kimik25thinking","providerId":"moonshot-ai","slug":"moonshot-ai-kimi-k2-5-thinking","name":"Kimi K2.5 Thinking","provider":"Moonshot AI","description":"Kimi K2.5 Thinking is a model from Moonshot AI. 11 benchmarks count toward its index, in 5 categories.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":52.621721544187,"overallScoreRange":{"low":45.24628310236456,"high":59.997159986009436},"badges":[],"capabilityScores":{"agentic":48.2903483651095,"coding":51.63692157539964,"reasoning":null,"multimodal":64.90741840666047,"knowledge":53.52857646501172,"multilingual":null,"instruction":null,"math":57.29506683176784},"capabilityCounts":{"math":1,"knowledge":2,"multimodal":1,"coding":5,"agentic":2},"benchmarks":[{"benchmarkId":"vals-ai:aime:overall","benchmarkName":"AIME","category":"math","domain":null,"eligible":true,"ranked":true,"score":95.625,"unit":"percent","family":"vals-ai:aime:overall","index":57.29506683176784,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/aime","measuredAt":"2026-04-16","stderr":0.343,"costPerTest":0.064142,"latency":322.611},{"benchmarkId":"vals-ai:medqa:overall","benchmarkName":"MedQA","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":94.367,"unit":"percent","family":"vals-ai:medqa:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medqa","measuredAt":"2026-04-16","stderr":0.21,"costPerTest":0.006992,"latency":41.628},{"benchmarkId":"vals-ai:mmlu_pro:overall","benchmarkName":"MMLU Pro","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":85.914,"unit":"percent","family":"mmlu-pro","index":52.722084785494275,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","measuredAt":"2026-09-01","stderr":0.343,"costPerTest":0.00793,"latency":39.204},{"benchmarkId":"vals-ai:mmmu:overall","benchmarkName":"MMMU Pro","category":"multimodal","domain":null,"eligible":true,"ranked":true,"score":84.335,"unit":"percent","family":"mmmu-pro","index":64.90741840666047,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmmu","measuredAt":"2026-09-01","stderr":0.874,"costPerTest":0.019563,"latency":110.011},{"benchmarkId":"vals-ai:gpqa:overall","benchmarkName":"GPQA Diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":84.091,"unit":"percent","family":"gpqa-diamond","index":54.33506814452917,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","measuredAt":"2026-09-01","stderr":2.135,"costPerTest":0.029962,"latency":151.843},{"benchmarkId":"vals-ai:lcb:overall","benchmarkName":"LiveCodeBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":83.868,"unit":"percent","family":"livecodebench","index":58.9325038975005,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","measuredAt":"2026-09-01","stderr":1.03,"costPerTest":0.038656,"latency":200.023},{"benchmarkId":"vals-ai:medscribe:overall","benchmarkName":"MedScribe","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":76.442,"unit":"percent","family":"vals-ai:medscribe:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medscribe","measuredAt":"2026-10-29","stderr":1.986,"costPerTest":0.02489,"latency":147.155},{"benchmarkId":"vals-ai:tax_eval_v2:overall","benchmarkName":"TaxEval v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":74.202,"unit":"percent","family":"vals-ai:tax_eval_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","measuredAt":"2026-09-01","stderr":0.846,"costPerTest":0.014221,"latency":90.464},{"benchmarkId":"vals-ai:swebench:overall","benchmarkName":"SWE-bench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":70,"unit":"percent","family":"swe-bench-verified","index":52.85695797433008,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/swebench","measuredAt":"2026-09-01","stderr":2.051,"costPerTest":0.167669,"latency":339.242},{"benchmarkId":"vals-ai:corp_fin_v2:overall","benchmarkName":"CorpFin v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":68.259,"unit":"percent","family":"vals-ai:corp_fin_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","measuredAt":"2026-08-12","stderr":0.917,"costPerTest":0.052412,"latency":80.779},{"benchmarkId":"vals-ai:mortgage_tax:overall","benchmarkName":"MortgageTax","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":66.534,"unit":"percent","family":"vals-ai:mortgage_tax:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mortgage_tax","measuredAt":"2026-09-01","stderr":0.925,"costPerTest":0.014918,"latency":79.787},{"benchmarkId":"vals-ai:case_law_v2:overall","benchmarkName":"CaseLaw v2","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":58.735,"unit":"percent","family":"vals-ai:case_law_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/case_law_v2","measuredAt":"2026-05-04","stderr":0.703,"costPerTest":0.024886,"latency":241.278},{"benchmarkId":"vals-ai:finance_agent:overall","benchmarkName":"Finance Agent v1.1","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":50.622,"unit":"percent","family":"vals-ai:finance_agent:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/finance_agent","measuredAt":"2026-06-04","stderr":2.845,"costPerTest":0.178983,"latency":268.752},{"benchmarkId":"vals-ai:terminal-bench-2-1:overall","benchmarkName":"Terminal-Bench 2.1","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":41.948,"unit":"percent","family":"terminal-bench-2-1","index":45.67463403626749,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2-1","measuredAt":"2026-10-01","stderr":0.749,"costPerTest":0.085844,"latency":1051.079},{"benchmarkId":"vals-ai:terminal-bench-2:overall","benchmarkName":"Terminal-Bench 2.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":40.449,"unit":"percent","family":"vals-ai:terminal-bench-2:overall","index":50.90606269395151,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2","measuredAt":"2026-06-04","stderr":5.232,"costPerTest":0.139997,"latency":756.405},{"benchmarkId":"vals-ai:medcode:overall","benchmarkName":"MedCode","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":39.316,"unit":"percent","family":"vals-ai:medcode:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medcode","measuredAt":"2026-10-29","stderr":2.119,"costPerTest":0.017275,"latency":75.452},{"benchmarkId":"vals-ai:fabv2:overall","benchmarkName":"Finance Agent (v2)","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":35.792,"unit":"percent","family":"vals-ai:fabv2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/fabv2","measuredAt":"2026-10-29","stderr":1.989,"costPerTest":0.294389,"latency":751.754},{"benchmarkId":"vals-ai:emb:overall","benchmarkName":"EMB","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":28.469,"unit":"percent","family":"vals-ai:emb:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/emb","measuredAt":"2026-10-20","stderr":2.23,"costPerTest":0.786639,"latency":3796.855},{"benchmarkId":"vals-ai:ioi-v1:overall","benchmarkName":"IOI v1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":17.667,"unit":"percent","family":"vals-ai:ioi-v1:overall","index":48.93072812599887,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/ioi-v1","measuredAt":"2026-08-09","stderr":6.084,"costPerTest":0.677261,"latency":3828.386},{"benchmarkId":"vals-ai:vibe-code:overall","benchmarkName":"Vibe Code Bench v1.1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":17.536,"unit":"percent","family":"vals-ai:vibe-code:overall","index":48.57678690218467,"aboveFit":false,"run":"OpenHands","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","measuredAt":"2026-11-09","stderr":3.258,"costPerTest":0.879292,"latency":2570.381},{"benchmarkId":"vals-ai:legal_research:overall","benchmarkName":"Legal Research Bench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":15.865,"unit":"percent","family":"vals-ai:legal_research:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_research","measuredAt":"2026-11-09","stderr":2.539,"costPerTest":0.23035,"latency":1332.514},{"benchmarkId":"vals-ai:code-migration:overall","benchmarkName":"Code Migration","category":"coding","domain":null,"eligible":true,"ranked":true,"score":6.962,"unit":"percent","family":"vals-ai:code-migration:overall","index":48.887630976984084,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/code-migration","measuredAt":"2026-10-29","stderr":1.29,"costPerTest":0.926909,"latency":3820.711}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":{"elo":1444.6866209257523,"eloUpper":1447.949811089762,"eloLower":1441.423430761742,"votes":73872,"rank":66,"rankUpper":null,"rankLower":null,"arenaName":"Kimi K2.5-thinking","organization":"moonshot","updatedAt":"2026-11-09"},"quality":{"coveragePercent":75,"benchmarkCount":11,"categoryCount":5,"confidence":"low","state":"available"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"mercury-3","providerId":"inception","slug":"mercury-3","name":"Mercury 3","provider":"Inception","description":"Mercury 3 is a reasoning model from Inception. 8 benchmarks count toward its index, in 5 categories.","logoUrl":null,"releaseDate":"2026-10-19","contextWindowTokens":1000000,"modalities":["text"],"openWeights":false,"details":{"apiId":"inception/mercury-3","maxOutputTokens":131072,"knowledgeCutoff":null,"retiresAt":"2026-11-08","tools":true,"structuredOutput":true,"cacheReadPerMillion":0.01,"weightsUrl":null},"reasoning":true,"overallScore":52.32067017661652,"overallScoreRange":{"low":44.10371185072328,"high":60.537628502509754},"badges":[],"capabilityScores":{"agentic":44.445799319156244,"coding":64.23294142592358,"reasoning":57.005627854513264,"multimodal":null,"knowledge":59.04841852094195,"multilingual":null,"instruction":60.831425035563925,"math":null},"capabilityCounts":{"knowledge":1,"instruction":1,"reasoning":1,"agentic":4,"coding":1},"benchmarks":[{"benchmarkId":"gpqaDiamond","benchmarkName":"GPQA Diamond","category":"knowledge","score":89.23,"unit":"percent","family":"gpqa-diamond","index":59.04841852094195,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"ifBench","benchmarkName":"Instruction Following Benchmark","category":"instruction","score":76.84,"unit":"percent","family":"ifbench-family","index":60.831425035563925,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mrcrv2","benchmarkName":"MRCRv2","category":"reasoning","score":75.5,"unit":"percent","family":"mrcrv2","index":57.005627854513264,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"deepSearchQa","benchmarkName":"DeepSearchQA","category":"agentic","score":71.33,"unit":"percent","family":"deepSearchQa","index":52.82720185021345,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:vibe-code:overall","benchmarkName":"Vibe Code Bench v1.1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":60.927,"unit":"percent","family":"vals-ai:vibe-code:overall","index":64.23294142592358,"aboveFit":false,"run":"OpenHands","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","measuredAt":"2026-11-09","stderr":5.379,"costPerTest":0.627191,"latency":566.58},{"benchmarkId":"multiChallenge","benchmarkName":"MultiChallenge: provider-reported runs","category":"instruction","score":60.35,"unit":"percent","family":"multiChallenge","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"browseCompPlus","benchmarkName":"BrowseComp-Plus","category":"agentic","score":55.18,"unit":"percent","family":"browseCompPlus","index":null,"aboveFit":false,"run":null,"source":"Zijian Chen et al.","sourceUrl":"https://arxiv.org/abs/2508.06600","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"deepSwe","benchmarkName":"DeepSWE","category":"agentic","score":46.31,"unit":"percent","family":"deepSwe","index":54.74279784262619,"aboveFit":false,"run":null,"source":"Datacurve AI","sourceUrl":"https://deepswe.datacurve.ai/blog","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"tau3Bench","benchmarkName":"τ³-Bench Tool-Agent-User Evaluation","category":"agentic","score":30.24,"unit":"percent","family":"tau3Bench","index":14.258423064730472,"aboveFit":false,"run":null,"source":"Sierra Research","sourceUrl":"https://github.com/sierra-research/tau2-bench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:legal_research:overall","benchmarkName":"Legal Research Bench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":28.846,"unit":"percent","family":"vals-ai:legal_research:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_research","measuredAt":"2026-11-09","stderr":3.149,"costPerTest":0.074351,"latency":446.517},{"benchmarkId":"vals-ai:tax_agent_bench:overall","benchmarkName":"Tax Agent Bench","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":18.169,"unit":"percent","family":"vals-ai:tax_agent_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench","measuredAt":"2026-11-09","stderr":2.536,"costPerTest":0.045925,"latency":269.272},{"benchmarkId":"vals-ai:hlab:overall","benchmarkName":"HLAB-Vals-1.1","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":5,"unit":"percent","family":"vals-ai:hlab:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/hlab","measuredAt":"2026-11-09","stderr":1.61,"costPerTest":0.188085,"latency":886.884},{"benchmarkId":"vals-ai:terminal-bench-4:overall","benchmarkName":"Terminal-Bench 4.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":3.03,"unit":"percent","family":"terminal-bench-4","index":55.954774519054865,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-4","measuredAt":"2026-11-09","stderr":0,"costPerTest":0.421803,"latency":1987.184}],"price":{"inputPerMillion":0.09999999999999999,"outputPerMillion":0.35,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":295,"timeToFirstToken":0.315,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":75,"benchmarkCount":8,"categoryCount":5,"confidence":"low","state":"available"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"ripple-65b-preview","providerId":"ricursive-intelligence","slug":"ripple-65b-preview","name":"Ripple-65B Preview","provider":"Ricursive Intelligence","description":"Ripple-65B Preview is a reasoning model from Ricursive Intelligence in the Ripple family. 17 benchmarks count toward its index, in 6 categories.","logoUrl":null,"releaseDate":"2026-11-04","contextWindowTokens":1048576,"modalities":["text"],"openWeights":true,"details":{"apiId":"ricursiveai/ripple-65b-preview","maxOutputTokens":131072,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.049999999999999996,"weightsUrl":"https://huggingface.co/ricursiveAI/Ripple-65B-Preview"},"reasoning":true,"overallScore":52.30073851637254,"overallScoreRange":{"low":46.87893745953504,"high":57.72253957321004},"badges":[],"capabilityScores":{"agentic":49.51960062738071,"coding":55.33386018655176,"reasoning":58.75408887809522,"multimodal":null,"knowledge":48.14295173481686,"multilingual":null,"instruction":63.03687008656155,"math":52.68644558826176},"capabilityCounts":{"math":4,"knowledge":3,"agentic":5,"instruction":1,"coding":3,"reasoning":1},"benchmarks":[{"benchmarkId":"aime2025","benchmarkName":"American Invitational Mathematics Examination 2025","category":"math","score":98.3,"unit":"percent","family":"aime2025","index":52.34733950560558,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aime2024","benchmarkName":"American Invitational Mathematics Examination 2024","category":"math","score":97.1,"unit":"percent","family":"aime2024","index":47.97880570665632,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aime2026","benchmarkName":"AIME 2026","category":"math","score":95,"unit":"percent","family":"aime2026","index":53.685325878862685,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hmmtFeb2026","benchmarkName":"Harvard-MIT Mathematics Tournament February 2026","category":"math","score":90.5,"unit":"percent","family":"hmmtFeb2026","index":56.734311261922464,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmlu","benchmarkName":"Massive Multitask Language Understanding","category":"knowledge","score":89.6,"unit":"percent","family":"mmlu","index":38.51399261741484,"aboveFit":false,"run":null,"source":"Dan Hendrycks, Collin Burns, Steven Basart, Andy Zou, Mantas Mazeika, Dawn Song, Jacob Steinhardt","sourceUrl":"https://arxiv.org/abs/2009.03300","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gpqaDiamond","benchmarkName":"GPQA Diamond","category":"knowledge","score":85.2,"unit":"percent","family":"gpqa-diamond","index":55.352212641015214,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmluPro","benchmarkName":"Massive Multitask Language Understanding Professional","category":"knowledge","score":84.2,"unit":"percent","family":"mmlu-pro","index":50.562649946020514,"aboveFit":false,"run":null,"source":"Yubo Wang, Xueguang Ma, Ge Zhang, Yuansheng Ni, Abhranil Chandra, Shiguang Guo, Weiming Ren, Aaran Arulraj, Xuan He, Ziyan Jiang, Tianle Li, Max Ku, Kai Wang, Alex Zhuang, Rongqi Fan, Xiang Yue, Wenhu Chen","sourceUrl":"https://arxiv.org/abs/2406.01574","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"terminalBench21","benchmarkName":"Terminal-Bench 2.1","category":"agentic","score":80.5,"unit":"percent","family":"terminal-bench-2-1","index":68.2219780474297,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"ifBench","benchmarkName":"Instruction Following Benchmark","category":"instruction","score":79.2,"unit":"percent","family":"ifbench-family","index":63.03687008656155,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"sweVerified","benchmarkName":"Software Engineering Benchmark Verified","category":"coding","score":76.5,"unit":"percent","family":"swe-bench-verified","index":57.65108222992408,"aboveFit":false,"run":null,"source":"Carlos E. Jimenez, John Yang, Alexander Wettig, Shunyu Yao, Kexin Pei, Ofir Press, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2310.06770","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":75,"unit":"percent","family":"lcr","index":58.75408887809522,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mcpAtlas","benchmarkName":"MCP Atlas","category":"agentic","score":71.67,"unit":"percent","family":"mcpAtlas","index":59.324545682073605,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"swePro","benchmarkName":"SWE-bench Pro","category":"coding","score":61.42,"unit":"percent","family":"swePro","index":61.145268718479436,"aboveFit":false,"run":null,"source":"Xiang Deng, Jeff Da, Edwin Pan, Yannis Yiming He, Charles Ide, Kanak Garg, Niklas Lauffer, Andrew Park, Nitin Pasari, Chetan Rane, Karmini Sampath, Maya Krishnan, Srivatsa Kundurthy, Sean Hendryx, Zifan Wang, Chen Bo Calvin Zhang, Noah Jacobson, Bing Liu, Brad Kenstler","sourceUrl":"https://arxiv.org/abs/2509.16941","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"deepSwe","benchmarkName":"DeepSWE","category":"agentic","score":44.3,"unit":"percent","family":"deepSwe","index":53.28158395379834,"aboveFit":false,"run":null,"source":"Datacurve AI","sourceUrl":"https://deepswe.datacurve.ai/blog","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"sciCode","benchmarkName":"Scientific Code Benchmark","category":"coding","score":39.4,"unit":"percent","family":"scicode-family","index":47.20522961125175,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"automationBench","benchmarkName":"AutomationBench","category":"agentic","score":36.4,"unit":"percent","family":"automationBench","index":51.32668233671742,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"tau3Bench","benchmarkName":"τ³-Bench Tool-Agent-User Evaluation","category":"agentic","score":31.27,"unit":"percent","family":"tau3Bench","index":15.443213116884493,"aboveFit":false,"run":null,"source":"Sierra Research","sourceUrl":"https://github.com/sierra-research/tau2-bench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":0.19999999999999998,"outputPerMillion":0.6,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":63,"timeToFirstToken":0.4,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":85,"benchmarkCount":17,"categoryCount":6,"confidence":"medium","state":"available"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"hy3-preview","providerId":"tencent","slug":"hy3-preview","name":"Hy3 Preview","provider":"Tencent","description":"Hy3 Preview is a reasoning model from Tencent in the Hy3 family. 11 benchmarks count toward its index, in 5 categories.","logoUrl":null,"releaseDate":"2026-04-23","contextWindowTokens":262144,"modalities":["text"],"openWeights":true,"details":{"apiId":"tencent/hy3-preview","maxOutputTokens":235929,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":false,"cacheReadPerMillion":0.06,"weightsUrl":"https://huggingface.co/tencent/Hy3-preview"},"reasoning":true,"overallScore":52.27669192872712,"overallScoreRange":{"low":44.90125348690468,"high":59.65213037054956},"badges":[],"capabilityScores":{"agentic":52.55272665650227,"coding":55.23887111655556,"reasoning":49.697282390165846,"multimodal":null,"knowledge":53.85278408234906,"multilingual":null,"instruction":47.991249187806545,"math":null},"capabilityCounts":{"knowledge":3,"coding":2,"reasoning":2,"instruction":1,"agentic":3},"benchmarks":[{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":89.7,"unit":"percent","family":"gpqa-diamond","index":59.47948967815425,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gpqa","benchmarkName":"Graduate-Level Google-Proof Q&A","category":"knowledge","score":87.2,"unit":"percent","family":"gpqa-diamond","index":57.18655799085479,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gpqaDiamond","benchmarkName":"GPQA Diamond","category":"knowledge","score":87.2,"unit":"percent","family":"gpqa-diamond","index":57.18655799085479,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"sweVerified","benchmarkName":"Software Engineering Benchmark Verified","category":"coding","score":74.4,"unit":"percent","family":"swe-bench-verified","index":56.102211316578334,"aboveFit":false,"run":null,"source":"Carlos E. Jimenez, John Yang, Alexander Wettig, Shunyu Yao, Kexin Pei, Ofir Press, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2310.06770","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":66.7,"unit":"percent","family":"lcr","index":53.05962051573424,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"ifBench","benchmarkName":"Instruction Following Benchmark","category":"instruction","score":63.1,"unit":"percent","family":"ifbench-family","index":47.991249187806545,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"terminalBench2","benchmarkName":"Terminal-Bench 2.0","category":"agentic","score":54.4,"unit":"percent","family":"terminalBench2","index":52.594833503566505,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaSciCode","benchmarkName":"Artificial Analysis SciCode","category":"coding","score":48.6,"unit":"percent","family":"scicode-family","index":59.1991881582673,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/scicode","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"sciCode","benchmarkName":"Scientific Code Benchmark","category":"coding","score":41.2,"unit":"percent","family":"scicode-family","index":49.55187367479828,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gertLabs","benchmarkName":"Gert Labs Composite Game Benchmark","category":"agentic","score":36.91,"unit":"percent","family":"gertLabs","index":45.43340348017995,"aboveFit":false,"run":null,"source":"Gert Labs","sourceUrl":"https://gertlabs.com/rankings","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gdpvalAaNormalized","benchmarkName":"GDPval-AA normalized","category":"agentic","score":35.8,"unit":"percent","family":"gdpvalAaNormalized","index":59.629942985760366,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":33.5,"unit":"percent","family":"hle","index":57.99695127181159,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"omniscienceAccuracy","benchmarkName":"Artificial Analysis Omniscience Accuracy","category":"knowledge","score":31.5,"unit":"percent","family":"omniscienceAccuracy","index":50.388013964499976,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hle","benchmarkName":"Humanity's Last Exam","category":"knowledge","score":25.5,"unit":"percent","family":"hle","index":49.970609311573234,"aboveFit":false,"run":null,"source":"Center for AI Safety, Scale AI, and thousands of expert contributors","sourceUrl":"https://lastexam.ai/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":4.9,"unit":"percent","family":"critpt","index":46.33494426459745,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":0.18,"outputPerMillion":0.6,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":69,"timeToFirstToken":2.0235,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":{"elo":1401.2970216608937,"eloUpper":1408.7060487290805,"eloLower":1393.8879945927065,"votes":6815,"rank":155,"rankUpper":null,"rankLower":null,"arenaName":"Hy3 preview","organization":"tencent","updatedAt":"2026-11-09"},"quality":{"coveragePercent":75,"benchmarkCount":11,"categoryCount":5,"confidence":"low","state":"available"},"hosts":[{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":0.18,"outputPerMillion":0.6,"cacheReadPerMillion":0.06,"contextTokens":262144,"maxOutputTokens":235929,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"kimi-k2-5","providerId":"moonshot-ai","slug":"kimi-k2-5","name":"Kimi K2.5","provider":"Moonshot AI","description":"Kimi K2.5 is a standard model from Moonshot AI. 55 benchmarks count toward its index, in 8 categories.","logoUrl":null,"releaseDate":"2026-01-27","contextWindowTokens":262144,"modalities":["text","image"],"openWeights":true,"details":{"apiId":"moonshotai/kimi-k2.5","maxOutputTokens":65536,"knowledgeCutoff":null,"retiresAt":"2026-11-18","tools":true,"structuredOutput":true,"cacheReadPerMillion":0.20900000000000002,"weightsUrl":"https://huggingface.co/moonshotai/Kimi-K2.5"},"reasoning":false,"overallScore":52.225124175164225,"overallScoreRange":{"low":50.0447618199581,"high":54.405486530370354},"badges":[],"capabilityScores":{"agentic":49.44713762022306,"coding":53.8644570506537,"reasoning":51.593457613898856,"multimodal":52.43702155750208,"knowledge":53.71354580005693,"multilingual":52.70362556411929,"instruction":53.45762926240856,"math":53.77007508783397},"capabilityCounts":{"math":8,"agentic":16,"knowledge":8,"multimodal":8,"instruction":2,"coding":6,"multilingual":2,"reasoning":5},"benchmarks":[{"benchmarkId":"aime2025Arcee","benchmarkName":"AIME25 first-party comparison snapshot","category":"math","score":96.3,"unit":"percent","family":"aime2025Arcee","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aime2025","benchmarkName":"American Invitational Mathematics Examination 2025","category":"math","score":96.1,"unit":"percent","family":"aime2025","index":51.1535627096588,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"tau2Bench","benchmarkName":"τ²-Bench Tool-Agent-User Evaluation","category":"agentic","score":95.9,"unit":"percent","family":"tau2-bench","index":66.42610157862087,"aboveFit":false,"run":null,"source":"Victor Barres, Honghua Dong, Soham Ray, Xujie Si, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2506.07982","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aime2026","benchmarkName":"AIME 2026","category":"math","score":95.8,"unit":"percent","family":"aime2026","index":54.261278291543526,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hmmtFeb2025","benchmarkName":"Harvard-MIT Mathematics Tournament February 2025","category":"math","score":95.4,"unit":"percent","family":"hmmtFeb2025","index":53.27552299821299,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmluRedux","benchmarkName":"MMLU-Redux","category":"knowledge","score":94.5,"unit":"percent","family":"mmluRedux","index":50.46926829204164,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"countBench","benchmarkName":"CountBench","category":"multimodal","score":94.1,"unit":"percent","family":"countBench","index":47.229383114904195,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"cEval","benchmarkName":"C-Eval","category":"knowledge","score":94,"unit":"percent","family":"cEval","index":51.16277820660081,"aboveFit":false,"run":null,"source":"C-Eval authors","sourceUrl":"https://arxiv.org/abs/2305.08322","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"ifeval","benchmarkName":"Instruction-Following Eval","category":"instruction","score":93.9,"unit":"percent","family":"ifeval","index":52.28898397172111,"aboveFit":false,"run":null,"source":"Jeffrey Zhou, Tianjian Lu, Swaroop Mishra, Siddhartha Brahma, Sujoy Basu, Yi Luan, Denny Zhou, Le Hou","sourceUrl":"https://arxiv.org/abs/2311.07911","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":92.2,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":62.560501893155276,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":3.9,"costPerTest":null,"latency":null},{"benchmarkId":"hmmtNov2025","benchmarkName":"Harvard-MIT Mathematics Tournament November 2025","category":"math","score":91.1,"unit":"percent","family":"hmmtNov2025","index":52.54291611772137,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"ai2dTest","benchmarkName":"AI2D test split","category":"multimodal","score":90.8,"unit":"percent","family":"ai2dTest","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"omniDocBench15","benchmarkName":"OmniDocBench 1.5","category":"multimodal","score":88.8,"unit":"percent","family":"omniDocBench15","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":87.9,"unit":"percent","family":"gpqa-diamond","index":57.82857886329864,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"refcocoAvg","benchmarkName":"RefCOCO average","category":"multimodal","score":87.8,"unit":"percent","family":"refcocoAvg","index":42.146935586436996,"aboveFit":false,"run":null,"source":"RefCOCO dataset authors","sourceUrl":"https://github.com/lichengunc/refer","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gpqa","benchmarkName":"Graduate-Level Google-Proof Q&A","category":"knowledge","score":87.6,"unit":"percent","family":"gpqa-diamond","index":57.55342706082269,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gpqaDiamond","benchmarkName":"GPQA Diamond","category":"knowledge","score":87.6,"unit":"percent","family":"gpqa-diamond","index":57.55342706082269,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":87.6,"unit":"percent","family":"gpqa-diamond","index":57.55342706082269,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.9,"costPerTest":null,"latency":null},{"benchmarkId":"videoMme","benchmarkName":"Video-MME","category":"multimodal","score":87.4,"unit":"percent","family":"videoMme","index":null,"aboveFit":false,"run":null,"source":"Video-MME benchmark team","sourceUrl":"https://mme-benchmark.github.io/","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"videoMmeWithSub","benchmarkName":"Video-MME with subtitle","category":"multimodal","score":87.4,"unit":"percent","family":"videoMmeWithSub","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmluPro","benchmarkName":"Massive Multitask Language Understanding Professional","category":"knowledge","score":87.1,"unit":"percent","family":"mmlu-pro","index":54.21630282494343,"aboveFit":false,"run":null,"source":"Yubo Wang, Xueguang Ma, Ge Zhang, Yuansheng Ni, Abhranil Chandra, Shiguang Guo, Weiming Ren, Aaran Arulraj, Xuan He, Ziyan Jiang, Tianle Li, Max Ku, Kai Wang, Alex Zhuang, Rongqi Fan, Xiang Yue, Wenhu Chen","sourceUrl":"https://arxiv.org/abs/2406.01574","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmluProArcee","benchmarkName":"MMLU-Pro first-party comparison snapshot","category":"knowledge","score":87.1,"unit":"percent","family":"mmlu-pro","index":54.21630282494343,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hmmtFeb2026","benchmarkName":"Harvard-MIT Mathematics Tournament February 2026","category":"math","score":87.1,"unit":"percent","family":"hmmtFeb2026","index":54.35492457541483,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"videoMmmu","benchmarkName":"VideoMMMU","category":"multimodal","score":86.6,"unit":"percent","family":"videoMmmu","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"liveCodeBenchV6","benchmarkName":"LiveCodeBench v6","category":"coding","score":85,"unit":"percent","family":"liveCodeBenchV6","index":54.20692860934721,"aboveFit":false,"run":null,"source":"LiveCodeBench maintainers","sourceUrl":"https://github.com/LiveCodeBench/LiveCodeBench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mlvuAvg","benchmarkName":"MLVU mean average","category":"multimodal","score":85,"unit":"percent","family":"mlvuAvg","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"weMath","benchmarkName":"We-Math","category":"multimodal","score":84.7,"unit":"percent","family":"weMath","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"dynaMath","benchmarkName":"DynaMath","category":"multimodal","score":84.4,"unit":"percent","family":"dynaMath","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmmu","benchmarkName":"Massive Multi-discipline Multimodal Understanding","category":"multimodal","score":84.3,"unit":"percent","family":"mmmu","index":53.34458984587063,"aboveFit":false,"run":null,"source":"MMMU authors","sourceUrl":"https://arxiv.org/abs/2401.05508","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mathVision","benchmarkName":"MathVision","category":"multimodal","score":84.2,"unit":"percent","family":"mathVision","index":55.36403326803516,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"include","benchmarkName":"INCLUDE","category":"multilingual","score":83.3,"unit":"percent","family":"include","index":53.71876799724793,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"videoMmeNoSub","benchmarkName":"Video-MME without subtitle","category":"multimodal","score":83.2,"unit":"percent","family":"videoMmeNoSub","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmluProX","benchmarkName":"MMLU-ProX","category":"multilingual","score":82.3,"unit":"percent","family":"mmluProX","index":51.688483130990655,"aboveFit":false,"run":null,"source":"MMLU-ProX authors","sourceUrl":"https://arxiv.org/abs/2503.10497","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmAnswerBench","benchmarkName":"MMAnswerBench","category":"math","score":81.8,"unit":"percent","family":"mmAnswerBench","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"realWorldQa","benchmarkName":"RealWorldQA","category":"multimodal","score":81,"unit":"percent","family":"realWorldQa","index":52.19616571183417,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmvu","benchmarkName":"Multimodal Multi-disciplinary Video Understanding","category":"multimodal","score":80.4,"unit":"percent","family":"mmvu","index":null,"aboveFit":false,"run":null,"source":"MMVU benchmark maintainers","sourceUrl":"https://www.kimi.com/blog/kimi-k2-5.html","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"ccOcr","benchmarkName":"CC-OCR","category":"multimodal","score":79.7,"unit":"percent","family":"ccOcr","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmmuPro","benchmarkName":"Massive Multi-discipline Multimodal Understanding Pro","category":"multimodal","score":78.5,"unit":"percent","family":"mmmu-pro","index":57.31367783969014,"aboveFit":false,"run":null,"source":"MMMU-Pro authors","sourceUrl":"https://arxiv.org/abs/2409.02813","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":78,"unit":"percent","family":"lcr","index":60.8123304548522,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"charxiv","benchmarkName":"CharXiv Reasoning","category":"multimodal","score":77.5,"unit":"percent","family":"charxiv","index":52.41782901281534,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"reactNativeEvals","benchmarkName":"React Native Evals","category":"coding","score":77.2,"unit":"percent","family":"reactNativeEvals","index":53.75132712160312,"aboveFit":false,"run":null,"source":"Callstack","sourceUrl":"https://rn-evals.vercel.app/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"deepSearchQa","benchmarkName":"DeepSearchQA","category":"agentic","score":77.1,"unit":"percent","family":"deepSearchQa","index":56.878804292771406,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"sweVerified","benchmarkName":"Software Engineering Benchmark Verified","category":"coding","score":76.8,"unit":"percent","family":"swe-bench-verified","index":57.87234950325919,"aboveFit":false,"run":null,"source":"Carlos E. Jimenez, John Yang, Alexander Wettig, Shunyu Yao, Kexin Pei, Ofir Press, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2310.06770","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaMmmuPro","benchmarkName":"Artificial Analysis MMMU-Pro","category":"multimodal","score":75.4,"unit":"percent","family":"mmmu-pro","index":53.279299817820736,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmmu-pro","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-swe-bench-verified:overall","benchmarkName":"SWE-Bench verified","category":"coding","domain":null,"eligible":true,"ranked":true,"score":73.7603305785124,"unit":"percent","family":"swe-bench-verified","index":55.630418287483636,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.0017832064928274,"costPerTest":null,"latency":null},{"benchmarkId":"arc-prize:v1-public-eval:overall","benchmarkName":"ARC-AGI-1 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":73.11999999999999,"unit":"percent","family":"arc-prize:v1-public-eval:overall","index":null,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.14,"latency":null},{"benchmarkId":"maxife","benchmarkName":"MAXIFE","category":"multilingual","score":72.8,"unit":"percent","family":"maxife","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"wideResearch","benchmarkName":"WideResearch","category":"agentic","score":72.7,"unit":"percent","family":"wideResearch","index":55.58469733484569,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"simpleVqa","benchmarkName":"SimpleVQA","category":"multimodal","score":71.2,"unit":"percent","family":"simpleVqa","index":61.50074709136468,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"sweVerifiedArcee","benchmarkName":"SWE-bench Verified (mini-swe-agent-v2)","category":"coding","score":70.8,"unit":"percent","family":"swe-bench-verified","index":53.44700403655703,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"swebench-experiments:swe-bench-verified:overall","benchmarkName":"SWE-bench Verified","category":"coding","domain":null,"eligible":true,"ranked":true,"score":70.8,"unit":"percent","family":"swe-bench-verified","index":53.44700403655703,"aboveFit":false,"run":"high effort · mini-SWE-agent","source":"SWE-bench team","sourceUrl":"https://github.com/SWE-bench/experiments/tree/main/evaluation/verified","measuredAt":"2026-09-01","stderr":null,"costPerTest":0.14655374074400002,"latency":null},{"benchmarkId":"aaIfBench","benchmarkName":"Artificial Analysis IFBench","category":"instruction","score":70.2,"unit":"percent","family":"ifbench-family","index":54.62627455309602,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"superGpqa","benchmarkName":"SuperGPQA: Scaling LLM Evaluation Across 285 Graduate Disciplines","category":"knowledge","score":69.2,"unit":"percent","family":"superGpqa","index":53.87490489469795,"aboveFit":false,"run":null,"source":"Xiaoxuan Du, Yao Yao, Kexin Ma, Bowen Wang, Tianyu Zheng, Kaiyan Zhu, Yiming Zhang, Yutao Zhu, Jiawei Zhou, Jingren Zhou","sourceUrl":"https://arxiv.org/abs/2502.14739","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"bfclV4","benchmarkName":"Berkeley Function Calling Leaderboard v4","category":"agentic","score":68.3,"unit":"percent","family":"bfclV4","index":48.57774891970743,"aboveFit":false,"run":null,"source":"Shishir G. Patil et al.","sourceUrl":"https://gorilla.cs.berkeley.edu/leaderboard.html","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"swe-bench:swe-bench-multilingual:overall","benchmarkName":"SWE-bench Multilingual","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":67.3,"unit":"percent","family":"swe-bench-multilingual","index":null,"aboveFit":false,"run":"mini-SWE-agent","source":"SWE-bench team","sourceUrl":"https://www.swebench.com/","measuredAt":"2026-02-20","stderr":null,"costPerTest":0.6928368501333334,"latency":null},{"benchmarkId":"swebench-experiments:swe-bench-multilingual:overall","benchmarkName":"SWE-bench Multilingual","category":"coding","domain":null,"eligible":true,"ranked":true,"score":67.3,"unit":"percent","family":"swe-bench-multilingual","index":null,"aboveFit":false,"run":"mini-SWE-agent","source":"SWE-bench team","sourceUrl":"https://github.com/SWE-bench/experiments/tree/main/evaluation/multilingual","measuredAt":"2026-09-02","stderr":null,"costPerTest":0.6928368501333334,"latency":null},{"benchmarkId":"tau3Bench","benchmarkName":"τ³-Bench Tool-Agent-User Evaluation","category":"agentic","score":65.7,"unit":"percent","family":"tau3Bench","index":55.04740874374155,"aboveFit":false,"run":null,"source":"Sierra Research","sourceUrl":"https://github.com/sierra-research/tau2-bench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"arc-prize:v1-semi-private:overall","benchmarkName":"ARC-AGI-1 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":65.33,"unit":"percent","family":"arc-prize:v1-semi-private:overall","index":57.021918467508115,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.15,"latency":null},{"benchmarkId":"osWorldVerified","benchmarkName":"OSWorld-Verified","category":"agentic","score":63.3,"unit":"percent","family":"osWorldVerified","index":51.58038258679388,"aboveFit":false,"run":null,"source":"Tianbao Xie, Danyang Zhang, Jixuan Chen, Xiaochuan Li, Siheng Zhao, Ruisheng Cao, Toh Jing Hua, Zhoujun Cheng, Dongchan Shin, Fangyu Lei, Yitao Liu, Yiheng Xu, Shuyan Zhou, Silvio Savarese, Caiming Xiong, Victor Zhong, Tao Yu","sourceUrl":"https://os-world.github.io/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"longBenchV2","benchmarkName":"LongBench v2","category":"reasoning","score":61,"unit":"percent","family":"longBenchV2","index":52.67178863182358,"aboveFit":false,"run":null,"source":"LongBench v2 authors","sourceUrl":"https://arxiv.org/abs/2412.15204","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"browseComp","benchmarkName":"BrowseComp","category":"agentic","score":60.6,"unit":"percent","family":"browseComp","index":47.784194294156336,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mcpTasks","benchmarkName":"MCP-Tasks","category":"agentic","score":59.1,"unit":"percent","family":"mcpTasks","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"sweRebench","benchmarkName":"SWE-Rebench","category":"coding","score":58.5,"unit":"percent","family":"sweRebench","index":null,"aboveFit":false,"run":null,"source":"Nebius","sourceUrl":"https://swe-rebench.com","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmLongBenchDoc","benchmarkName":"MMLongBench-Doc","category":"multimodal","score":58.5,"unit":"percent","family":"mmLongBenchDoc","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"nova63","benchmarkName":"NOVA-63","category":"multilingual","score":56,"unit":"percent","family":"nova63","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"qwenClawBench","benchmarkName":"QwenClawBench","category":"agentic","score":54.3,"unit":"percent","family":"qwenClawBench","index":52.58698573409087,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"clawEval","benchmarkName":"Claw-Eval","category":"agentic","score":52.3,"unit":"percent","family":"clawEval","index":40.735368379190895,"aboveFit":false,"run":null,"source":"Bowen Ye, Rang Li, Qibin Yang, Yuanxin Liu, Linli Yao, Hanglong Lv, Zhihui Xie, Chenxin An, Lei Li, Lingpeng Kong, Qi Liu, Zhifang Sui, Tong Yang","sourceUrl":"https://arxiv.org/abs/2604.06132","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"terminalBench2","benchmarkName":"Terminal-Bench 2.0","category":"agentic","score":50.8,"unit":"percent","family":"terminalBench2","index":50.03398970223154,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"swePro","benchmarkName":"SWE-bench Pro","category":"coding","score":50.7,"unit":"percent","family":"swePro","index":52.01600972014426,"aboveFit":false,"run":null,"source":"Xiang Deng, Jeff Da, Edwin Pan, Yannis Yiming He, Charles Ide, Kanak Garg, Niklas Lauffer, Andrew Park, Nitin Pasari, Chetan Rane, Karmini Sampath, Maya Krishnan, Srivatsa Kundurthy, Sean Hendryx, Zifan Wang, Chen Bo Calvin Zhang, Noah Jacobson, Bing Liu, Brad Kenstler","sourceUrl":"https://arxiv.org/abs/2509.16941","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"sciCode","benchmarkName":"Scientific Code Benchmark","category":"coding","score":48.7,"unit":"percent","family":"scicode-family","index":59.32955727290877,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gertLabs","benchmarkName":"Gert Labs Composite Game Benchmark","category":"agentic","score":45.88,"unit":"percent","family":"gertLabs","index":52.696563054655,"aboveFit":false,"run":null,"source":"Gert Labs","sourceUrl":"https://gertlabs.com/rankings","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"polyMath","benchmarkName":"PolyMath","category":"multilingual","score":43.1,"unit":"percent","family":"polyMath","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"omniscienceAccuracy","benchmarkName":"Artificial Analysis Omniscience Accuracy","category":"knowledge","score":35.2,"unit":"percent","family":"omniscienceAccuracy","index":55.050961105520265,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-simpleqa-verified:overall","benchmarkName":"SimpleQA Verified","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":34.300000000000004,"unit":"percent","family":"epoch-ai:epoch-simpleqa-verified:overall","index":52.49398025360938,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.5019206922357007,"costPerTest":null,"latency":null},{"benchmarkId":"nl2Repo","benchmarkName":"NL2Repo","category":"coding","score":32,"unit":"percent","family":"nl2Repo","index":49.34420841789851,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":30.7,"unit":"percent","family":"hle","index":55.18773158572816,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hle","benchmarkName":"Humanity's Last Exam","category":"knowledge","score":30.1,"unit":"percent","family":"hle","index":54.585755938710285,"aboveFit":false,"run":null,"source":"Center for AI Safety, Scale AI, and thousands of expert contributors","sourceUrl":"https://lastexam.ai/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mcpAtlas","benchmarkName":"MCP Atlas","category":"agentic","score":29.5,"unit":"percent","family":"mcpAtlas","index":30.92745339572655,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-2025-02-28-private:overall","benchmarkName":"FrontierMath-2025-02-28-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":27.900000000000002,"unit":"percent","family":"epoch-ai:epoch-frontiermath-2025-02-28-private:overall","index":54.54366913291133,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.6,"costPerTest":null,"latency":null},{"benchmarkId":"toolathlon","benchmarkName":"Toolathlon","category":"agentic","score":27.8,"unit":"percent","family":"toolathlon","index":44.80525766889299,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gdpvalAaNormalized","benchmarkName":"GDPval-AA normalized","category":"agentic","score":17,"unit":"percent","family":"gdpvalAaNormalized","index":46.18837986595393,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"deepPlanning","benchmarkName":"DeepPlanning","category":"agentic","score":14.4,"unit":"percent","family":"deepPlanning","index":null,"aboveFit":false,"run":null,"source":"DeepPlanning authors","sourceUrl":"https://arxiv.org/abs/2601.18137","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"researchClawBench","benchmarkName":"ResearchClawBench","category":"agentic","score":14,"unit":"percent","family":"researchClawBench","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"arc-prize:v2-public-eval:overall","benchmarkName":"ARC-AGI-2 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":12.08,"unit":"percent","family":"arc-prize:v2-public-eval:overall","index":null,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.3,"latency":null},{"benchmarkId":"arc-prize:v2-semi-private:overall","benchmarkName":"ARC-AGI-2 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":11.81,"unit":"percent","family":"arc-prize:v2-semi-private:overall","index":44.61089622077095,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.28,"latency":null},{"benchmarkId":"apexAgentsAa","benchmarkName":"APEX-Agents-AA","category":"agentic","score":11.5,"unit":"percent","family":"apexAgentsAa","index":49.9787229009987,"aboveFit":false,"run":null,"source":"Artificial Analysis / Mercor","sourceUrl":"https://artificialanalysis.ai/evaluations/apex-agents-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"jobBench","benchmarkName":"JobBench","category":"agentic","score":8.73,"unit":"percent","family":"jobBench","index":41.322143471191296,"aboveFit":false,"run":null,"source":"Yuetai Li, Yichen Feng, Zhangchen Xu, Zixian Ma, Kaiyuan Zheng, Fengqing Jiang, Xinghua Sun, Rulin Shao, Zichen Chen, Yue Huang, Xinyang Han, Brian Lee, Kayla Xu, Shenglai Zeng, Hang Hua, Xiangliang Zhang, Basel Alomair, Ranjay Krishna, Luke Zettlemoyer, Pang Wei Koh, Bhaskar Ramasubramanian, Luyao Niu, Xiang Yue, Radha Poovendran","sourceUrl":"https://arxiv.org/abs/2605.26329","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-private:overall","benchmarkName":"FrontierMath-Tier-4-2025-07-01-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":4.2,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-private:overall","index":47.46822498405367,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.9000000000000004,"costPerTest":null,"latency":null},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":3.1,"unit":"percent","family":"critpt","index":42.85035429453946,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":0.532,"outputPerMillion":3.3249999999999997,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":36,"timeToFirstToken":0.945,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":100,"benchmarkCount":55,"categoryCount":8,"confidence":"high","state":"available"},"hosts":[{"host":"deepinfra","name":"Deep Infra","kind":"host","url":"https://deepinfra.com/models","inputPerMillion":0.45,"outputPerMillion":2.25,"cacheReadPerMillion":0.07,"contextTokens":262144,"maxOutputTokens":32768,"deprecated":true},{"host":"siliconflow","name":"SiliconFlow","kind":"host","url":"https://cloud.siliconflow.com/models","inputPerMillion":0.45,"outputPerMillion":2.25,"cacheReadPerMillion":0.07,"contextTokens":262000,"maxOutputTokens":262000,"deprecated":false},{"host":"amazon-bedrock","name":"Amazon Bedrock","kind":"cloud","url":"https://docs.aws.amazon.com/bedrock/latest/userguide/models-supported.html","inputPerMillion":0.6,"outputPerMillion":3,"cacheReadPerMillion":null,"contextTokens":262144,"maxOutputTokens":16384,"deprecated":false},{"host":"azure","name":"Microsoft Azure","kind":"cloud","url":"https://learn.microsoft.com/en-us/azure/ai-services/openai/concepts/models","inputPerMillion":0.6,"outputPerMillion":3,"cacheReadPerMillion":null,"contextTokens":262144,"maxOutputTokens":262144,"deprecated":false},{"host":"novita-ai","name":"Novita AI","kind":"host","url":"https://novita.ai/docs/guides/introduction","inputPerMillion":0.6,"outputPerMillion":3,"cacheReadPerMillion":0.1,"contextTokens":262144,"maxOutputTokens":262144,"deprecated":false},{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":0.532,"outputPerMillion":3.325,"cacheReadPerMillion":0.209,"contextTokens":262144,"maxOutputTokens":65536,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:swe-bench-team","name":"SWE-bench team","url":"https://github.com","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:arc-prize-foundation","name":"ARC Prize Foundation","url":"https://arcprize.org","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"ternary-bonsai-2-27b","providerId":"prism-ml","slug":"ternary-bonsai-2-27b","name":"Ternary Bonsai 2 27B","provider":"Prism ML","description":"Ternary Bonsai 2 27B is a reasoning model from Prism ML in the Ternary Bonsai 2 family. 13 benchmarks count toward its index, in 7 categories.","logoUrl":null,"releaseDate":"2026-09-17","contextWindowTokens":262144,"modalities":["text","image"],"openWeights":true,"details":{"apiId":"prism-ml/ternary-bonsai-2-27b","maxOutputTokens":32768,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.0375,"weightsUrl":"https://huggingface.co/prism-ml/Ternary-Bonsai-2-27B-gguf"},"reasoning":true,"overallScore":52.18291238717265,"overallScoreRange":{"low":47.19816130069167,"high":57.167663473653626},"badges":[],"capabilityScores":{"agentic":54.021379817609166,"coding":52.22055830181489,"reasoning":60.126249929266535,"multimodal":50.68580648040037,"knowledge":48.57891498884666,"multilingual":null,"instruction":52.62520037833172,"math":51.1246818210513},"capabilityCounts":{"math":3,"instruction":2,"coding":2,"knowledge":2,"agentic":2,"multimodal":1,"reasoning":1},"benchmarks":[{"benchmarkId":"math500","benchmarkName":"MATH-500 Problem Set","category":"math","score":98.8,"unit":"percent","family":"math-500","index":48.534494644449424,"aboveFit":false,"run":null,"source":"Dan Hendrycks, Collin Burns, Saurav Kadavath, Akul Arora, Steven Basart, Eric Tang, Dawn Song, Jacob Steinhardt","sourceUrl":"https://arxiv.org/abs/2103.03874","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gsm8k","benchmarkName":"Grade School Math 8K","category":"math","score":96.66,"unit":"percent","family":"gsm8k","index":null,"aboveFit":false,"run":null,"source":"Karl Cobbe and colleagues (OpenAI)","sourceUrl":"https://arxiv.org/abs/2110.14168","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aime2026","benchmarkName":"AIME 2026","category":"math","score":95.83,"unit":"percent","family":"aime2026","index":54.28287650701906,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aime2025","benchmarkName":"American Invitational Mathematics Examination 2025","category":"math","score":95,"unit":"percent","family":"aime2025","index":50.55667431168541,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"ifeval","benchmarkName":"Instruction-Following Eval","category":"instruction","score":91.31,"unit":"percent","family":"ifeval","index":47.07298586721531,"aboveFit":false,"run":null,"source":"Jeffrey Zhou, Tianjian Lu, Swaroop Mishra, Siddhartha Brahma, Sujoy Basu, Yi Luan, Denny Zhou, Le Hou","sourceUrl":"https://arxiv.org/abs/2311.07911","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"liveCodeBenchV6","benchmarkName":"LiveCodeBench v6","category":"coding","score":90.07,"unit":"percent","family":"liveCodeBenchV6","index":58.36968834490966,"aboveFit":false,"run":null,"source":"LiveCodeBench maintainers","sourceUrl":"https://github.com/LiveCodeBench/LiveCodeBench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"omniDocBench16","benchmarkName":"OmniDocBench v1.6","category":"multimodal","score":89.13,"unit":"percent","family":"omniDocBench16","index":null,"aboveFit":false,"run":null,"source":"Linke Ouyang et al.","sourceUrl":"https://arxiv.org/abs/2412.07626","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmluRedux","benchmarkName":"MMLU-Redux","category":"knowledge","score":89.09,"unit":"percent","family":"mmluRedux","index":41.292000638723025,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aokvqa","benchmarkName":"A Benchmark for Visual Question Answering using World Knowledge","category":"multimodal","score":86.81,"unit":"percent","family":"aokvqa","index":null,"aboveFit":false,"run":null,"source":"Dustin Schwenk, Apoorv Khandelwal, Christopher Clark, Kenneth Marino, Roozbeh Mottaghi","sourceUrl":"https://arxiv.org/abs/2206.01718","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gpqa","benchmarkName":"Graduate-Level Google-Proof Q&A","category":"knowledge","score":85.76,"unit":"percent","family":"gpqa-diamond","index":55.8658293389703,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gpqaDiamond","benchmarkName":"GPQA Diamond","category":"knowledge","score":85.76,"unit":"percent","family":"gpqa-diamond","index":55.8658293389703,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"tau2Bench","benchmarkName":"τ²-Bench Tool-Agent-User Evaluation","category":"agentic","score":80.22,"unit":"percent","family":"tau2-bench","index":56.02127518369009,"aboveFit":false,"run":null,"source":"Victor Barres, Honghua Dong, Soham Ray, Xujie Si, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2506.07982","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"realWorldQa","benchmarkName":"RealWorldQA","category":"multimodal","score":80.13,"unit":"percent","family":"realWorldQa","index":50.68580648040037,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"charxivOverall","benchmarkName":"CharXiv Descriptive and Reasoning Combined","category":"multimodal","score":80,"unit":"percent","family":"charxivOverall","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":77,"unit":"percent","family":"lcr","index":60.126249929266535,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"bfclV3","benchmarkName":"Berkeley Function Calling Leaderboard v3","category":"agentic","score":74.92,"unit":"percent","family":"bfclV3","index":null,"aboveFit":false,"run":null,"source":"Shishir G. Patil et al.","sourceUrl":"https://gorilla.cs.berkeley.edu/leaderboard.html","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"ifBench","benchmarkName":"Instruction Following Benchmark","category":"instruction","score":74,"unit":"percent","family":"ifbench-family","index":58.17741488944813,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"sweVerified","benchmarkName":"Software Engineering Benchmark Verified","category":"coding","score":60.8,"unit":"percent","family":"swe-bench-verified","index":46.07142825872012,"aboveFit":false,"run":null,"source":"Carlos E. Jimenez, John Yang, Alexander Wettig, Shunyu Yao, Kexin Pei, Ofir Press, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2310.06770","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"bigCodeBench","benchmarkName":"BigCodeBench","category":"coding","score":58.07,"unit":"percent","family":"bigCodeBench","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"ocrBenchV2","benchmarkName":"OCRBench V2","category":"multimodal","score":56.88,"unit":"percent","family":"ocrBenchV2","index":null,"aboveFit":false,"run":null,"source":"OCRBench authors","sourceUrl":"https://arxiv.org/abs/2501.00321","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"terminalBench21","benchmarkName":"Terminal-Bench 2.1","category":"agentic","score":52.8,"unit":"percent","family":"terminal-bench-2-1","index":52.02148445152824,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":0.075,"outputPerMillion":0.5,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":21,"timeToFirstToken":1.501,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":95,"benchmarkCount":13,"categoryCount":7,"confidence":"medium","state":"available"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"solar-pro-4","providerId":"upstage","slug":"solar-pro-4","name":"Solar Pro 4","provider":"Upstage","description":"Solar Pro 4 is a reasoning model from Upstage in the Solar family. 16 benchmarks count toward its index, in 5 categories.","logoUrl":null,"releaseDate":"2026-08-11","contextWindowTokens":524288,"modalities":["text"],"openWeights":false,"details":{"apiId":"upstage/solar-pro4","maxOutputTokens":131072,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.03,"weightsUrl":null},"reasoning":true,"overallScore":52.13826779243485,"overallScoreRange":{"low":46.095767792434856,"high":58.18076779243485},"badges":[],"capabilityScores":{"agentic":49.874544028062665,"coding":57.97050413378601,"reasoning":52.527473841864236,"multimodal":null,"knowledge":50.06378867761493,"multilingual":null,"instruction":null,"math":53.901308033618},"capabilityCounts":{"math":1,"knowledge":4,"coding":3,"reasoning":2,"agentic":6},"benchmarks":[{"benchmarkId":"aime2026","benchmarkName":"AIME 2026","category":"math","score":95.3,"unit":"percent","family":"aime2026","index":53.901308033618,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":89.1,"unit":"percent","family":"gpqa-diamond","index":58.929186073202374,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gpqaDiamond","benchmarkName":"GPQA Diamond","category":"knowledge","score":89,"unit":"percent","family":"gpqa-diamond","index":58.8374688057104,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"liveCodeBench","benchmarkName":"LiveCodeBench: Holistic and Contamination Free Evaluation of Large Language Models for Code","category":"coding","score":87.8,"unit":"percent","family":"livecodebench","index":62.19504640993927,"aboveFit":false,"run":null,"source":"Naman Jain, King Han, Alex Gu, Wen-Ding Li, Fanjia Yan, Tianjun Zhang, Sida Wang, Armando Solar-Lezama, Koushik Sen, Ion Stoica","sourceUrl":"https://arxiv.org/abs/2403.07974","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmluPro","benchmarkName":"Massive Multitask Language Understanding Professional","category":"knowledge","score":86.3,"unit":"percent","family":"mmlu-pro","index":53.20839858248194,"aboveFit":false,"run":null,"source":"Yubo Wang, Xueguang Ma, Ge Zhang, Yuansheng Ni, Abhranil Chandra, Shiguang Guo, Weiming Ren, Aaran Arulraj, Xuan He, Ziyan Jiang, Tianle Li, Max Ku, Kai Wang, Alex Zhuang, Rongqi Fan, Xiang Yue, Wenhu Chen","sourceUrl":"https://arxiv.org/abs/2406.01574","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":71,"unit":"percent","family":"lcr","index":56.00976677575258,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"sweVerified","benchmarkName":"Software Engineering Benchmark Verified","category":"coding","score":70.6,"unit":"percent","family":"swe-bench-verified","index":53.299492521000296,"aboveFit":false,"run":null,"source":"Carlos E. Jimenez, John Yang, Alexander Wettig, Shunyu Yao, Kexin Pei, Ofir Press, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2310.06770","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mcpAtlas","benchmarkName":"MCP Atlas","category":"agentic","score":61.4,"unit":"percent","family":"mcpAtlas","index":52.408772910416396,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"terminalBench21","benchmarkName":"Terminal-Bench 2.1","category":"agentic","score":57,"unit":"percent","family":"terminal-bench-2-1","index":54.47787698231474,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"browseComp","benchmarkName":"BrowseComp","category":"agentic","score":49.2,"unit":"percent","family":"browseComp","index":38.77796850576406,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaSciCode","benchmarkName":"Artificial Analysis SciCode","category":"coding","score":48,"unit":"percent","family":"scicode-family","index":58.416973470418455,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/scicode","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gdpvalAaNormalized","benchmarkName":"GDPval-AA normalized","category":"agentic","score":30.3,"unit":"percent","family":"gdpvalAaNormalized","index":55.697570796455295,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":29.8,"unit":"percent","family":"hle","index":54.28476811520135,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"apexAgents","benchmarkName":"APEX-Agents","category":"agentic","score":18.7,"unit":"percent","family":"apexAgents","index":52.366477820787566,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"omniscienceAccuracy","benchmarkName":"Artificial Analysis Omniscience Accuracy","category":"knowledge","score":18.4,"unit":"percent","family":"omniscienceAccuracy","index":33.87866057332005,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":6.3,"unit":"percent","family":"critpt","index":49.0451809079759,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lmarena:agent-steerability:overall","benchmarkName":"Agent Arena steerability","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":-20.791264402325748,"unit":"index","family":"agent-arena","index":49.43712249780023,"aboveFit":false,"run":null,"source":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset","measuredAt":"2026-11-11","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lmarena:agent-bash-recovery:overall","benchmarkName":"Agent Arena command recovery","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":-25.557417868208244,"unit":"index","family":"agent-arena","index":45.51859715263793,"aboveFit":false,"run":null,"source":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset","measuredAt":"2026-10-08","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lmarena:agent-task-outcome:overall","benchmarkName":"Agent Arena task outcome","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":-29.94469728011657,"unit":"index","family":"agent-arena","index":41.91156567643087,"aboveFit":false,"run":null,"source":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset","measuredAt":"2026-11-11","stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":0.15,"outputPerMillion":0.6,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":44,"timeToFirstToken":2.9515,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":{"elo":1388.1968381331894,"eloUpper":1393.2222702989407,"eloLower":1383.1714059674382,"votes":27104,"rank":174,"rankUpper":null,"rankLower":null,"arenaName":"Solar Pro 4","organization":"upstage","updatedAt":"2026-11-09"},"quality":{"coveragePercent":80,"benchmarkCount":16,"categoryCount":5,"confidence":"low","state":"available"},"hosts":[{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":0.15,"outputPerMillion":0.6,"cacheReadPerMillion":0.03,"contextTokens":524288,"maxOutputTokens":131072,"deprecated":false},{"host":"upstage","name":"Upstage","kind":"maker","url":"https://developers.upstage.ai/docs/apis/chat","inputPerMillion":0.3,"outputPerMillion":1.2,"cacheReadPerMillion":0.06,"contextTokens":524288,"maxOutputTokens":131072,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:lmarena","name":"LMArena","url":"https://huggingface.co","license":"CC BY 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"qwen3-6-27b","providerId":"alibaba","slug":"qwen3-6-27b","name":"Qwen3.6-27B","provider":"Alibaba","description":"Qwen3.6-27B is a reasoning model from Alibaba. 47 benchmarks count toward its index, in 8 categories.","logoUrl":null,"releaseDate":"2026-04-21","contextWindowTokens":262144,"modalities":["text","image","video"],"openWeights":true,"details":{"apiId":"qwen/qwen3.6-27b","maxOutputTokens":81920,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":null,"weightsUrl":"https://huggingface.co/Qwen/Qwen3.6-27B"},"reasoning":true,"overallScore":52.096201607078896,"overallScoreRange":{"low":49.73756469455794,"high":54.45483851959985},"badges":[],"capabilityScores":{"agentic":56.677932867721665,"coding":52.78585883741575,"reasoning":49.04320356087081,"multimodal":53.431544180695184,"knowledge":48.176209797399444,"multilingual":60.88321060917342,"instruction":43.24423604519212,"math":50.99449780050646},"capabilityCounts":{"multimodal":9,"agentic":9,"math":7,"knowledge":8,"coding":7,"reasoning":4,"instruction":2,"multilingual":1},"benchmarks":[{"benchmarkId":"countBench","benchmarkName":"CountBench","category":"multimodal","score":97.8,"unit":"percent","family":"countBench","index":54.33644059624182,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vStar","benchmarkName":"V*","category":"multimodal","score":94.7,"unit":"percent","family":"vStar","index":54.998438521726726,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"tau2Bench","benchmarkName":"τ²-Bench Tool-Agent-User Evaluation","category":"agentic","score":94.2,"unit":"percent","family":"tau2-bench","index":65.29802728835415,"aboveFit":false,"run":null,"source":"Victor Barres, Honghua Dong, Soham Ray, Xujie Si, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2506.07982","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aime2026","benchmarkName":"AIME 2026","category":"math","score":94.1,"unit":"percent","family":"aime2026","index":53.037379414596735,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hmmtFeb2025","benchmarkName":"Harvard-MIT Mathematics Tournament February 2025","category":"math","score":93.8,"unit":"percent","family":"hmmtFeb2025","index":51.82156255858652,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmluRedux","benchmarkName":"MMLU-Redux","category":"knowledge","score":93.5,"unit":"percent","family":"mmluRedux","index":48.772915675901416,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"refcocoAvg","benchmarkName":"RefCOCO average","category":"multimodal","score":92.5,"unit":"percent","family":"refcocoAvg","index":52.30392086368822,"aboveFit":false,"run":null,"source":"RefCOCO dataset authors","sourceUrl":"https://github.com/lichengunc/refer","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"cEval","benchmarkName":"C-Eval","category":"knowledge","score":91.4,"unit":"percent","family":"cEval","index":47.67650382094591,"aboveFit":false,"run":null,"source":"C-Eval authors","sourceUrl":"https://arxiv.org/abs/2305.08322","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hmmtNov2025","benchmarkName":"Harvard-MIT Mathematics Tournament November 2025","category":"math","score":90.7,"unit":"percent","family":"hmmtNov2025","index":52.10353576546395,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gpqa","benchmarkName":"Graduate-Level Google-Proof Q&A","category":"knowledge","score":87.8,"unit":"percent","family":"gpqa-diamond","index":57.73686159580665,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"videoMmeWithSub","benchmarkName":"Video-MME with subtitle","category":"multimodal","score":87.7,"unit":"percent","family":"videoMmeWithSub","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mlvuAvg","benchmarkName":"MLVU mean average","category":"multimodal","score":86.6,"unit":"percent","family":"mlvuAvg","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmluPro","benchmarkName":"Massive Multitask Language Understanding Professional","category":"knowledge","score":86.2,"unit":"percent","family":"mmlu-pro","index":53.08241055217426,"aboveFit":false,"run":null,"source":"Yubo Wang, Xueguang Ma, Ge Zhang, Yuansheng Ni, Abhranil Chandra, Shiguang Guo, Weiming Ren, Aaran Arulraj, Xuan He, Ziyan Jiang, Tianle Li, Max Ku, Kai Wang, Alex Zhuang, Rongqi Fan, Xiang Yue, Wenhu Chen","sourceUrl":"https://arxiv.org/abs/2406.01574","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"dynaMath","benchmarkName":"DynaMath","category":"multimodal","score":85.6,"unit":"percent","family":"dynaMath","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":84.84848484848484,"unit":"percent","family":"gpqa-diamond","index":55.02981254922523,"aboveFit":false,"run":"none effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.554565042660364,"costPerTest":null,"latency":null},{"benchmarkId":"videoMmmu","benchmarkName":"VideoMMMU","category":"multimodal","score":84.4,"unit":"percent","family":"videoMmmu","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hmmtFeb2026","benchmarkName":"Harvard-MIT Mathematics Tournament February 2026","category":"math","score":84.3,"unit":"percent","family":"hmmtFeb2026","index":52.39542965711443,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":84.2,"unit":"percent","family":"gpqa-diamond","index":54.43503996609543,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"realWorldQa","benchmarkName":"RealWorldQA","category":"multimodal","score":84.1,"unit":"percent","family":"realWorldQa","index":57.57790550200055,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"liveCodeBenchV6","benchmarkName":"LiveCodeBench v6","category":"coding","score":83.9,"unit":"percent","family":"liveCodeBenchV6","index":53.3037657475881,"aboveFit":false,"run":null,"source":"LiveCodeBench maintainers","sourceUrl":"https://github.com/LiveCodeBench/LiveCodeBench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmmu","benchmarkName":"Massive Multi-discipline Multimodal Understanding","category":"multimodal","score":82.9,"unit":"percent","family":"mmmu","index":51.82100965549977,"aboveFit":false,"run":null,"source":"MMMU authors","sourceUrl":"https://arxiv.org/abs/2401.05508","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mStar","benchmarkName":"MStar","category":"multimodal","score":81.4,"unit":"percent","family":"mStar","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"ccOcr","benchmarkName":"CC-OCR","category":"multimodal","score":81.2,"unit":"percent","family":"ccOcr","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmAnswerBench","benchmarkName":"MMAnswerBench","category":"math","score":80.8,"unit":"percent","family":"mmAnswerBench","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-math:overall","benchmarkName":"LiveBench Mathematics","category":"math","domain":null,"eligible":true,"ranked":true,"score":79.8685,"unit":"percent","family":"livebench:livebench-math:overall","index":51.08909364510234,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Mathematics","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.12277440633245383,"latency":null},{"benchmarkId":"charxiv","benchmarkName":"CharXiv Reasoning","category":"multimodal","score":78.4,"unit":"percent","family":"charxiv","index":53.44247759582547,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":77.3,"unit":"percent","family":"lcr","index":60.332074086942235,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"sweVerified","benchmarkName":"Software Engineering Benchmark Verified","category":"coding","score":77.2,"unit":"percent","family":"swe-bench-verified","index":58.16737253437267,"aboveFit":false,"run":null,"source":"Carlos E. Jimenez, John Yang, Alexander Wettig, Shunyu Yao, Kexin Pei, Ofir Press, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2310.06770","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmmuPro","benchmarkName":"Massive Multi-discipline Multimodal Understanding Pro","category":"multimodal","score":75.8,"unit":"percent","family":"mmmu-pro","index":53.799864723868396,"aboveFit":false,"run":null,"source":"MMMU-Pro authors","sourceUrl":"https://arxiv.org/abs/2409.02813","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaMmmuPro","benchmarkName":"Artificial Analysis MMMU-Pro","category":"multimodal","score":74.6,"unit":"percent","family":"mmmu-pro","index":52.238170005725394,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmmu-pro","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"clawEval","benchmarkName":"Claw-Eval","category":"agentic","score":72.4,"unit":"percent","family":"clawEval","index":69.82242580719725,"aboveFit":true,"run":null,"source":"Bowen Ye, Rang Li, Qibin Yang, Yuanxin Liu, Linli Yao, Hanglong Lv, Zhihui Xie, Chenxin An, Lei Li, Lingpeng Kong, Qi Liu, Zhifang Sui, Tong Yang","sourceUrl":"https://arxiv.org/abs/2604.06132","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-coding:overall","benchmarkName":"LiveBench Coding","category":"coding","domain":null,"eligible":true,"ranked":true,"score":71.785,"unit":"percent","family":"livebench:livebench-coding:overall","index":55.783411081020255,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Coding","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.019827350427350426,"latency":null},{"benchmarkId":"vals-ai:tax_eval_v2:overall","benchmarkName":"TaxEval v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":71.26,"unit":"percent","family":"vals-ai:tax_eval_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","measuredAt":"2026-09-01","stderr":0.882,"costPerTest":0.030256,"latency":147.778},{"benchmarkId":"livebench:livebench-data-analysis:overall","benchmarkName":"LiveBench Data Analysis","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":70.42666666666666,"unit":"percent","family":"livebench","index":52.85281957070089,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Data%20Analysis","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.08841333333333333,"latency":null},{"benchmarkId":"androidWorld","benchmarkName":"AndroidWorld","category":"agentic","score":70.3,"unit":"percent","family":"androidWorld","index":52.39542965711443,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-reasoning:overall","benchmarkName":"LiveBench Reasoning","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":70.28375,"unit":"percent","family":"livebench","index":52.667680462084945,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Reasoning","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.13343960396039606,"latency":null},{"benchmarkId":"vals-ai:swebench:overall","benchmarkName":"SWE-bench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":70,"unit":"percent","family":"swe-bench-verified","index":52.85695797433008,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/swebench","measuredAt":"2026-09-01","stderr":2.051,"costPerTest":0.91414,"latency":1028.733},{"benchmarkId":"vals-ai:mortgage_tax:overall","benchmarkName":"MortgageTax","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":68.283,"unit":"percent","family":"vals-ai:mortgage_tax:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mortgage_tax","measuredAt":"2026-09-01","stderr":0.916,"costPerTest":0.016906,"latency":42.074},{"benchmarkId":"aaIfBench","benchmarkName":"Artificial Analysis IFBench","category":"instruction","score":67.6,"unit":"percent","family":"ifbench-family","index":52.1965469545393,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":66.66666666666666,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":47.7053605921038,"aboveFit":false,"run":"none effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":7.106690545187015,"costPerTest":null,"latency":null},{"benchmarkId":"superGpqa","benchmarkName":"SuperGPQA: Scaling LLM Evaluation Across 285 Graduate Disciplines","category":"knowledge","score":66,"unit":"percent","family":"superGpqa","index":51.29898922141316,"aboveFit":false,"run":null,"source":"Xiaoxuan Du, Yao Yao, Kexin Ma, Bowen Wang, Tianyu Zheng, Kaiyan Zhu, Yiming Zhang, Yutao Zhu, Jiawei Zhou, Jingren Zhou","sourceUrl":"https://arxiv.org/abs/2502.14739","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":65.34857142857142,"unit":"percent","family":"euroeval","index":60.88321060917342,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.4703841244023823,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-language:overall","benchmarkName":"LiveBench Language","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":63.30433333333334,"unit":"percent","family":"livebench:livebench-language:overall","index":46.14389034657894,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Language","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.047503333333333335,"latency":null},{"benchmarkId":"erqa","benchmarkName":"ERQA","category":"multimodal","score":62.5,"unit":"percent","family":"erqa","index":56.042658522545516,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:corp_fin_v2:overall","benchmarkName":"CorpFin v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":62.315,"unit":"percent","family":"vals-ai:corp_fin_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","measuredAt":"2026-08-12","stderr":0.954,"costPerTest":0.053549,"latency":62.759},{"benchmarkId":"terminalBench2","benchmarkName":"Terminal-Bench 2.0","category":"agentic","score":59.3,"unit":"percent","family":"terminalBench2","index":56.08042645538355,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"simpleVqa","benchmarkName":"SimpleVQA","category":"multimodal","score":56.1,"unit":"percent","family":"simpleVqa","index":47.34202900393171,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gertLabs","benchmarkName":"Gert Labs Composite Game Benchmark","category":"agentic","score":54.84,"unit":"percent","family":"gertLabs","index":59.951625461265536,"aboveFit":false,"run":null,"source":"Gert Labs","sourceUrl":"https://gertlabs.com/rankings","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"swePro","benchmarkName":"SWE-bench Pro","category":"coding","score":53.5,"unit":"percent","family":"swePro","index":54.40051766747061,"aboveFit":false,"run":null,"source":"Xiang Deng, Jeff Da, Edwin Pan, Yannis Yiming He, Charles Ide, Kanak Garg, Niklas Lauffer, Andrew Park, Nitin Pasari, Chetan Rane, Karmini Sampath, Maya Krishnan, Srivatsa Kundurthy, Sean Hendryx, Zifan Wang, Chen Bo Calvin Zhang, Noah Jacobson, Bing Liu, Brad Kenstler","sourceUrl":"https://arxiv.org/abs/2509.16941","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"qwenClawBench","benchmarkName":"QwenClawBench","category":"agentic","score":53.4,"unit":"percent","family":"qwenClawBench","index":51.8486498501477,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-instruction-following:overall","benchmarkName":"LiveBench Instruction Following","category":"instruction","domain":null,"eligible":true,"ranked":true,"score":53.229,"unit":"percent","family":"livebench:livebench-instruction-following:overall","index":34.29192513584493,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=IF","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.04636,"latency":null},{"benchmarkId":"vals-ai:case_law_v2:overall","benchmarkName":"CaseLaw v2","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":53.162,"unit":"percent","family":"vals-ai:case_law_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/case_law_v2","measuredAt":"2026-05-04","stderr":2.168,"costPerTest":0.023877,"latency":42.493},{"benchmarkId":"vals-ai:terminal-bench-2:overall","benchmarkName":"Terminal-Bench 2.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":44.944,"unit":"percent","family":"vals-ai:terminal-bench-2:overall","index":53.61255278791465,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2","measuredAt":"2026-06-04","stderr":5.303,"costPerTest":0.298145,"latency":2007.631},{"benchmarkId":"aaSciCode","benchmarkName":"Artificial Analysis SciCode","category":"coding","score":42.8,"unit":"percent","family":"scicode-family","index":51.63777950906184,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/scicode","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-agentic-coding:overall","benchmarkName":"LiveBench Agentic Coding","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":39.29266666666667,"unit":"percent","family":"livebench:livebench-agentic-coding:overall","index":49.6845482758134,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Agentic%20Coding","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.8130222222222222,"latency":null},{"benchmarkId":"nl2Repo","benchmarkName":"NL2Repo","category":"coding","score":36.2,"unit":"percent","family":"nl2Repo","index":52.305349604276174,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tiers-1-3-v2-private:overall","benchmarkName":"FrontierMath-Tiers-1-3-v2-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":34.03508771929825,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tiers-1-3-v2-private:overall","index":48.80912297057741,"aboveFit":false,"run":"none effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.8116467881852296,"costPerTest":null,"latency":null},{"benchmarkId":"gdpvalAaNormalized","benchmarkName":"GDPval-AA normalized","category":"agentic","score":24.3,"unit":"percent","family":"gdpvalAaNormalized","index":51.4077102263043,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hle","benchmarkName":"Humanity's Last Exam","category":"knowledge","score":24,"unit":"percent","family":"hle","index":48.46567019402854,"aboveFit":false,"run":null,"source":"Center for AI Safety, Scale AI, and thousands of expert contributors","sourceUrl":"https://lastexam.ai/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":23.1,"unit":"percent","family":"hle","index":47.56270672350173,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"omniscienceAccuracy","benchmarkName":"Artificial Analysis Omniscience Accuracy","category":"knowledge","score":19.6,"unit":"percent","family":"omniscienceAccuracy","index":35.39096775419149,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:vibe-code:overall","benchmarkName":"Vibe Code Bench v1.1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":11.941,"unit":"percent","family":"vals-ai:vibe-code:overall","index":46.558022998141915,"aboveFit":false,"run":"OpenHands","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","measuredAt":"2026-11-09","stderr":3.414,"costPerTest":4.655127,"latency":9762.856},{"benchmarkId":"epoch-ai:epoch-mystery-game-puzzles:overall","benchmarkName":"Mystery Game Puzzles","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":7.000000000000001,"unit":"percent","family":"epoch-ai:epoch-mystery-game-puzzles:overall","index":44.1019024790064,"aboveFit":false,"run":"none effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.564323999762428,"costPerTest":null,"latency":null},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":1.1,"unit":"percent","family":"critpt","index":38.97858766114168,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":0.32,"outputPerMillion":3.1999999999999997,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":60,"timeToFirstToken":0.6175,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":100,"benchmarkCount":47,"categoryCount":8,"confidence":"high","state":"available"},"hosts":[{"host":"stackit","name":"STACKIT","kind":"cloud","url":"https://docs.stackit.cloud/products/data-and-ai/ai-model-serving/basics/available-shared-models","inputPerMillion":0.53,"outputPerMillion":0.76,"cacheReadPerMillion":null,"contextTokens":262144,"maxOutputTokens":16384,"deprecated":true},{"host":"chutes","name":"Chutes","kind":"host","url":"https://llm.chutes.ai/v1/models","inputPerMillion":0.25,"outputPerMillion":2,"cacheReadPerMillion":0.025,"contextTokens":262144,"maxOutputTokens":65536,"deprecated":false},{"host":"groq","name":"Groq","kind":"host","url":"https://console.groq.com/docs/models","inputPerMillion":0.6,"outputPerMillion":3,"cacheReadPerMillion":0.3,"contextTokens":131072,"maxOutputTokens":16384,"deprecated":true},{"host":"ovhcloud","name":"OVHcloud","kind":"cloud","url":"https://www.ovhcloud.com/en/public-cloud/ai-endpoints/catalog//","inputPerMillion":0.47,"outputPerMillion":3.19,"cacheReadPerMillion":null,"contextTokens":262144,"maxOutputTokens":262144,"deprecated":false},{"host":"siliconflow","name":"SiliconFlow","kind":"host","url":"https://cloud.siliconflow.com/models","inputPerMillion":0.3,"outputPerMillion":3.2,"cacheReadPerMillion":null,"contextTokens":262144,"maxOutputTokens":262144,"deprecated":false},{"host":"deepinfra","name":"Deep Infra","kind":"host","url":"https://deepinfra.com/models","inputPerMillion":0.32,"outputPerMillion":3.2,"cacheReadPerMillion":null,"contextTokens":262144,"maxOutputTokens":65536,"deprecated":false},{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":0.32,"outputPerMillion":3.2,"cacheReadPerMillion":null,"contextTokens":262144,"maxOutputTokens":81920,"deprecated":false},{"host":"novita-ai","name":"Novita AI","kind":"host","url":"https://novita.ai/docs/guides/introduction","inputPerMillion":0.6,"outputPerMillion":3.6,"cacheReadPerMillion":null,"contextTokens":262144,"maxOutputTokens":65536,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:livebench","name":"LiveBench","url":"https://livebench.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gemini-3-5-flash-lite","providerId":"google","slug":"gemini-3-5-flash-lite","name":"Gemini 3.5 Flash-Lite","provider":"Google","description":"Gemini 3.5 Flash-Lite is a reasoning model from Google. 33 benchmarks count toward its index, in 8 categories.","logoUrl":null,"releaseDate":"2026-07-21","contextWindowTokens":1048576,"modalities":["text","image","video","file","audio"],"openWeights":false,"details":{"apiId":"google/gemini-3.5-flash-lite","maxOutputTokens":65536,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.03,"weightsUrl":null},"reasoning":true,"overallScore":52.041108672250324,"overallScoreRange":{"low":49.22627297544669,"high":54.85594436905396},"badges":[],"capabilityScores":{"agentic":54.7483811447483,"coding":54.40167814653302,"reasoning":47.66528958465619,"multimodal":58.834446383525474,"knowledge":49.96022644039781,"multilingual":54.23471502659859,"instruction":62.63806860287544,"math":46.47194420424875},"capabilityCounts":{"knowledge":5,"multimodal":2,"coding":8,"reasoning":8,"agentic":4,"math":4,"instruction":1,"multilingual":1},"benchmarks":[{"benchmarkId":"vals-ai:mmlu_pro:overall","benchmarkName":"MMLU Pro","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":85.836,"unit":"percent","family":"mmlu-pro","index":52.623814121854274,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","measuredAt":"2026-09-01","stderr":0.344,"costPerTest":0.004522,"latency":4.823},{"benchmarkId":"vals-ai:legal_bench:overall","benchmarkName":"LegalBench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":84.073,"unit":"percent","family":"vals-ai:legal_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","measuredAt":"2026-10-13","stderr":0.437,"costPerTest":0.00128,"latency":1.666},{"benchmarkId":"vals-ai:gpqa:overall","benchmarkName":"GPQA Diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":83.838,"unit":"percent","family":"gpqa-diamond","index":54.10302345777446,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","measuredAt":"2026-09-01","stderr":1.847,"costPerTest":0.01235,"latency":13.848},{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":83.8,"unit":"percent","family":"gpqa-diamond","index":54.06817089612751,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:mmmu:overall","benchmarkName":"MMMU Pro","category":"multimodal","domain":null,"eligible":true,"ranked":true,"score":83.584,"unit":"percent","family":"mmmu-pro","index":63.93005779555598,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmmu","measuredAt":"2026-09-01","stderr":0.891,"costPerTest":0.006577,"latency":7.799},{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":83.33333333333334,"unit":"percent","family":"gpqa-diamond","index":53.64015698116496,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.6552207828215257,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:sage:overall","benchmarkName":"SAGE","category":null,"domain":"Education","eligible":false,"ranked":true,"score":79.516,"unit":"percent","family":"vals-ai:sage:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/sage","measuredAt":"2026-11-05","stderr":2.042,"costPerTest":0.009537,"latency":9.78},{"benchmarkId":"vals-ai:lcb:overall","benchmarkName":"LiveCodeBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":79.014,"unit":"percent","family":"livecodebench","index":54.90493997192121,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","measuredAt":"2026-09-01","stderr":1.098,"costPerTest":0.019272,"latency":21.084},{"benchmarkId":"aaMmmuPro","benchmarkName":"Artificial Analysis MMMU-Pro","category":"multimodal","score":79,"unit":"percent","family":"mmmu-pro","index":57.96438397224972,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmmu-pro","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-coding:overall","benchmarkName":"LiveBench Coding","category":"coding","domain":null,"eligible":true,"ranked":true,"score":76.0715,"unit":"percent","family":"livebench:livebench-coding:overall","index":61.81189151740449,"aboveFit":false,"run":"high effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Coding","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.010346153846153847,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":76,"unit":"percent","family":"lcr","index":59.44016940368088,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:swebench:overall","benchmarkName":"SWE-bench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":75,"unit":"percent","family":"swe-bench-verified","index":56.544745863248544,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/swebench","measuredAt":"2026-09-01","stderr":1.938,"costPerTest":0.30898,"latency":439.669},{"benchmarkId":"osWorldVerified","benchmarkName":"OSWorld-Verified","category":"agentic","score":74,"unit":"percent","family":"osWorldVerified","index":60.21803312533277,"aboveFit":false,"run":null,"source":"Tianbao Xie, Danyang Zhang, Jixuan Chen, Xiaochuan Li, Siheng Zhao, Ruisheng Cao, Toh Jing Hua, Zhoujun Cheng, Dongchan Shin, Fangyu Lei, Yitao Liu, Yiheng Xu, Shuyan Zhou, Silvio Savarese, Caiming Xiong, Victor Zhong, Tao Yu","sourceUrl":"https://os-world.github.io/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-math:overall","benchmarkName":"LiveBench Mathematics","category":"math","domain":null,"eligible":true,"ranked":true,"score":73.73949999999999,"unit":"percent","family":"livebench:livebench-math:overall","index":42.54698809528008,"aboveFit":false,"run":"high effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Mathematics","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.02781108179419525,"latency":null},{"benchmarkId":"vals-ai:tax_eval_v2:overall","benchmarkName":"TaxEval v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":72.608,"unit":"percent","family":"vals-ai:tax_eval_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","measuredAt":"2026-09-01","stderr":0.875,"costPerTest":0.00729,"latency":8.815},{"benchmarkId":"mrcrv2","benchmarkName":"MRCRv2","category":"reasoning","score":72.2,"unit":"percent","family":"mrcrv2","index":54.92368857683304,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-language:overall","benchmarkName":"LiveBench Language","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":71.82033333333334,"unit":"percent","family":"livebench:livebench-language:overall","index":55.77386521993136,"aboveFit":false,"run":"high effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Language","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.016439333333333334,"latency":null},{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":71.11111111111111,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":50.291111906473084,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":6.83294324254051,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:medscribe:overall","benchmarkName":"MedScribe","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":70.886,"unit":"percent","family":"vals-ai:medscribe:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medscribe","measuredAt":"2026-10-29","stderr":2.031,"costPerTest":0.019867,"latency":18.991},{"benchmarkId":"vals-ai:mortgage_tax:overall","benchmarkName":"MortgageTax","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":68.68,"unit":"percent","family":"vals-ai:mortgage_tax:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mortgage_tax","measuredAt":"2026-09-01","stderr":0.914,"costPerTest":0.004323,"latency":4.879},{"benchmarkId":"livebench:livebench-instruction-following:overall","benchmarkName":"LiveBench Instruction Following","category":"instruction","domain":null,"eligible":true,"ranked":true,"score":67.23774999999999,"unit":"percent","family":"livebench:livebench-instruction-following:overall","index":62.63806860287544,"aboveFit":false,"run":"high effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=IF","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.0233355,"latency":null},{"benchmarkId":"arc-prize:v1-public-eval:overall","benchmarkName":"ARC-AGI-1 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":66.375,"unit":"percent","family":"arc-prize:v1-public-eval:overall","index":null,"aboveFit":false,"run":"high effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.07732991824999999,"latency":null},{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":63.01875000000001,"unit":"percent","family":"euroeval","index":57.722179007240086,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":0.6353345575364211,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-french:overall","benchmarkName":"EuroEval French","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":61.628750000000004,"unit":"percent","family":"euroeval","index":55.83626883712822,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/french","measuredAt":null,"stderr":0.32184769301643285,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-italian:overall","benchmarkName":"EuroEval Italian","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":61.5575,"unit":"percent","family":"euroeval","index":55.73959898128615,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/italian","measuredAt":null,"stderr":0.5402111855561674,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-portuguese:overall","benchmarkName":"EuroEval Portuguese","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":61.32,"unit":"percent","family":"euroeval","index":55.41736612847927,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/portuguese","measuredAt":null,"stderr":0.4789376264191403,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:corp_fin_v2:overall","benchmarkName":"CorpFin v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":60.684,"unit":"percent","family":"vals-ai:corp_fin_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","measuredAt":"2026-08-12","stderr":0.962,"costPerTest":0.028548,"latency":3.836},{"benchmarkId":"livebench:livebench-reasoning:overall","benchmarkName":"LiveBench Reasoning","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":60.1875,"unit":"percent","family":"livebench","index":39.58865497849027,"aboveFit":false,"run":"high effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Reasoning","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.031312871287128716,"latency":null},{"benchmarkId":"euroeval:euroeval-dutch:overall","benchmarkName":"EuroEval Dutch","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":59.576666666666675,"unit":"percent","family":"euroeval","index":53.052063924717906,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/dutch","measuredAt":null,"stderr":0.5209535416835099,"costPerTest":null,"latency":null},{"benchmarkId":"parseBenchLlamaIndex","benchmarkName":"ParseBench: document parsing for AI agents (LlamaIndex leaderboard)","category":"multimodal","score":59.49,"unit":"percent","family":"parseBenchLlamaIndex","index":56.721671883148105,"aboveFit":false,"run":null,"source":"Boyang Zhang, Sebastián G. Acosta, Preston Carlson, Sacha Bron, Pierre-Loïc Doulcet, Daniel B. Ospina and Simon Suo (LlamaIndex)","sourceUrl":"https://arxiv.org/abs/2604.08538","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:cyber:overall","benchmarkName":"CyberBench v1.1","category":null,"domain":"Other","eligible":false,"ranked":true,"score":59.107,"unit":"percent","family":"vals-ai:cyber:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/cyber","measuredAt":"2026-11-05","stderr":6.17,"costPerTest":0.425124,"latency":1075.887},{"benchmarkId":"euroeval:euroeval-spanish:overall","benchmarkName":"EuroEval Spanish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":57.15,"unit":"percent","family":"euroeval","index":49.759635618143726,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/spanish","measuredAt":null,"stderr":0.48972090265783014,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":54.32000000000001,"unit":"percent","family":"euroeval","index":45.91997678259228,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.4906430599171846,"costPerTest":null,"latency":null},{"benchmarkId":"swePro","benchmarkName":"SWE-bench Pro","category":"coding","score":54.2,"unit":"percent","family":"swePro","index":54.9966446543022,"aboveFit":false,"run":null,"source":"Xiang Deng, Jeff Da, Edwin Pan, Yannis Yiming He, Charles Ide, Kanak Garg, Niklas Lauffer, Andrew Park, Nitin Pasari, Chetan Rane, Karmini Sampath, Maya Krishnan, Srivatsa Kundurthy, Sean Hendryx, Zifan Wang, Chen Bo Calvin Zhang, Noah Jacobson, Bing Liu, Brad Kenstler","sourceUrl":"https://arxiv.org/abs/2509.16941","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"terminalBench21","benchmarkName":"Terminal-Bench 2.1","category":"agentic","score":54,"unit":"percent","family":"terminal-bench-2-1","index":52.72331088889581,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"arc-prize:v1-semi-private:overall","benchmarkName":"ARC-AGI-1 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":53.5,"unit":"percent","family":"arc-prize:v1-semi-private:overall","index":51.58324036932101,"aboveFit":false,"run":"high effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.092850542,"latency":null},{"benchmarkId":"livebench:livebench-data-analysis:overall","benchmarkName":"LiveBench Data Analysis","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":53.24966666666666,"unit":"percent","family":"livebench","index":30.601149818891482,"aboveFit":false,"run":"high effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Data%20Analysis","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.04497,"latency":null},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":52.43124999999999,"unit":"percent","family":"euroeval","index":43.35737762158596,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.5525226239711818,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:public-benefits-bench:overall","benchmarkName":"Public Benefits Bench v1.1","category":null,"domain":"Public benefits","eligible":false,"ranked":true,"score":51.827,"unit":"percent","family":"vals-ai:public-benefits-bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/public-benefits-bench","measuredAt":"2026-10-12","stderr":1.3,"costPerTest":0.09565,"latency":69.26},{"benchmarkId":"vals-ai:terminal-bench-2-1:overall","benchmarkName":"Terminal-Bench 2.1","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":50.187,"unit":"percent","family":"terminal-bench-2-1","index":50.493257384160344,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2-1","measuredAt":"2026-10-01","stderr":0.991,"costPerTest":0.337052,"latency":416.383},{"benchmarkId":"vals-ai:fabv2:overall","benchmarkName":"Finance Agent (v2)","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":47.442,"unit":"percent","family":"vals-ai:fabv2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/fabv2","measuredAt":"2026-10-29","stderr":0.513,"costPerTest":0.389575,"latency":142.913},{"benchmarkId":"livebench:livebench-agentic-coding:overall","benchmarkName":"LiveBench Agentic Coding","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":45.25266666666667,"unit":"percent","family":"livebench:livebench-agentic-coding:overall","index":55.830994766997236,"aboveFit":false,"run":"high effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Agentic%20Coding","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.32987083333333334,"latency":null},{"benchmarkId":"vals-ai:medcode:overall","benchmarkName":"MedCode","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":43.489,"unit":"percent","family":"vals-ai:medcode:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medcode","measuredAt":"2026-10-29","stderr":1.951,"costPerTest":0.008093,"latency":6.019},{"benchmarkId":"vals-ai:emb:overall","benchmarkName":"EMB","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":43.231,"unit":"percent","family":"vals-ai:emb:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/emb","measuredAt":"2026-10-20","stderr":3.105,"costPerTest":1.231783,"latency":493.632},{"benchmarkId":"aaEnterpriseOpsGym","benchmarkName":"Artificial Analysis EnterpriseOps-Gym","category":"agentic","score":42.3,"unit":"percent","family":"aaEnterpriseOpsGym","index":null,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/enterprise-ops-gym-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaSciCode","benchmarkName":"Artificial Analysis SciCode","category":"coding","score":41.3,"unit":"percent","family":"scicode-family","index":49.68224278943974,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/scicode","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:vibe-code:overall","benchmarkName":"Vibe Code Bench v1.1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":37.161,"unit":"percent","family":"vals-ai:vibe-code:overall","index":55.6577952340594,"aboveFit":false,"run":"OpenHands","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","measuredAt":"2026-11-09","stderr":4.627,"costPerTest":0.833111,"latency":501.522},{"benchmarkId":"omniscienceAccuracy","benchmarkName":"Artificial Analysis Omniscience Accuracy","category":"knowledge","score":26.5,"unit":"percent","family":"omniscienceAccuracy","index":44.086734044202295,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:ioi-v1:overall","benchmarkName":"IOI v1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":26.167,"unit":"percent","family":"vals-ai:ioi-v1:overall","index":53.159963601177466,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/ioi-v1","measuredAt":"2026-08-09","stderr":1.167,"costPerTest":0.636385,"latency":882.37},{"benchmarkId":"epoch-ai:epoch-frontiermath-tiers-1-3-v2-private:overall","benchmarkName":"FrontierMath-Tiers-1-3-v2-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":25.964912280701753,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tiers-1-3-v2-private:overall","index":44.56137231799126,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.601675082238553,"costPerTest":null,"latency":null},{"benchmarkId":"gdpvalAaNormalized","benchmarkName":"GDPval-AA normalized","category":"agentic","score":24.2,"unit":"percent","family":"gdpvalAaNormalized","index":51.33621255013512,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-mystery-game-puzzles:overall","benchmarkName":"Mystery Game Puzzles","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":19,"unit":"percent","family":"epoch-ai:epoch-mystery-game-puzzles:overall","index":53.39488123946846,"aboveFit":false,"run":"low effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":3.9427724440366254,"costPerTest":null,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":18.8,"unit":"percent","family":"hle","index":43.24854791987361,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-chess-puzzles:overall","benchmarkName":"Chess Puzzles","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":16,"unit":"percent","family":"epoch-ai:epoch-chess-puzzles:overall","index":46.12891884754047,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":3.6845294917747067,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:legal_research:overall","benchmarkName":"Legal Research Bench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":13.942,"unit":"percent","family":"vals-ai:legal_research:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_research","measuredAt":"2026-11-09","stderr":2.408,"costPerTest":0.124723,"latency":59.197},{"benchmarkId":"arc-prize:v2-semi-private:overall","benchmarkName":"ARC-AGI-2 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":10.277777777777779,"unit":"percent","family":"arc-prize:v2-semi-private:overall","index":43.90739982894193,"aboveFit":false,"run":"high effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.13899638250000004,"latency":null},{"benchmarkId":"arc-prize:v2-public-eval:overall","benchmarkName":"ARC-AGI-2 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":7.916666666666666,"unit":"percent","family":"arc-prize:v2-public-eval:overall","index":null,"aboveFit":false,"run":"high effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.14696105666666673,"latency":null},{"benchmarkId":"vals-ai:tax_agent_bench:overall","benchmarkName":"Tax Agent Bench","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":7.254,"unit":"percent","family":"vals-ai:tax_agent_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench","measuredAt":"2026-11-09","stderr":1.872,"costPerTest":0.081536,"latency":47.461},{"benchmarkId":"vals-ai:code-migration:overall","benchmarkName":"Code Migration","category":"coding","domain":null,"eligible":true,"ranked":true,"score":6.148,"unit":"percent","family":"vals-ai:code-migration:overall","index":48.455201540711144,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/code-migration","measuredAt":"2026-10-29","stderr":1.708,"costPerTest":0.525186,"latency":1322.302},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":0,"unit":"percent","family":"critpt","index":36.8491160127729,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:programbench:overall","benchmarkName":"ProgramBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":0,"unit":"percent","family":"vals-ai:programbench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/programbench","measuredAt":"2026-11-09","stderr":0,"costPerTest":0.337485,"latency":434.999},{"benchmarkId":"epoch-ai:epoch-frontiermath-tier-4-v2-private:overall","benchmarkName":"FrontierMath-Tier-4-v2-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":0,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tier-4-v2-private:overall","index":48.48830449725058,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":0.3,"outputPerMillion":2.5,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":130,"timeToFirstToken":0.785,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":{"elo":1434.5444034963316,"eloUpper":1439.0419344092447,"eloLower":1430.046872583418,"votes":35114,"rank":90,"rankUpper":null,"rankLower":null,"arenaName":"Gemini 3.5 Flash-Lite","organization":"google","updatedAt":"2026-11-09"},"quality":{"coveragePercent":100,"benchmarkCount":33,"categoryCount":8,"confidence":"high","state":"available"},"hosts":[{"host":"google-ai-studio","name":"Google AI Studio","kind":"maker","url":"https://ai.google.dev/gemini-api/docs/models","inputPerMillion":0.3,"outputPerMillion":2.5,"cacheReadPerMillion":0.03,"contextTokens":1048576,"maxOutputTokens":65536,"deprecated":false},{"host":"google-vertex","name":"Google Vertex AI","kind":"maker","url":"https://cloud.google.com/vertex-ai/generative-ai/docs/models","inputPerMillion":0.3,"outputPerMillion":2.5,"cacheReadPerMillion":0.03,"contextTokens":1048576,"maxOutputTokens":65536,"deprecated":false},{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":0.3,"outputPerMillion":2.5,"cacheReadPerMillion":0.03,"contextTokens":1048576,"maxOutputTokens":65536,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:livebench","name":"LiveBench","url":"https://livebench.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:arc-prize-foundation","name":"ARC Prize Foundation","url":"https://arcprize.org","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"claude-opus-4-5","providerId":"anthropic","slug":"claude-opus-4-5","name":"Claude Opus 4.5","provider":"Anthropic","description":"Claude Opus 4.5 is a standard model from Anthropic. 60 benchmarks count toward its index, in 8 categories.","logoUrl":null,"releaseDate":"2025-11-24","contextWindowTokens":200000,"modalities":["file","image","text"],"openWeights":false,"details":{"apiId":"anthropic/claude-opus-4.5","maxOutputTokens":64000,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.5,"weightsUrl":null},"reasoning":false,"overallScore":51.99840841156522,"overallScoreRange":{"low":49.91087038795943,"high":54.08594643517102},"badges":[],"capabilityScores":{"agentic":54.66049021622117,"coding":55.016551442029474,"reasoning":51.755173595909646,"multimodal":42.1667568707385,"knowledge":54.599847884877114,"multilingual":56.58524555779824,"instruction":41.2318513832651,"math":51.33598780362078},"capabilityCounts":{"knowledge":8,"math":10,"agentic":15,"instruction":2,"multimodal":10,"multilingual":2,"coding":6,"reasoning":7},"benchmarks":[{"benchmarkId":"mmluRedux","benchmarkName":"MMLU-Redux","category":"knowledge","score":96.6,"unit":"percent","family":"mmluRedux","index":54.0316087859361,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aime2026","benchmarkName":"AIME 2026","category":"math","score":95.1,"unit":"percent","family":"aime2026","index":53.75731993044779,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:mgsm:overall","benchmarkName":"MGSM","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":94.764,"unit":"percent","family":"vals-ai:mgsm:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mgsm","measuredAt":"2026-01-09","stderr":1.35,"costPerTest":0.027118,"latency":5.559},{"benchmarkId":"hmmtNov2025","benchmarkName":"Harvard-MIT Mathematics Tournament November 2025","category":"math","score":93.3,"unit":"percent","family":"hmmtNov2025","index":54.95950805513719,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:medqa:overall","benchmarkName":"MedQA","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":93.158,"unit":"percent","family":"vals-ai:medqa:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medqa","measuredAt":"2026-04-16","stderr":0.23,"costPerTest":0.043201,"latency":10.357},{"benchmarkId":"hmmtFeb2025","benchmarkName":"Harvard-MIT Mathematics Tournament February 2025","category":"math","score":92.9,"unit":"percent","family":"hmmtFeb2025","index":51.003709811296645,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"tau2-bench:telecom:overall","benchmarkName":"τ²-bench Telecom","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":92.32,"unit":"percent","family":"tau2-bench","index":64.05050983794152,"aboveFit":false,"run":"high effort · Sierra","source":"Sierra Research","sourceUrl":"https://github.com/sierra-research/tau2-bench","measuredAt":"2026-03-02","stderr":null,"costPerTest":0.71992,"latency":null},{"benchmarkId":"cEval","benchmarkName":"C-Eval","category":"knowledge","score":92.2,"unit":"percent","family":"cEval","index":48.74920363191664,"aboveFit":false,"run":null,"source":"C-Eval authors","sourceUrl":"https://arxiv.org/abs/2305.08322","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"ifeval","benchmarkName":"Instruction-Following Eval","category":"instruction","score":90.9,"unit":"percent","family":"ifeval","index":46.2472873255754,"aboveFit":false,"run":null,"source":"Jeffrey Zhou, Tianjian Lu, Swaroop Mishra, Siddhartha Brahma, Sujoy Basu, Yi Luan, Denny Zhou, Le Hou","sourceUrl":"https://arxiv.org/abs/2311.07911","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"countBench","benchmarkName":"CountBench","category":"multimodal","score":90.6,"unit":"percent","family":"countBench","index":40.50649090282806,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmluPro","benchmarkName":"Massive Multitask Language Understanding Professional","category":"knowledge","score":89.5,"unit":"percent","family":"mmlu-pro","index":57.24001555232793,"aboveFit":false,"run":null,"source":"Yubo Wang, Xueguang Ma, Ge Zhang, Yuansheng Ni, Abhranil Chandra, Shiguang Guo, Weiming Ren, Aaran Arulraj, Xuan He, Ziyan Jiang, Tianle Li, Max Ku, Kai Wang, Alex Zhuang, Rongqi Fan, Xiang Yue, Wenhu Chen","sourceUrl":"https://arxiv.org/abs/2406.01574","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"omniDocBench15","benchmarkName":"OmniDocBench 1.5","category":"multimodal","score":87.7,"unit":"percent","family":"omniDocBench15","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"ai2dTest","benchmarkName":"AI2D test split","category":"multimodal","score":87.7,"unit":"percent","family":"ai2dTest","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gpqa","benchmarkName":"Graduate-Level Google-Proof Q&A","category":"knowledge","score":87,"unit":"percent","family":"gpqa-diamond","index":57.00312345587083,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"tau2Bench","benchmarkName":"τ²-Bench Tool-Agent-User Evaluation","category":"agentic","score":86.3,"unit":"percent","family":"tau2-bench","index":60.05579970417346,"aboveFit":false,"run":null,"source":"Victor Barres, Honghua Dong, Soham Ray, Xujie Si, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2506.07982","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"include","benchmarkName":"INCLUDE","category":"multilingual","score":86.2,"unit":"percent","family":"include","index":58.13272412334296,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmluProX","benchmarkName":"MMLU-ProX","category":"multilingual","score":85.7,"unit":"percent","family":"mmluProX","index":55.037766992253516,"aboveFit":false,"run":null,"source":"MMLU-ProX authors","sourceUrl":"https://arxiv.org/abs/2503.10497","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:mmlu_pro:overall","benchmarkName":"MMLU Pro","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":85.59,"unit":"percent","family":"mmlu-pro","index":52.31388356729737,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","measuredAt":"2026-09-01","stderr":0.367,"costPerTest":0.052065,"latency":8.248},{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":85.47979797979798,"unit":"percent","family":"gpqa-diamond","index":55.608835702583676,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.150814772072325,"costPerTest":null,"latency":null},{"benchmarkId":"hmmtFeb2026","benchmarkName":"Harvard-MIT Mathematics Tournament February 2026","category":"math","score":85.3,"unit":"percent","family":"hmmtFeb2026","index":53.09524927079314,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"liveCodeBenchV6","benchmarkName":"LiveCodeBench v6","category":"coding","score":84.8,"unit":"percent","family":"liveCodeBenchV6","index":54.04271717993646,"aboveFit":false,"run":null,"source":"LiveCodeBench maintainers","sourceUrl":"https://github.com/LiveCodeBench/LiveCodeBench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"videoMmmu","benchmarkName":"VideoMMMU","category":"multimodal","score":84.4,"unit":"percent","family":"videoMmmu","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmAnswerBench","benchmarkName":"MMAnswerBench","category":"math","score":84,"unit":"percent","family":"mmAnswerBench","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"tau2-bench:airline:overall","benchmarkName":"τ²-bench Airline","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":84,"unit":"percent","family":"tau2-bench","index":58.52958154675377,"aboveFit":false,"run":"high effort · Sierra","source":"Sierra Research","sourceUrl":"https://github.com/sierra-research/tau2-bench","measuredAt":"2026-03-02","stderr":null,"costPerTest":0.39919,"latency":null},{"benchmarkId":"vals-ai:medscribe:overall","benchmarkName":"MedScribe","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":83.246,"unit":"percent","family":"vals-ai:medscribe:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medscribe","measuredAt":"2026-10-29","stderr":1.926,"costPerTest":0.281674,"latency":43.305},{"benchmarkId":"vals-ai:legal_bench:overall","benchmarkName":"LegalBench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":82.837,"unit":"percent","family":"vals-ai:legal_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","measuredAt":"2026-10-13","stderr":0.434,"costPerTest":0.01319,"latency":2.376},{"benchmarkId":"mlvuAvg","benchmarkName":"MLVU mean average","category":"multimodal","score":81.7,"unit":"percent","family":"mlvuAvg","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":81.66666666666667,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":56.4322712781001,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":5.018903659106632,"costPerTest":null,"latency":null},{"benchmarkId":"videoMmeNoSub","benchmarkName":"Video-MME without subtitle","category":"multimodal","score":81.4,"unit":"percent","family":"videoMmeNoSub","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:mmmu:overall","benchmarkName":"MMMU Pro","category":"multimodal","domain":null,"eligible":true,"ranked":true,"score":81.098,"unit":"percent","family":"mmmu-pro","index":60.69474690446973,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmmu","measuredAt":"2026-09-01","stderr":0.941,"costPerTest":0.021549,"latency":19.557},{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":81,"unit":"percent","family":"gpqa-diamond","index":51.500087406352115,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"sweVerified","benchmarkName":"Software Engineering Benchmark Verified","category":"coding","score":80.9,"unit":"percent","family":"swe-bench-verified","index":60.89633557217233,"aboveFit":false,"run":null,"source":"Carlos E. Jimenez, John Yang, Alexander Wettig, Shunyu Yao, Kexin Pei, Ofir Press, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2310.06770","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmmu","benchmarkName":"Massive Multi-discipline Multimodal Understanding","category":"multimodal","score":80.7,"unit":"percent","family":"mmmu","index":49.426812213488404,"aboveFit":false,"run":null,"source":"MMMU authors","sourceUrl":"https://arxiv.org/abs/2401.05508","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"dynaMath","benchmarkName":"DynaMath","category":"multimodal","score":79.7,"unit":"percent","family":"dynaMath","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"tau2-bench:retail:overall","benchmarkName":"τ²-bench Retail","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":79.61,"unit":"percent","family":"tau2-bench","index":55.616495585417915,"aboveFit":false,"run":"high effort · Sierra","source":"Sierra Research","sourceUrl":"https://github.com/sierra-research/tau2-bench","measuredAt":"2026-04-30","stderr":null,"costPerTest":0.38695,"latency":null},{"benchmarkId":"vals-ai:gpqa:overall","benchmarkName":"GPQA Diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":79.546,"unit":"percent","family":"gpqa-diamond","index":50.16651833701876,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","measuredAt":"2026-09-01","stderr":2.463,"costPerTest":0.074932,"latency":13.54},{"benchmarkId":"maxife","benchmarkName":"MAXIFE","category":"multilingual","score":79.2,"unit":"percent","family":"maxife","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"polyMath","benchmarkName":"PolyMath","category":"multilingual","score":79,"unit":"percent","family":"polyMath","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"videoMmeWithSub","benchmarkName":"Video-MME with subtitle","category":"multimodal","score":77.6,"unit":"percent","family":"videoMmeWithSub","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"realWorldQa","benchmarkName":"RealWorldQA","category":"multimodal","score":77,"unit":"percent","family":"realWorldQa","index":45.251985337425936,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:aime:overall","benchmarkName":"AIME","category":"math","domain":null,"eligible":true,"ranked":true,"score":76.875,"unit":"percent","family":"vals-ai:aime:overall","index":49.10045882361649,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/aime","measuredAt":"2026-04-16","stderr":1.389,"costPerTest":0.089253,"latency":17.442},{"benchmarkId":"swebench-experiments:swe-bench-verified:overall","benchmarkName":"SWE-bench Verified","category":"coding","domain":null,"eligible":true,"ranked":true,"score":76.80000000000001,"unit":"percent","family":"swe-bench-verified","index":57.872349503259194,"aboveFit":false,"run":"medium effort · live-SWE-agent","source":"SWE-bench team","sourceUrl":"https://github.com/SWE-bench/experiments/tree/main/evaluation/verified","measuredAt":"2026-09-01","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-swe-bench-verified:overall","benchmarkName":"SWE-Bench verified","category":"coding","domain":null,"eligible":true,"ranked":true,"score":76.65289256198346,"unit":"percent","family":"swe-bench-verified","index":57.763849297601745,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.924895909142592,"costPerTest":null,"latency":null},{"benchmarkId":"wideResearch","benchmarkName":"WideResearch","category":"agentic","score":76.4,"unit":"percent","family":"wideResearch","index":59.011466453452485,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:lcb:overall","benchmarkName":"LiveCodeBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":75.034,"unit":"percent","family":"livecodebench","index":51.602569880490314,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","measuredAt":"2026-09-01","stderr":1.12,"costPerTest":0.077323,"latency":13.599},{"benchmarkId":"vals-ai:tax_eval_v2:overall","benchmarkName":"TaxEval v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":74.325,"unit":"percent","family":"vals-ai:tax_eval_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","measuredAt":"2026-09-01","stderr":0.854,"costPerTest":0.058665,"latency":14.876},{"benchmarkId":"mathVision","benchmarkName":"MathVision","category":"multimodal","score":74.3,"unit":"percent","family":"mathVision","index":45.932593837788986,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aiNeedle","benchmarkName":"AI-Needle","category":"reasoning","score":74,"unit":"percent","family":"aiNeedle","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mcpTasks","benchmarkName":"MCP-Tasks","category":"agentic","score":71.8,"unit":"percent","family":"mcpTasks","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaMmmuPro","benchmarkName":"Artificial Analysis MMMU-Pro","category":"multimodal","score":71.2,"unit":"percent","family":"mmmu-pro","index":47.81336830432024,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmmu-pro","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":70.7,"unit":"percent","family":"lcr","index":55.80394261807688,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"swebench-experiments:swe-bench-multilingual:overall","benchmarkName":"SWE-bench Multilingual","category":"coding","domain":null,"eligible":true,"ranked":true,"score":70.7,"unit":"percent","family":"swe-bench-multilingual","index":null,"aboveFit":false,"run":"mini-SWE-agent","source":"SWE-bench team","sourceUrl":"https://github.com/SWE-bench/experiments/tree/main/evaluation/multilingual","measuredAt":"2026-09-02","stderr":null,"costPerTest":0.8335529758333333,"latency":null},{"benchmarkId":"mmmuPro","benchmarkName":"Massive Multi-discipline Multimodal Understanding Pro","category":"multimodal","score":70.6,"unit":"percent","family":"mmmu-pro","index":47.032520945248734,"aboveFit":false,"run":null,"source":"MMMU-Pro authors","sourceUrl":"https://arxiv.org/abs/2409.02813","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"superGpqa","benchmarkName":"SuperGPQA: Scaling LLM Evaluation Across 285 Graduate Disciplines","category":"knowledge","score":70.6,"unit":"percent","family":"superGpqa","index":55.00186800176004,"aboveFit":false,"run":null,"source":"Xiaoxuan Du, Yao Yao, Kexin Ma, Bowen Wang, Tianyu Zheng, Kaiyan Zhu, Yiming Zhang, Yutao Zhu, Jiawei Zhou, Jingren Zhou","sourceUrl":"https://arxiv.org/abs/2502.14739","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"tau3Bench","benchmarkName":"τ³-Bench Tool-Agent-User Evaluation","category":"agentic","score":70.2,"unit":"percent","family":"tau3Bench","index":60.223675961890166,"aboveFit":false,"run":null,"source":"Sierra Research","sourceUrl":"https://github.com/sierra-research/tau2-bench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"weMath","benchmarkName":"We-Math","category":"multimodal","score":70,"unit":"percent","family":"weMath","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:mortgage_tax:overall","benchmarkName":"MortgageTax","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":68.68,"unit":"percent","family":"vals-ai:mortgage_tax:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mortgage_tax","measuredAt":"2026-09-01","stderr":0.908,"costPerTest":0.092448,"latency":10.534},{"benchmarkId":"charxiv","benchmarkName":"CharXiv Reasoning","category":"multimodal","score":68.5,"unit":"percent","family":"charxiv","index":42.17134318271401,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vStar","benchmarkName":"V*","category":"multimodal","score":67,"unit":"percent","family":"vStar","index":22.149908647066454,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"osWorldVerified","benchmarkName":"OSWorld-Verified","category":"agentic","score":66.3,"unit":"percent","family":"osWorldVerified","index":54.00215376582348,"aboveFit":false,"run":null,"source":"Tianbao Xie, Danyang Zhang, Jixuan Chen, Xiaochuan Li, Siheng Zhao, Ruisheng Cao, Toh Jing Hua, Zhoujun Cheng, Dongchan Shin, Fangyu Lei, Yitao Liu, Yiheng Xu, Shuyan Zhou, Silvio Savarese, Caiming Xiong, Victor Zhong, Tao Yu","sourceUrl":"https://os-world.github.io/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"osWorld","benchmarkName":"OSWorld","category":"agentic","score":66.3,"unit":"percent","family":"osWorld","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"simpleVqa","benchmarkName":"SimpleVQA","category":"multimodal","score":65.7,"unit":"percent","family":"simpleVqa","index":56.343598119120884,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"longBenchV2","benchmarkName":"LongBench v2","category":"reasoning","score":64.4,"unit":"percent","family":"longBenchV2","index":56.69976752334651,"aboveFit":false,"run":null,"source":"LongBench v2 authors","sourceUrl":"https://arxiv.org/abs/2412.15204","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gertLabs","benchmarkName":"Gert Labs Composite Game Benchmark","category":"agentic","score":64.23,"unit":"percent","family":"gertLabs","index":67.55486608605045,"aboveFit":false,"run":null,"source":"Gert Labs","sourceUrl":"https://gertlabs.com/rankings","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:corp_fin_v2:overall","benchmarkName":"CorpFin v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":61.305,"unit":"percent","family":"vals-ai:corp_fin_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","measuredAt":"2026-08-12","stderr":0.96,"costPerTest":0.150159,"latency":9.401},{"benchmarkId":"clawEval","benchmarkName":"Claw-Eval","category":"agentic","score":59.6,"unit":"percent","family":"clawEval","index":51.29932455951161,"aboveFit":false,"run":null,"source":"Bowen Ye, Rang Li, Qibin Yang, Yuanxin Liu, Linli Yao, Hanglong Lv, Zhihui Xie, Chenxin An, Lei Li, Lingpeng Kong, Qi Liu, Zhifang Sui, Tong Yang","sourceUrl":"https://arxiv.org/abs/2604.06132","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"terminalBench2","benchmarkName":"Terminal-Bench 2.0","category":"agentic","score":59.3,"unit":"percent","family":"terminalBench2","index":56.08042645538355,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:terminal-bench-2:overall","benchmarkName":"Terminal-Bench 2.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":58.427,"unit":"percent","family":"vals-ai:terminal-bench-2:overall","index":61.730818847292845,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2","measuredAt":"2026-06-04","stderr":5.254,"costPerTest":1.175583,"latency":627.493},{"benchmarkId":"ifBench","benchmarkName":"Instruction Following Benchmark","category":"instruction","score":58,"unit":"percent","family":"ifbench-family","index":43.22524505217608,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"swePro","benchmarkName":"SWE-bench Pro","category":"coding","score":57.1,"unit":"percent","family":"swePro","index":57.46631359974735,"aboveFit":false,"run":null,"source":"Xiang Deng, Jeff Da, Edwin Pan, Yannis Yiming He, Charles Ide, Kanak Garg, Niklas Lauffer, Andrew Park, Nitin Pasari, Chetan Rane, Karmini Sampath, Maya Krishnan, Srivatsa Kundurthy, Sean Hendryx, Zifan Wang, Chen Bo Calvin Zhang, Noah Jacobson, Bing Liu, Brad Kenstler","sourceUrl":"https://arxiv.org/abs/2509.16941","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"nova63","benchmarkName":"NOVA-63","category":"multilingual","score":56.7,"unit":"percent","family":"nova63","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:terminal-bench:overall","benchmarkName":"Terminal-Bench 1.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":56.25,"unit":"percent","family":"vals-ai:terminal-bench:overall","index":58.92507730908882,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench","measuredAt":"2026-01-12","stderr":5.546,"costPerTest":0.744198,"latency":177.102},{"benchmarkId":"swebench-experiments:swe-bench-test:overall","benchmarkName":"SWE-bench (full test split)","category":"coding","domain":null,"eligible":true,"ranked":true,"score":52.615518744551004,"unit":"percent","family":"swebench-experiments:swe-bench-test:overall","index":null,"aboveFit":false,"run":"Sonar Foundation Agent","source":"SWE-bench team","sourceUrl":"https://github.com/SWE-bench/experiments/tree/main/evaluation/test","measuredAt":"2025-12-19","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"qwenClawBench","benchmarkName":"QwenClawBench","category":"agentic","score":52.3,"unit":"percent","family":"qwenClawBench","index":50.94623932532828,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"cyberGym","benchmarkName":"CyberGym","category":"agentic","score":50.6,"unit":"percent","family":"cyberGym","index":48.53071247610889,"aboveFit":false,"run":null,"source":"Zhun Wang, Tianneng Shi, Jingxuan He, Matthew Cai, Jialin Zhang, Dawn Song","sourceUrl":"https://www.cybergym.io/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"erqa","benchmarkName":"ERQA","category":"multimodal","score":46.8,"unit":"percent","family":"erqa","index":41.50831098672915,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"screenSpotPro","benchmarkName":"ScreenSpot Pro","category":"multimodal","score":45.7,"unit":"percent","family":"screenSpotPro","index":30.56315717590289,"aboveFit":false,"run":null,"source":"Kaixin Li, Ziyang Meng, Hongzhan Lin, Ziyang Luo, Yuchen Tian, Jing Ma, Zhiyong Huang, Tat-Seng Chua","sourceUrl":"https://arxiv.org/abs/2504.07981","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-simpleqa-verified:overall","benchmarkName":"SimpleQA Verified","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":45.7,"unit":"percent","family":"epoch-ai:epoch-simpleqa-verified:overall","index":61.99118662649501,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.57606915901365,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:medcode:overall","benchmarkName":"MedCode","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":45.174,"unit":"percent","family":"vals-ai:medcode:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medcode","measuredAt":"2026-10-29","stderr":1.888,"costPerTest":0.006826,"latency":5.023},{"benchmarkId":"toolathlon","benchmarkName":"Toolathlon","category":"agentic","score":43.5,"unit":"percent","family":"toolathlon","index":57.43908798892857,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"nl2Repo","benchmarkName":"NL2Repo","category":"coding","score":43.2,"unit":"percent","family":"nl2Repo","index":57.24058491490561,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaIfBench","benchmarkName":"Artificial Analysis IFBench","category":"instruction","score":43,"unit":"percent","family":"ifbench-family","index":29.207585829733524,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mcpAtlas","benchmarkName":"MCP Atlas","category":"agentic","score":42.3,"unit":"percent","family":"mcpAtlas","index":39.54691702544536,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"omniscienceAccuracy","benchmarkName":"Artificial Analysis Omniscience Accuracy","category":"knowledge","score":40.9,"unit":"percent","family":"omniscienceAccuracy","index":62.234420214659615,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tiers-1-3-v2-private:overall","benchmarkName":"FrontierMath-Tiers-1-3-v2-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":34.385964912280706,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tiers-1-3-v2-private:overall","index":48.993807781559426,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.8185763989072585,"costPerTest":null,"latency":null},{"benchmarkId":"jobBench","benchmarkName":"JobBench","category":"agentic","score":32.3,"unit":"percent","family":"jobBench","index":54.35869327490798,"aboveFit":false,"run":null,"source":"Yuetai Li, Yichen Feng, Zhangchen Xu, Zixian Ma, Kaiyuan Zheng, Fengqing Jiang, Xinghua Sun, Rulin Shao, Zichen Chen, Yue Huang, Xinyang Han, Brian Lee, Kayla Xu, Shenglai Zeng, Hang Hua, Xiangliang Zhang, Basel Alomair, Ranjay Krishna, Luke Zettlemoyer, Pang Wei Koh, Bhaskar Ramasubramanian, Luyao Niu, Xiang Yue, Radha Poovendran","sourceUrl":"https://arxiv.org/abs/2605.26329","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hle","benchmarkName":"Humanity's Last Exam","category":"knowledge","score":30.8,"unit":"percent","family":"hle","index":55.28806086023114,"aboveFit":false,"run":null,"source":"Center for AI Safety, Scale AI, and thousands of expert contributors","sourceUrl":"https://lastexam.ai/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-2025-02-28-public:overall","benchmarkName":"FrontierMath-2025-02-28-Public","category":"math","domain":null,"eligible":false,"ranked":true,"score":30,"unit":"percent","family":"epoch-ai:epoch-frontiermath-2025-02-28-public:overall","index":null,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":16.32993161855452,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-furniture-assembly:overall","benchmarkName":"Furniture Assembly","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":28.333333333333332,"unit":"percent","family":"epoch-ai:epoch-furniture-assembly:overall","index":59.87186229150464,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":5.866531842313124,"costPerTest":null,"latency":null},{"benchmarkId":"deepPlanning","benchmarkName":"DeepPlanning","category":"agentic","score":26.4,"unit":"percent","family":"deepPlanning","index":null,"aboveFit":false,"run":null,"source":"DeepPlanning authors","sourceUrl":"https://arxiv.org/abs/2601.18137","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"tau2-bench:banking-knowledge:overall","benchmarkName":"τ²-bench Banking","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":24.74,"unit":"percent","family":"tau2-bench","index":19.2062389342794,"aboveFit":false,"run":"high effort · Sierra","source":"Sierra Research","sourceUrl":"https://github.com/sierra-research/tau2-bench","measuredAt":"2026-10-20","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:ioi-v1:overall","benchmarkName":"IOI v1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":23.584,"unit":"percent","family":"vals-ai:ioi-v1:overall","index":51.8747735738379,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/ioi-v1","measuredAt":"2026-08-09","stderr":7.547,"costPerTest":3.048735,"latency":7682.383},{"benchmarkId":"vitaBench","benchmarkName":"VITA-Bench","category":"agentic","score":23.3,"unit":"percent","family":"vitaBench","index":41.728312167351355,"aboveFit":false,"run":null,"source":"Meituan LongCat Team","sourceUrl":"https://vitabench.github.io/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-mystery-game-puzzles:overall","benchmarkName":"Mystery Game Puzzles","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":22,"unit":"percent","family":"epoch-ai:epoch-mystery-game-puzzles:overall","index":55.71812592958398,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":4.163331998932266,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-2025-02-28-private:overall","benchmarkName":"FrontierMath-2025-02-28-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":20.689655172413794,"unit":"percent","family":"epoch-ai:epoch-frontiermath-2025-02-28-private:overall","index":48.17853804064686,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.382827611454509,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-ebr-bench:overall","benchmarkName":"EBR-bench","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":14.285714285714285,"unit":"percent","family":"epoch-ai:epoch-ebr-bench:overall","index":58.98288410018353,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":13.2,"unit":"percent","family":"hle","index":37.63010854770676,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-chess-puzzles:overall","benchmarkName":"Chess Puzzles","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":5,"unit":"percent","family":"epoch-ai:epoch-chess-puzzles:overall","index":37.779751700889406,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.190429135575903,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tier-4-v2-private:overall","benchmarkName":"FrontierMath-Tier-4-v2-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":4.878048780487805,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tier-4-v2-private:overall","index":50.40464614989685,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":3.4059122057973035,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-private:overall","benchmarkName":"FrontierMath-Tier-4-2025-07-01-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":4.166666666666666,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-private:overall","index":47.43436889471329,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.914766351555603,"costPerTest":null,"latency":null},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":0.3,"unit":"percent","family":"critpt","index":37.42988100778257,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":5,"outputPerMillion":25,"batch":{"inputPerMillion":2.5,"outputPerMillion":12.5},"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":39,"timeToFirstToken":1.086,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":{"elo":1451.0030369703863,"eloUpper":1454.1178279044866,"eloLower":1447.888246036286,"votes":72313,"rank":52,"rankUpper":null,"rankLower":null,"arenaName":"Claude Opus 4.5-20251101","organization":"anthropic","updatedAt":"2026-11-09"},"quality":{"coveragePercent":100,"benchmarkCount":60,"categoryCount":8,"confidence":"high","state":"available"},"hosts":[{"host":"anthropic","name":"Anthropic","kind":"maker","url":"https://docs.anthropic.com/en/docs/about-claude/models","inputPerMillion":5,"outputPerMillion":25,"cacheReadPerMillion":0.5,"contextTokens":200000,"maxOutputTokens":64000,"deprecated":false},{"host":"databricks","name":"Databricks","kind":"cloud","url":"https://docs.databricks.com/aws/en/machine-learning/foundation-models/","inputPerMillion":5,"outputPerMillion":25,"cacheReadPerMillion":0.5,"contextTokens":200000,"maxOutputTokens":64000,"deprecated":false},{"host":"digitalocean","name":"DigitalOcean","kind":"cloud","url":"https://docs.digitalocean.com/products/gradient-ai-platform/details/models/","inputPerMillion":5,"outputPerMillion":25,"cacheReadPerMillion":0.5,"contextTokens":200000,"maxOutputTokens":8192,"deprecated":false},{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":5,"outputPerMillion":25,"cacheReadPerMillion":0.5,"contextTokens":200000,"maxOutputTokens":64000,"deprecated":false},{"host":"sap-ai-core","name":"SAP AI Core","kind":"cloud","url":"https://help.sap.com/docs/sap-ai-core","inputPerMillion":5,"outputPerMillion":25,"cacheReadPerMillion":0.5,"contextTokens":200000,"maxOutputTokens":64000,"deprecated":false},{"host":"snowflake","name":"Snowflake Cortex","kind":"cloud","url":"https://docs.snowflake.com/en/user-guide/snowflake-cortex/cortex-rest-api","inputPerMillion":5,"outputPerMillion":25,"cacheReadPerMillion":0.5,"contextTokens":200000,"maxOutputTokens":64000,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:sierra-research","name":"Sierra Research","url":"https://github.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:swe-bench-team","name":"SWE-bench team","url":"https://github.com","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"a-x-k2","providerId":"sk-telecom","slug":"a-x-k2","name":"A.X K2","provider":"SK Telecom","description":"A.X K2 is a reasoning model from SK Telecom in the A.X family. 10 benchmarks count toward its index, in 6 categories.","logoUrl":null,"releaseDate":"2026-07-29","contextWindowTokens":262000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":true,"overallScore":51.90031776096021,"overallScoreRange":{"low":45.286914784467946,"high":58.51372073745248},"badges":[],"capabilityScores":{"agentic":46.33707738224357,"coding":51.3385034539044,"reasoning":53.32892083960864,"multimodal":null,"knowledge":54.473043220198846,"multilingual":null,"instruction":59.95298505762419,"math":55.19720096214989},"capabilityCounts":{"math":1,"knowledge":2,"coding":2,"instruction":1,"reasoning":2,"agentic":2},"benchmarks":[{"benchmarkId":"aime2026","benchmarkName":"AIME 2026","category":"math","score":97.1,"unit":"percent","family":"aime2026","index":55.19720096214989,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"apexShortlist","benchmarkName":"Apex Shortlist","category":"math","score":88.6,"unit":"percent","family":"apexShortlist","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":85.7,"unit":"percent","family":"gpqa-diamond","index":55.81079897847511,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gpqaDiamond","benchmarkName":"GPQA Diamond","category":"knowledge","score":85.6,"unit":"percent","family":"gpqa-diamond","index":55.719081710983126,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"liveCodeBenchV6","benchmarkName":"LiveCodeBench v6","category":"coding","score":84,"unit":"percent","family":"liveCodeBenchV6","index":53.38587146229347,"aboveFit":false,"run":null,"source":"LiveCodeBench maintainers","sourceUrl":"https://github.com/LiveCodeBench/LiveCodeBench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"ifBench","benchmarkName":"Instruction Following Benchmark","category":"instruction","score":75.9,"unit":"percent","family":"ifbench-family","index":59.95298505762419,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":66,"unit":"percent","family":"lcr","index":52.57936414782428,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"apex","benchmarkName":"Apex","category":"math","score":45.8,"unit":"percent","family":"apex","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"sciCode","benchmarkName":"Scientific Code Benchmark","category":"coding","score":41,"unit":"percent","family":"scicode-family","index":49.29113544551533,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaSciCode","benchmarkName":"Artificial Analysis SciCode","category":"coding","score":41,"unit":"percent","family":"scicode-family","index":49.29113544551533,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/scicode","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"terminalBench21","benchmarkName":"Terminal-Bench 2.1","category":"agentic","score":36,"unit":"percent","family":"terminal-bench-2-1","index":42.195914328382216,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":29.6,"unit":"percent","family":"hle","index":54.084109566195394,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hle","benchmarkName":"Humanity's Last Exam","category":"knowledge","score":27.8,"unit":"percent","family":"hle","index":52.27818262514176,"aboveFit":false,"run":null,"source":"Center for AI Safety, Scale AI, and thousands of expert contributors","sourceUrl":"https://lastexam.ai/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gdpvalAaNormalized","benchmarkName":"GDPval-AA normalized","category":"agentic","score":23,"unit":"percent","family":"gdpvalAaNormalized","index":50.47824043610492,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":8.9,"unit":"percent","family":"critpt","index":54.078477531393006,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":85,"benchmarkCount":10,"categoryCount":6,"confidence":"low","state":"available"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"glm-5-turbo","providerId":"z-ai","slug":"glm-5-turbo","name":"GLM-5-Turbo","provider":"Z.AI","description":"GLM-5-Turbo is a reasoning model from Z.AI in the GLM-5 family. 8 benchmarks count toward its index, in 4 categories.","logoUrl":null,"releaseDate":"2026-03-15","contextWindowTokens":202752,"modalities":["text"],"openWeights":false,"details":{"apiId":"z-ai/glm-5-turbo","maxOutputTokens":131072,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":false,"cacheReadPerMillion":0.24,"weightsUrl":null},"reasoning":true,"overallScore":51.89409948835709,"overallScoreRange":{"low":41.677141162463855,"high":62.11105781425033},"badges":[],"capabilityScores":{"agentic":56.97583527311099,"coding":null,"reasoning":46.959952075722555,"multimodal":null,"knowledge":51.2176764475375,"multilingual":null,"instruction":57.42980639758453,"math":null},"capabilityCounts":{"agentic":2,"knowledge":3,"instruction":1,"reasoning":2},"benchmarks":[{"benchmarkId":"tau2Bench","benchmarkName":"τ²-Bench Tool-Agent-User Evaluation","category":"agentic","score":98.5,"unit":"percent","family":"tau2-bench","index":68.15139166961706,"aboveFit":false,"run":null,"source":"Victor Barres, Honghua Dong, Soham Ray, Xujie Si, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2506.07982","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":84.7,"unit":"percent","family":"gpqa-diamond","index":54.89362630355532,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaIfBench","benchmarkName":"Artificial Analysis IFBench","category":"instruction","score":73.2,"unit":"percent","family":"ifbench-family","index":57.42980639758453,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":71.7,"unit":"percent","family":"lcr","index":56.49002314366254,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"clawEval","benchmarkName":"Claw-Eval","category":"agentic","score":55.8,"unit":"percent","family":"clawEval","index":45.800278876604935,"aboveFit":false,"run":null,"source":"Bowen Ye, Rang Li, Qibin Yang, Yuanxin Liu, Linli Yao, Hanglong Lv, Zhihui Xie, Chenxin An, Lei Li, Lingpeng Kong, Qi Liu, Zhifang Sui, Tong Yang","sourceUrl":"https://arxiv.org/abs/2604.06132","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"omniscienceAccuracy","benchmarkName":"Artificial Analysis Omniscience Accuracy","category":"knowledge","score":28.4,"unit":"percent","family":"omniscienceAccuracy","index":46.48122041391541,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":27.8,"unit":"percent","family":"hle","index":52.27818262514176,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":0.3,"unit":"percent","family":"critpt","index":37.42988100778257,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":1.2,"outputPerMillion":4,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":24.5,"timeToFirstToken":3.349,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":55.00000000000001,"benchmarkCount":8,"categoryCount":4,"confidence":"low","state":"available"},"hosts":[{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":1.2,"outputPerMillion":4,"cacheReadPerMillion":0.24,"contextTokens":202752,"maxOutputTokens":131072,"deprecated":false},{"host":"z-ai","name":"Z.AI","kind":"maker","url":"https://docs.z.ai/guides/overview/pricing","inputPerMillion":1.2,"outputPerMillion":4,"cacheReadPerMillion":0.24,"contextTokens":200000,"maxOutputTokens":131072,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gemini-3-1-flash-lite","providerId":"google","slug":"gemini-3-1-flash-lite","name":"Gemini 3.1 Flash-Lite","provider":"Google","description":"Gemini 3.1 Flash-Lite is a standard model from Google. 8 benchmarks count toward its index, in 5 categories.","logoUrl":null,"releaseDate":"2026-03-03","contextWindowTokens":1048576,"modalities":["text","image","video","file","audio"],"openWeights":false,"details":{"apiId":"google/gemini-3.1-flash-lite","maxOutputTokens":65536,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.024999999999999998,"weightsUrl":null},"reasoning":false,"overallScore":51.875967378462434,"overallScoreRange":{"low":43.159009052569196,"high":60.59292570435567},"badges":[],"capabilityScores":{"agentic":46.688464499180654,"coding":null,"reasoning":59.71242328598879,"multimodal":50.14755945830019,"knowledge":52.25050141310468,"multilingual":null,"instruction":null,"math":50.47370545405646},"capabilityCounts":{"knowledge":1,"math":2,"multimodal":2,"agentic":1,"reasoning":2},"benchmarks":[{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":81.81818181818183,"unit":"percent","family":"gpqa-diamond","index":52.25050141310468,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.7479603010538827,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":80,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":55.46261453521162,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":6.030226891555273,"costPerTest":null,"latency":null},{"benchmarkId":"charxiv","benchmarkName":"CharXiv Reasoning","category":"multimodal","score":73.2,"unit":"percent","family":"charxiv","index":47.52228578287804,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"parseBenchLlamaIndex","benchmarkName":"ParseBench: document parsing for AI agents (LlamaIndex leaderboard)","category":"multimodal","score":56.66,"unit":"percent","family":"parseBenchLlamaIndex","index":52.77283313372234,"aboveFit":false,"run":null,"source":"Boyang Zhang, Sebastián G. Acosta, Preston Carlson, Sacha Bron, Pierre-Loïc Doulcet, Daniel B. Ospina and Simon Suo (LlamaIndex)","sourceUrl":"https://arxiv.org/abs/2604.08538","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gertLabs","benchmarkName":"Gert Labs Composite Game Benchmark","category":"agentic","score":38.46,"unit":"percent","family":"gertLabs","index":46.688464499180654,"aboveFit":false,"run":null,"source":"Gert Labs","sourceUrl":"https://gertlabs.com/rankings","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tiers-1-3-v2-private:overall","benchmarkName":"FrontierMath-Tiers-1-3-v2-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":27.719298245614034,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tiers-1-3-v2-private:overall","index":45.484796372901286,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.6560920735701257,"costPerTest":null,"latency":null},{"benchmarkId":"jevBench15","benchmarkName":"jevBench15","category":"reasoning","score":19.583532784444895,"unit":"percent","family":"jevBench15","index":63.33314603564682,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"jevBench14","benchmarkName":"jevBench14","category":"reasoning","score":14.26151759252857,"unit":"percent","family":"jevBench14","index":56.09170053633076,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":0.25,"outputPerMillion":1.5,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":134,"timeToFirstToken":0.767,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":70,"benchmarkCount":8,"categoryCount":5,"confidence":"low","state":"available"},"hosts":[{"host":"google-vertex","name":"Google Vertex AI","kind":"maker","url":"https://cloud.google.com/vertex-ai/generative-ai/docs/models","inputPerMillion":0.25,"outputPerMillion":1.5,"cacheReadPerMillion":0.025,"contextTokens":1048576,"maxOutputTokens":65536,"deprecated":false},{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":0.25,"outputPerMillion":1.5,"cacheReadPerMillion":0.025,"contextTokens":1048576,"maxOutputTokens":65536,"deprecated":false},{"host":"sap-ai-core","name":"SAP AI Core","kind":"cloud","url":"https://help.sap.com/docs/sap-ai-core","inputPerMillion":0.25,"outputPerMillion":1.5,"cacheReadPerMillion":0.025,"contextTokens":1048576,"maxOutputTokens":65536,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"qwen3-5-122b-a10b","providerId":"alibaba","slug":"qwen3-5-122b-a10b","name":"Qwen3.5-122B-A10B","provider":"Alibaba","description":"Qwen3.5-122B-A10B is a reasoning model from Alibaba. 24 benchmarks count toward its index, in 7 categories.","logoUrl":null,"releaseDate":"2026-03-04","contextWindowTokens":262144,"modalities":["text","image","video"],"openWeights":true,"details":{"apiId":"qwen/qwen3.5-122b-a10b","maxOutputTokens":65536,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":null,"weightsUrl":"https://huggingface.co/Qwen/Qwen3.5-122B-A10B"},"reasoning":true,"overallScore":51.85865223379645,"overallScoreRange":{"low":47.55796480539612,"high":56.15933966219678},"badges":[],"capabilityScores":{"agentic":51.36221546850611,"coding":50.37563003246435,"reasoning":50.30667997571216,"multimodal":53.89392982017658,"knowledge":52.9474897770177,"multilingual":51.58997478212999,"instruction":55.52405873267755,"math":null},"capabilityCounts":{"agentic":5,"instruction":2,"multimodal":5,"knowledge":4,"multilingual":1,"coding":3,"reasoning":4},"benchmarks":[{"benchmarkId":"tau2Bench","benchmarkName":"τ²-Bench Tool-Agent-User Evaluation","category":"agentic","score":93.6,"unit":"percent","family":"tau2-bench","index":64.89988342120118,"aboveFit":false,"run":null,"source":"Victor Barres, Honghua Dong, Soham Ray, Xujie Si, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2506.07982","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"ifeval","benchmarkName":"Instruction-Following Eval","category":"instruction","score":93.4,"unit":"percent","family":"ifeval","index":51.28203453069682,"aboveFit":false,"run":null,"source":"Jeffrey Zhou, Tianjian Lu, Swaroop Mishra, Siddhartha Brahma, Sujoy Basu, Yi Luan, Denny Zhou, Le Hou","sourceUrl":"https://arxiv.org/abs/2311.07911","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vStar","benchmarkName":"V*","category":"multimodal","score":93.2,"unit":"percent","family":"vStar","index":53.21963726497617,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaGlobalMmluLite","benchmarkName":"Artificial Analysis Global-MMLU-Lite","category":"multilingual","score":89.1,"unit":"percent","family":"aaGlobalMmluLite","index":null,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/global-mmlu-lite","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmluPro","benchmarkName":"Massive Multitask Language Understanding Professional","category":"knowledge","score":86.7,"unit":"percent","family":"mmlu-pro","index":53.712350703712694,"aboveFit":false,"run":null,"source":"Yubo Wang, Xueguang Ma, Ge Zhang, Yuansheng Ni, Abhranil Chandra, Shiguang Guo, Weiming Ren, Aaran Arulraj, Xuan He, Ziyan Jiang, Tianle Li, Max Ku, Kai Wang, Alex Zhuang, Rongqi Fan, Xiang Yue, Wenhu Chen","sourceUrl":"https://arxiv.org/abs/2406.01574","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gpqa","benchmarkName":"Graduate-Level Google-Proof Q&A","category":"knowledge","score":86.6,"unit":"percent","family":"gpqa-diamond","index":56.63625438590291,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mathVision","benchmarkName":"MathVision","category":"multimodal","score":86.2,"unit":"percent","family":"mathVision","index":57.269374567074784,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":85.7,"unit":"percent","family":"gpqa-diamond","index":55.81079897847511,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmmu","benchmarkName":"Massive Multi-discipline Multimodal Understanding","category":"multimodal","score":83.9,"unit":"percent","family":"mmmu","index":52.90928122005039,"aboveFit":false,"run":null,"source":"MMMU authors","sourceUrl":"https://arxiv.org/abs/2401.05508","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmluProX","benchmarkName":"MMLU-ProX","category":"multilingual","score":82.2,"unit":"percent","family":"mmluProX","index":51.58997478212999,"aboveFit":false,"run":null,"source":"MMLU-ProX authors","sourceUrl":"https://arxiv.org/abs/2503.10497","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"liveCodeBenchV6","benchmarkName":"LiveCodeBench v6","category":"coding","score":78.9,"unit":"percent","family":"liveCodeBenchV6","index":49.19848001231941,"aboveFit":false,"run":null,"source":"LiveCodeBench maintainers","sourceUrl":"https://github.com/LiveCodeBench/LiveCodeBench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"charxiv","benchmarkName":"CharXiv Reasoning","category":"multimodal","score":77.2,"unit":"percent","family":"charxiv","index":52.0762794851453,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmmuPro","benchmarkName":"Massive Multi-discipline Multimodal Understanding Pro","category":"multimodal","score":76.9,"unit":"percent","family":"mmmu-pro","index":55.23141821549949,"aboveFit":false,"run":null,"source":"MMMU-Pro authors","sourceUrl":"https://arxiv.org/abs/2409.02813","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":76.3,"unit":"percent","family":"lcr","index":59.645993561356576,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaIfBench","benchmarkName":"Artificial Analysis IFBench","category":"instruction","score":75.7,"unit":"percent","family":"ifbench-family","index":59.766082934658286,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaMmmuPro","benchmarkName":"Artificial Analysis MMMU-Pro","category":"multimodal","score":75,"unit":"percent","family":"mmmu-pro","index":52.75873491177307,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmmu-pro","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmvu","benchmarkName":"Multimodal Multi-disciplinary Video Understanding","category":"multimodal","score":74.7,"unit":"percent","family":"mmvu","index":null,"aboveFit":false,"run":null,"source":"MMVU benchmark maintainers","sourceUrl":"https://www.kimi.com/blog/kimi-k2-5.html","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"sweVerified","benchmarkName":"Software Engineering Benchmark Verified","category":"coding","score":72,"unit":"percent","family":"swe-bench-verified","index":54.332073129897466,"aboveFit":false,"run":null,"source":"Carlos E. Jimenez, John Yang, Alexander Wettig, Shunyu Yao, Kexin Pei, Ofir Press, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2310.06770","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"superGpqa","benchmarkName":"SuperGPQA: Scaling LLM Evaluation Across 285 Graduate Disciplines","category":"knowledge","score":67.1,"unit":"percent","family":"superGpqa","index":52.1844602341048,"aboveFit":false,"run":null,"source":"Xiaoxuan Du, Yao Yao, Kexin Ma, Bowen Wang, Tianyu Zheng, Kaiyan Zhu, Yiming Zhang, Yutao Zhu, Jiawei Zhou, Jingren Zhou","sourceUrl":"https://arxiv.org/abs/2502.14739","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"browseComp","benchmarkName":"BrowseComp","category":"agentic","score":63.8,"unit":"percent","family":"browseComp","index":50.31225767335417,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"longBenchV2","benchmarkName":"LongBench v2","category":"reasoning","score":60.2,"unit":"percent","family":"longBenchV2","index":51.72402889264172,"aboveFit":false,"run":null,"source":"LongBench v2 authors","sourceUrl":"https://arxiv.org/abs/2412.15204","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"osWorldVerified","benchmarkName":"OSWorld-Verified","category":"agentic","score":58,"unit":"percent","family":"osWorldVerified","index":47.30192017050827,"aboveFit":false,"run":null,"source":"Tianbao Xie, Danyang Zhang, Jixuan Chen, Xiaochuan Li, Siheng Zhao, Ruisheng Cao, Toh Jing Hua, Zhoujun Cheng, Dongchan Shin, Fangyu Lei, Yitao Liu, Yiheng Xu, Shuyan Zhou, Silvio Savarese, Caiming Xiong, Victor Zhong, Tao Yu","sourceUrl":"https://os-world.github.io/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"terminalBench2","benchmarkName":"Terminal-Bench 2.0","category":"agentic","score":49.4,"unit":"percent","family":"terminalBench2","index":49.038106001712386,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaSciCode","benchmarkName":"Artificial Analysis SciCode","category":"coding","score":39.7,"unit":"percent","family":"scicode-family","index":47.59633695517618,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/scicode","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":25.2,"unit":"percent","family":"hle","index":49.669621488064294,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-mystery-game-puzzles:overall","benchmarkName":"Mystery Game Puzzles","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":17,"unit":"percent","family":"epoch-ai:epoch-mystery-game-puzzles:overall","index":51.84605144605811,"aboveFit":false,"run":"none effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":3.7752516806863685,"costPerTest":null,"latency":null},{"benchmarkId":"gdpvalAaNormalized","benchmarkName":"GDPval-AA normalized","category":"agentic","score":15.7,"unit":"percent","family":"gdpvalAaNormalized","index":45.25891007575455,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":0.6,"unit":"percent","family":"critpt","index":38.010646002792235,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":0.26,"outputPerMillion":2.08,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":70,"timeToFirstToken":0.639,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":{"elo":1417.3448440077705,"eloUpper":1421.6163575900819,"eloLower":1413.073330425459,"votes":29496,"rank":126,"rankUpper":null,"rankLower":null,"arenaName":"Qwen3.5-122B-A10B","organization":"alibaba","updatedAt":"2026-11-09"},"quality":{"coveragePercent":90,"benchmarkCount":24,"categoryCount":7,"confidence":"medium","state":"available"},"hosts":[{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":0.26,"outputPerMillion":2.08,"cacheReadPerMillion":null,"contextTokens":262144,"maxOutputTokens":65536,"deprecated":false},{"host":"siliconflow","name":"SiliconFlow","kind":"host","url":"https://cloud.siliconflow.com/models","inputPerMillion":0.26,"outputPerMillion":2.08,"cacheReadPerMillion":null,"contextTokens":262144,"maxOutputTokens":262144,"deprecated":false},{"host":"deepinfra","name":"Deep Infra","kind":"host","url":"https://deepinfra.com/models","inputPerMillion":0.29,"outputPerMillion":2.4,"cacheReadPerMillion":null,"contextTokens":262144,"maxOutputTokens":65536,"deprecated":true},{"host":"novita-ai","name":"Novita AI","kind":"host","url":"https://novita.ai/docs/guides/introduction","inputPerMillion":0.4,"outputPerMillion":3.2,"cacheReadPerMillion":null,"contextTokens":262144,"maxOutputTokens":65536,"deprecated":false},{"host":"regolo","name":"Regolo","kind":"host","url":"https://docs.regolo.ai/","inputPerMillion":1.1215,"outputPerMillion":4.7103,"cacheReadPerMillion":null,"contextTokens":262144,"maxOutputTokens":16384,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"mimo-v2-pro","providerId":"xiaomi","slug":"mimo-v2-pro","name":"MiMo-V2-Pro","provider":"Xiaomi","description":"MiMo-V2-Pro is a reasoning model from Xiaomi. 10 benchmarks count toward its index, in 5 categories.","logoUrl":null,"releaseDate":"2026-03-18","contextWindowTokens":1000000,"modalities":["text"],"openWeights":false,"details":null,"reasoning":true,"overallScore":51.78811830724937,"overallScoreRange":{"low":44.1747153307571,"high":59.40152128374164},"badges":[],"capabilityScores":{"agentic":53.2568559479144,"coding":58.75741859659962,"reasoning":45.79361518222693,"multimodal":null,"knowledge":52.0342089535661,"multilingual":null,"instruction":53.31795969233471,"math":null},"capabilityCounts":{"agentic":3,"knowledge":3,"coding":1,"instruction":1,"reasoning":2},"benchmarks":[{"benchmarkId":"tau2Bench","benchmarkName":"τ²-Bench Tool-Agent-User Evaluation","category":"agentic","score":95,"unit":"percent","family":"tau2-bench","index":65.82888577789143,"aboveFit":false,"run":null,"source":"Victor Barres, Honghua Dong, Soham Ray, Xujie Si, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2506.07982","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":87,"unit":"percent","family":"gpqa-diamond","index":57.00312345587083,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"sweVerified","benchmarkName":"Software Engineering Benchmark Verified","category":"coding","score":78,"unit":"percent","family":"swe-bench-verified","index":58.75741859659962,"aboveFit":false,"run":null,"source":"Carlos E. Jimenez, John Yang, Alexander Wettig, Shunyu Yao, Kexin Pei, Ofir Press, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2310.06770","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaIfBench","benchmarkName":"Artificial Analysis IFBench","category":"instruction","score":68.8,"unit":"percent","family":"ifbench-family","index":53.31795969233471,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":68.3,"unit":"percent","family":"lcr","index":54.15734935667129,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"clawEval","benchmarkName":"Claw-Eval","category":"agentic","score":57.8,"unit":"percent","family":"clawEval","index":48.69451344655581,"aboveFit":false,"run":null,"source":"Bowen Ye, Rang Li, Qibin Yang, Yuanxin Liu, Linli Yao, Hanglong Lv, Zhihui Xie, Chenxin An, Lei Li, Lingpeng Kong, Qi Liu, Zhifang Sui, Tong Yang","sourceUrl":"https://arxiv.org/abs/2604.06132","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gertLabs","benchmarkName":"Gert Labs Composite Game Benchmark","category":"agentic","score":36.68,"unit":"percent","family":"gertLabs","index":45.24716861929597,"aboveFit":false,"run":null,"source":"Gert Labs","sourceUrl":"https://gertlabs.com/rankings","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":30.4,"unit":"percent","family":"hle","index":54.886743762219226,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"omniscienceAccuracy","benchmarkName":"Artificial Analysis Omniscience Accuracy","category":"knowledge","score":26.6,"unit":"percent","family":"omniscienceAccuracy","index":44.21275964260825,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"researchClawBench","benchmarkName":"ResearchClawBench","category":"agentic","score":15.3,"unit":"percent","family":"researchClawBench","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":0.3,"unit":"percent","family":"critpt","index":37.42988100778257,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":{"elo":1436.575837346993,"eloUpper":1441.2342849368224,"eloLower":1431.9173897571636,"votes":25245,"rank":85,"rankUpper":null,"rankLower":null,"arenaName":"MiMo-V2-Pro","organization":"xiaomi","updatedAt":"2026-11-09"},"quality":{"coveragePercent":75,"benchmarkCount":10,"categoryCount":5,"confidence":"low","state":"available"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"o3","providerId":"openai","slug":"o3","name":"o3","provider":"OpenAI","description":"o3 is a reasoning model from OpenAI. 22 benchmarks count toward its index, in 7 categories.","logoUrl":null,"releaseDate":"2025-04-16","contextWindowTokens":200000,"modalities":["image","text","file"],"openWeights":false,"details":{"apiId":"openai/o3","maxOutputTokens":100000,"knowledgeCutoff":"2024-06-30","retiresAt":"2026-12-11","tools":true,"structuredOutput":true,"cacheReadPerMillion":0.5,"weightsUrl":null},"reasoning":true,"overallScore":51.772468792101115,"overallScoreRange":{"low":47.82501320863089,"high":55.71992437557134},"badges":[],"capabilityScores":{"agentic":51.82749311634555,"coding":50.99002748625831,"reasoning":52.852479390393995,"multimodal":53.094499276173806,"knowledge":53.70501833063206,"multilingual":null,"instruction":55.74768729089143,"math":49.097570166531455},"capabilityCounts":{"math":7,"knowledge":4,"coding":2,"agentic":1,"multimodal":1,"reasoning":6,"instruction":1},"benchmarks":[{"benchmarkId":"aaMath500","benchmarkName":"Artificial Analysis MATH-500","category":"math","score":99.2,"unit":"percent","family":"math-500","index":48.95719792942138,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/math-500","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-math-level-5:overall","benchmarkName":"MATH level 5","category":"math","domain":null,"eligible":true,"ranked":true,"score":97.77190332326285,"unit":"percent","family":"epoch-ai:epoch-math-level-5:overall","index":46.49113896487836,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0.3323033827389427,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:medqa:overall","benchmarkName":"MedQA","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":96.058,"unit":"percent","family":"vals-ai:medqa:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medqa","measuredAt":"2026-04-16","stderr":0.178,"costPerTest":0.00542,"latency":8.507},{"benchmarkId":"vals-ai:math500:overall","benchmarkName":"MATH 500","category":"math","domain":null,"eligible":true,"ranked":true,"score":94.6,"unit":"percent","family":"math-500","index":44.096110152243924,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/math500","measuredAt":"2026-01-09","stderr":1.011,"costPerTest":0.010795,"latency":16.591},{"benchmarkId":"vals-ai:mgsm:overall","benchmarkName":"MGSM","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":91.746,"unit":"percent","family":"vals-ai:mgsm:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mgsm","measuredAt":"2026-01-09","stderr":1.673,"costPerTest":0.003754,"latency":6.76},{"benchmarkId":"vals-ai:mmlu_pro:overall","benchmarkName":"MMLU Pro","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":85.595,"unit":"percent","family":"mmlu-pro","index":52.320182968812745,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","measuredAt":"2026-09-01","stderr":0.345,"costPerTest":0.011827,"latency":16.735},{"benchmarkId":"aaMmluPro","benchmarkName":"Artificial Analysis MMLU-Pro","category":"knowledge","score":85.3,"unit":"percent","family":"mmlu-pro","index":51.94851827940507,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmlu-pro","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:aime:overall","benchmarkName":"AIME","category":"math","domain":null,"eligible":true,"ranked":true,"score":85.278,"unit":"percent","family":"vals-ai:aime:overall","index":52.772954348549604,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/aime","measuredAt":"2026-04-16","stderr":1.389,"costPerTest":0.060099,"latency":266.179},{"benchmarkId":"vals-ai:gpqa:overall","benchmarkName":"GPQA Diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":84.091,"unit":"percent","family":"gpqa-diamond","index":54.33506814452917,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","measuredAt":"2026-09-01","stderr":1.861,"costPerTest":0.035552,"latency":65.777},{"benchmarkId":"vals-ai:lcb:overall","benchmarkName":"LiveCodeBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":83.914,"unit":"percent","family":"livecodebench","index":58.97067199403463,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","measuredAt":"2026-09-01","stderr":1.03,"costPerTest":0.032963,"latency":63.948},{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":83.88888888888889,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":57.725146935284734,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":4.381258550587166,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:legal_bench:overall","benchmarkName":"LegalBench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":83.761,"unit":"percent","family":"vals-ai:legal_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","measuredAt":"2026-10-13","stderr":0.424,"costPerTest":null,"latency":5.979},{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":82.7,"unit":"percent","family":"gpqa-diamond","index":53.059280953715756,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":81.81818181818183,"unit":"percent","family":"gpqa-diamond","index":52.25050141310468,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.1267123846387204,"costPerTest":null,"latency":null},{"benchmarkId":"aaLiveCodeBench","benchmarkName":"Artificial Analysis LiveCodeBench","category":"coding","score":80.8,"unit":"percent","family":"livecodebench","index":56.3868578069201,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/livecodebench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"tau2Bench","benchmarkName":"τ²-Bench Tool-Agent-User Evaluation","category":"agentic","score":80.7,"unit":"percent","family":"tau2-bench","index":56.33979027741247,"aboveFit":false,"run":null,"source":"Victor Barres, Honghua Dong, Soham Ray, Xujie Si, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2506.07982","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:mmmu:overall","benchmarkName":"MMMU Pro","category":"multimodal","domain":null,"eligible":true,"ranked":true,"score":80.416,"unit":"percent","family":"mmmu-pro","index":59.807183739658456,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmmu","measuredAt":"2026-09-01","stderr":0.954,"costPerTest":0.015444,"latency":39.275},{"benchmarkId":"vals-ai:medscribe:overall","benchmarkName":"MedScribe","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":76.654,"unit":"percent","family":"vals-ai:medscribe:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medscribe","measuredAt":"2026-10-29","stderr":1.871,"costPerTest":0.040334,"latency":46.392},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":74.7,"unit":"percent","family":"lcr","index":58.548264720419525,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:tax_eval_v2:overall","benchmarkName":"TaxEval v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":74.571,"unit":"percent","family":"vals-ai:tax_eval_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","measuredAt":"2026-09-01","stderr":0.854,"costPerTest":0.012997,"latency":24.024},{"benchmarkId":"tau2-bench:retail:overall","benchmarkName":"τ²-bench Retail","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":73.9,"unit":"percent","family":"tau2-bench","index":51.82749311634555,"aboveFit":false,"run":"OpenAI","source":"Sierra Research","sourceUrl":"https://github.com/sierra-research/tau2-bench","measuredAt":"2026-04-30","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaIfBench","benchmarkName":"Artificial Analysis IFBench","category":"instruction","score":71.4,"unit":"percent","family":"ifbench-family","index":55.74768729089143,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaMmmuPro","benchmarkName":"Artificial Analysis MMMU-Pro","category":"multimodal","score":70.1,"unit":"percent","family":"mmmu-pro","index":46.38181481268915,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmmu-pro","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:mortgage_tax:overall","benchmarkName":"MortgageTax","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":65.7,"unit":"percent","family":"vals-ai:mortgage_tax:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mortgage_tax","measuredAt":"2026-09-01","stderr":0.933,"costPerTest":0.009972,"latency":21.216},{"benchmarkId":"arc-prize:v1-public-eval:overall","benchmarkName":"ARC-AGI-1 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":64.25,"unit":"percent","family":"arc-prize:v1-public-eval:overall","index":null,"aboveFit":false,"run":"high effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.4041,"latency":null},{"benchmarkId":"epoch-ai:epoch-swe-bench-verified:overall","benchmarkName":"SWE-Bench verified","category":"coding","domain":null,"eligible":true,"ranked":true,"score":62.31884057971014,"unit":"percent","family":"swe-bench-verified","index":47.191660637730706,"aboveFit":false,"run":"medium effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.20723356581245,"costPerTest":null,"latency":null},{"benchmarkId":"arc-prize:v1-semi-private:overall","benchmarkName":"ARC-AGI-1 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":60.83,"unit":"percent","family":"arc-prize:v1-semi-private:overall","index":54.95310600412333,"aboveFit":false,"run":"high effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.5002,"latency":null},{"benchmarkId":"vals-ai:corp_fin_v2:overall","benchmarkName":"CorpFin v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":59.713,"unit":"percent","family":"vals-ai:corp_fin_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","measuredAt":"2026-08-12","stderr":0.965,"costPerTest":0.133487,"latency":19.153},{"benchmarkId":"swe-bench:swe-bench-verified:overall","benchmarkName":"SWE-bench Verified","category":"coding","domain":null,"eligible":true,"ranked":true,"score":58.4,"unit":"percent","family":"swe-bench-verified","index":44.30129007203926,"aboveFit":false,"run":"mini-SWE-agent","source":"SWE-bench team","sourceUrl":"https://www.swebench.com/","measuredAt":"2026-02-26","stderr":null,"costPerTest":0.333652748,"latency":null},{"benchmarkId":"swebench-experiments:swe-bench-verified:overall","benchmarkName":"SWE-bench Verified","category":"coding","domain":null,"eligible":true,"ranked":true,"score":58.4,"unit":"percent","family":"swe-bench-verified","index":44.30129007203926,"aboveFit":false,"run":"mini-SWE-agent","source":"SWE-bench team","sourceUrl":"https://github.com/SWE-bench/experiments/tree/main/evaluation/verified","measuredAt":"2026-09-01","stderr":null,"costPerTest":0.333652748,"latency":null},{"benchmarkId":"tau2-bench:airline:overall","benchmarkName":"τ²-bench Airline","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":52,"unit":"percent","family":"tau2-bench","index":37.29524196526238,"aboveFit":false,"run":"OpenAI","source":"Sierra Research","sourceUrl":"https://github.com/sierra-research/tau2-bench","measuredAt":"2026-03-02","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-simpleqa-verified:overall","benchmarkName":"SimpleQA Verified","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":49.4,"unit":"percent","family":"epoch-ai:epoch-simpleqa-verified:overall","index":65.07361325629122,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.5818160898606832,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:medcode:overall","benchmarkName":"MedCode","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":47.29,"unit":"percent","family":"vals-ai:medcode:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medcode","measuredAt":"2026-10-29","stderr":2.161,"costPerTest":0.029818,"latency":17.676},{"benchmarkId":"epoch-ai:epoch-chess-puzzles:overall","benchmarkName":"Chess Puzzles","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":36,"unit":"percent","family":"epoch-ai:epoch-chess-puzzles:overall","index":61.309222750542396,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":4.824181513244217,"costPerTest":null,"latency":null},{"benchmarkId":"swe-bench:swe-bench-multimodal:overall","benchmarkName":"SWE-bench Multimodal","category":"multimodal","domain":null,"eligible":true,"ranked":true,"score":35.98,"unit":"percent","family":"swe-bench-multimodal","index":null,"aboveFit":false,"run":"GUIRepair","source":"SWE-bench team","sourceUrl":"https://www.swebench.com/","measuredAt":"2025-11-17","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"swebench-experiments:swe-bench-multimodal:overall","benchmarkName":"SWE-bench Multimodal","category":"multimodal","domain":null,"eligible":true,"ranked":true,"score":35.97678916827853,"unit":"percent","family":"swe-bench-multimodal","index":null,"aboveFit":false,"run":"GUIRepair","source":"SWE-bench team","sourceUrl":"https://github.com/SWE-bench/experiments/tree/main/evaluation/multimodal","measuredAt":"2025-11-17","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tiers-1-3-v2-private:overall","benchmarkName":"FrontierMath-Tiers-1-3-v2-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":33.33333333333333,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tiers-1-3-v2-private:overall","index":48.4397533486134,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.7972711943222968,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-mystery-game-puzzles:overall","benchmarkName":"Mystery Game Puzzles","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":28.999999999999996,"unit":"percent","family":"epoch-ai:epoch-mystery-game-puzzles:overall","index":61.139030206520175,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":4.560480215720688,"costPerTest":null,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":20.1,"unit":"percent","family":"hle","index":44.552828488412345,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-2025-02-28-private:overall","benchmarkName":"FrontierMath-2025-02-28-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":18.685121107266436,"unit":"percent","family":"epoch-ai:epoch-frontiermath-2025-02-28-private:overall","index":46.408980216621956,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.2968716344899227,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-2025-02-28-public:overall","benchmarkName":"FrontierMath-2025-02-28-Public","category":"math","domain":null,"eligible":false,"ranked":true,"score":10,"unit":"percent","family":"epoch-ai:epoch-frontiermath-2025-02-28-public:overall","index":null,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":10,"costPerTest":null,"latency":null},{"benchmarkId":"arc-prize:v2-semi-private:overall","benchmarkName":"ARC-AGI-2 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":6.529999999999999,"unit":"percent","family":"arc-prize:v2-semi-private:overall","index":42.186664999616845,"aboveFit":false,"run":"high effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.8339,"latency":null},{"benchmarkId":"arc-prize:v2-public-eval:overall","benchmarkName":"ARC-AGI-2 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":2.92,"unit":"percent","family":"arc-prize:v2-public-eval:overall","index":null,"aboveFit":false,"run":"high effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.9001,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-private:overall","benchmarkName":"FrontierMath-Tier-4-2025-07-01-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":2.083333333333333,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-private:overall","index":45.31836331093945,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.083333333333333,"costPerTest":null,"latency":null},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":1.1,"unit":"percent","family":"critpt","index":38.97858766114168,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-public:overall","benchmarkName":"FrontierMath-Tier-4-2025-07-01-Public","category":"math","domain":null,"eligible":false,"ranked":true,"score":0,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-public:overall","index":null,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":2,"outputPerMillion":8,"batch":{"inputPerMillion":1,"outputPerMillion":4},"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":65,"timeToFirstToken":2.681,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":{"elo":1409.750886127375,"eloUpper":1413.363029483272,"eloLower":1406.138742771478,"votes":58811,"rank":143,"rankUpper":null,"rankLower":null,"arenaName":"o3-2025-04-16","organization":"openai","updatedAt":"2026-11-09"},"quality":{"coveragePercent":95,"benchmarkCount":22,"categoryCount":7,"confidence":"medium","state":"available"},"hosts":[{"host":"digitalocean","name":"DigitalOcean","kind":"cloud","url":"https://docs.digitalocean.com/products/gradient-ai-platform/details/models/","inputPerMillion":2,"outputPerMillion":8,"cacheReadPerMillion":0.5,"contextTokens":200000,"maxOutputTokens":100000,"deprecated":false},{"host":"azure","name":"Microsoft Azure","kind":"cloud","url":"https://learn.microsoft.com/en-us/azure/ai-services/openai/concepts/models","inputPerMillion":2,"outputPerMillion":8,"cacheReadPerMillion":0.5,"contextTokens":200000,"maxOutputTokens":100000,"deprecated":false},{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":2,"outputPerMillion":8,"cacheReadPerMillion":0.5,"contextTokens":200000,"maxOutputTokens":100000,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:sierra-research","name":"Sierra Research","url":"https://github.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:arc-prize-foundation","name":"ARC Prize Foundation","url":"https://arcprize.org","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:swe-bench-team","name":"SWE-bench team","url":"https://www.swebench.com","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"glm-5","providerId":"z-ai","slug":"glm-5","name":"GLM-5","provider":"Z.AI","description":"GLM-5 is a standard model from Z.AI. 38 benchmarks count toward its index, in 7 categories.","logoUrl":null,"releaseDate":"2026-02-12","contextWindowTokens":204800,"modalities":["text"],"openWeights":true,"details":{"apiId":"z-ai/glm-5","maxOutputTokens":131072,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.19999999999999998,"weightsUrl":"https://huggingface.co/zai-org/GLM-5"},"reasoning":false,"overallScore":51.58343989176897,"overallScoreRange":{"low":47.96031951208454,"high":55.2065602714534},"badges":[],"capabilityScores":{"agentic":51.56230855092096,"coding":53.48393005138088,"reasoning":48.26675144364955,"multimodal":null,"knowledge":51.75283271471395,"multilingual":52.47654992187603,"instruction":53.12983113464796,"math":52.491910064999104},"capabilityCounts":{"agentic":14,"math":7,"instruction":2,"knowledge":5,"multilingual":1,"coding":4,"reasoning":5},"benchmarks":[{"benchmarkId":"tau2Bench","benchmarkName":"τ²-Bench Tool-Agent-User Evaluation","category":"agentic","score":98.2,"unit":"percent","family":"tau2-bench","index":67.95231973604058,"aboveFit":false,"run":null,"source":"Victor Barres, Honghua Dong, Soham Ray, Xujie Si, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2506.07982","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hmmtFeb2025","benchmarkName":"Harvard-MIT Mathematics Tournament February 2025","category":"math","score":97.5,"unit":"percent","family":"hmmtFeb2025","index":55.183846075222725,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hmmtNov2025","benchmarkName":"Harvard-MIT Mathematics Tournament November 2025","category":"math","score":96.9,"unit":"percent","family":"hmmtNov2025","index":58.91393122545401,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aime2026","benchmarkName":"AIME 2026","category":"math","score":95.8,"unit":"percent","family":"aime2026","index":54.261278291543526,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aime2025Arcee","benchmarkName":"AIME25 first-party comparison snapshot","category":"math","score":93.3,"unit":"percent","family":"aime2025Arcee","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"ifeval","benchmarkName":"Instruction-Following Eval","category":"instruction","score":92.6,"unit":"percent","family":"ifeval","index":49.67091542505794,"aboveFit":false,"run":null,"source":"Jeffrey Zhou, Tianjian Lu, Swaroop Mishra, Siddhartha Brahma, Sujoy Basu, Yi Luan, Denny Zhou, Le Hou","sourceUrl":"https://arxiv.org/abs/2311.07911","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":87.81725888324873,"unit":"percent","family":"gpqa-diamond","index":57.75269097192202,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.336331210092387,"costPerTest":null,"latency":null},{"benchmarkId":"tau2-bench:telecom:overall","benchmarkName":"τ²-bench Telecom","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":86.84,"unit":"percent","family":"tau2-bench","index":60.41412918461113,"aboveFit":false,"run":"enabled effort · Sierra","source":"Sierra Research","sourceUrl":"https://github.com/sierra-research/tau2-bench","measuredAt":"2026-03-02","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hmmtFeb2026","benchmarkName":"Harvard-MIT Mathematics Tournament February 2026","category":"math","score":86.4,"unit":"percent","family":"hmmtFeb2026","index":53.86505084583973,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gpqa","benchmarkName":"Graduate-Level Google-Proof Q&A","category":"knowledge","score":86,"unit":"percent","family":"gpqa-diamond","index":56.085950780951045,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gpqaDiamond","benchmarkName":"GPQA Diamond","category":"knowledge","score":86,"unit":"percent","family":"gpqa-diamond","index":56.085950780951045,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmluProArcee","benchmarkName":"MMLU-Pro first-party comparison snapshot","category":"knowledge","score":85.8,"unit":"percent","family":"mmlu-pro","index":52.578458430943506,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmluPro","benchmarkName":"Massive Multitask Language Understanding Professional","category":"knowledge","score":85.7,"unit":"percent","family":"mmlu-pro","index":52.452470400635825,"aboveFit":false,"run":null,"source":"Yubo Wang, Xueguang Ma, Ge Zhang, Yuansheng Ni, Abhranil Chandra, Shiguang Guo, Weiming Ren, Aaran Arulraj, Xuan He, Ziyan Jiang, Tianle Li, Max Ku, Kai Wang, Alex Zhuang, Rongqi Fan, Xiang Yue, Wenhu Chen","sourceUrl":"https://arxiv.org/abs/2406.01574","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmluProX","benchmarkName":"MMLU-ProX","category":"multilingual","score":83.1,"unit":"percent","family":"mmluProX","index":52.47654992187603,"aboveFit":false,"run":null,"source":"MMLU-ProX authors","sourceUrl":"https://arxiv.org/abs/2503.10497","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmAnswerBench","benchmarkName":"MMAnswerBench","category":"math","score":82.5,"unit":"percent","family":"mmAnswerBench","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"tau2-bench:airline:overall","benchmarkName":"τ²-bench Airline","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":82.5,"unit":"percent","family":"tau2-bench","index":57.53422187887136,"aboveFit":false,"run":"enabled effort · Sierra","source":"Sierra Research","sourceUrl":"https://github.com/sierra-research/tau2-bench","measuredAt":"2026-03-02","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":82,"unit":"percent","family":"gpqa-diamond","index":52.417260081271905,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":80,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":55.46261453521162,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":6,"costPerTest":null,"latency":null},{"benchmarkId":"sweVerified","benchmarkName":"Software Engineering Benchmark Verified","category":"coding","score":77.8,"unit":"percent","family":"swe-bench-verified","index":58.60990708104288,"aboveFit":false,"run":null,"source":"Carlos E. Jimenez, John Yang, Alexander Wettig, Shunyu Yao, Kexin Pei, Ofir Press, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2310.06770","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":75.7,"unit":"percent","family":"lcr","index":59.234345246005184,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"reactNativeEvals","benchmarkName":"React Native Evals","category":"coding","score":74.8,"unit":"percent","family":"reactNativeEvals","index":51.15666785649282,"aboveFit":false,"run":null,"source":"Callstack","sourceUrl":"https://rn-evals.vercel.app/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"tau2-bench:retail:overall","benchmarkName":"τ²-bench Retail","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":73.68,"unit":"percent","family":"tau2-bench","index":51.681507031722795,"aboveFit":false,"run":"enabled effort · Sierra","source":"Sierra Research","sourceUrl":"https://github.com/sierra-research/tau2-bench","measuredAt":"2026-04-30","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"sweVerifiedArcee","benchmarkName":"SWE-bench Verified (mini-swe-agent-v2)","category":"coding","score":72.8,"unit":"percent","family":"swe-bench-verified","index":54.92211919212442,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"swebench-experiments:swe-bench-verified:overall","benchmarkName":"SWE-bench Verified","category":"coding","domain":null,"eligible":true,"ranked":true,"score":72.8,"unit":"percent","family":"swe-bench-verified","index":54.92211919212442,"aboveFit":false,"run":"high effort · mini-SWE-agent","source":"SWE-bench team","sourceUrl":"https://github.com/SWE-bench/experiments/tree/main/evaluation/verified","measuredAt":"2026-09-01","stderr":null,"costPerTest":0.53438913622,"latency":null},{"benchmarkId":"aaIfBench","benchmarkName":"Artificial Analysis IFBench","category":"instruction","score":72.3,"unit":"percent","family":"ifbench-family","index":56.58874684423797,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-swe-bench-verified:overall","benchmarkName":"SWE-Bench verified","category":"coding","domain":null,"eligible":true,"ranked":true,"score":72.07792207792207,"unit":"percent","family":"swe-bench-verified","index":54.38954514894554,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.0894141602415233,"costPerTest":null,"latency":null},{"benchmarkId":"wideResearch","benchmarkName":"WideResearch","category":"agentic","score":69.8,"unit":"percent","family":"wideResearch","index":52.89885126891064,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"swe-bench:swe-bench-multilingual:overall","benchmarkName":"SWE-bench Multilingual","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":69.7,"unit":"percent","family":"swe-bench-multilingual","index":null,"aboveFit":false,"run":"mini-SWE-agent","source":"SWE-bench team","sourceUrl":"https://www.swebench.com/","measuredAt":"2026-02-20","stderr":null,"costPerTest":0.6424743668666667,"latency":null},{"benchmarkId":"swebench-experiments:swe-bench-multilingual:overall","benchmarkName":"SWE-bench Multilingual","category":"coding","domain":null,"eligible":true,"ranked":true,"score":69.7,"unit":"percent","family":"swe-bench-multilingual","index":null,"aboveFit":false,"run":"mini-SWE-agent","source":"SWE-bench team","sourceUrl":"https://github.com/SWE-bench/experiments/tree/main/evaluation/multilingual","measuredAt":"2026-09-02","stderr":null,"costPerTest":0.6424743668666667,"latency":null},{"benchmarkId":"superGpqa","benchmarkName":"SuperGPQA: Scaling LLM Evaluation Across 285 Graduate Disciplines","category":"knowledge","score":66.8,"unit":"percent","family":"superGpqa","index":51.942968139734354,"aboveFit":false,"run":null,"source":"Xiaoxuan Du, Yao Yao, Kexin Ma, Bowen Wang, Tianyu Zheng, Kaiyan Zhu, Yiming Zhang, Yutao Zhu, Jiawei Zhou, Jingren Zhou","sourceUrl":"https://arxiv.org/abs/2502.14739","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"tau3Bench","benchmarkName":"τ³-Bench Tool-Agent-User Evaluation","category":"agentic","score":65.6,"unit":"percent","family":"tau3Bench","index":54.93238058333824,"aboveFit":false,"run":null,"source":"Sierra Research","sourceUrl":"https://github.com/sierra-research/tau2-bench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aiNeedle","benchmarkName":"AI-Needle","category":"reasoning","score":63.3,"unit":"percent","family":"aiNeedle","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"sweRebench","benchmarkName":"SWE-Rebench","category":"coding","score":62.8,"unit":"percent","family":"sweRebench","index":null,"aboveFit":false,"run":null,"source":"Nebius","sourceUrl":"https://swe-rebench.com","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"browseComp","benchmarkName":"BrowseComp","category":"agentic","score":62,"unit":"percent","family":"browseComp","index":48.890222022555385,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mcpTasks","benchmarkName":"MCP-Tasks","category":"agentic","score":60.8,"unit":"percent","family":"mcpTasks","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"longBenchV2","benchmarkName":"LongBench v2","category":"reasoning","score":60.8,"unit":"percent","family":"longBenchV2","index":52.43484869702811,"aboveFit":false,"run":null,"source":"LongBench v2 authors","sourceUrl":"https://arxiv.org/abs/2412.15204","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"arc-prize:v1-public-eval:overall","benchmarkName":"ARC-AGI-1 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":58.63,"unit":"percent","family":"arc-prize:v1-public-eval:overall","index":null,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.14,"latency":null},{"benchmarkId":"clawEval","benchmarkName":"Claw-Eval","category":"agentic","score":57.7,"unit":"percent","family":"clawEval","index":48.549801718058276,"aboveFit":false,"run":null,"source":"Bowen Ye, Rang Li, Qibin Yang, Yuanxin Liu, Linli Yao, Hanglong Lv, Zhihui Xie, Chenxin An, Lei Li, Lingpeng Kong, Qi Liu, Zhifang Sui, Tong Yang","sourceUrl":"https://arxiv.org/abs/2604.06132","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"terminalBench2","benchmarkName":"Terminal-Bench 2.0","category":"agentic","score":56.2,"unit":"percent","family":"terminalBench2","index":53.875255404234,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"swePro","benchmarkName":"SWE-bench Pro","category":"coding","score":55.1,"unit":"percent","family":"swePro","index":55.763093637371384,"aboveFit":false,"run":null,"source":"Xiang Deng, Jeff Da, Edwin Pan, Yannis Yiming He, Charles Ide, Kanak Garg, Niklas Lauffer, Andrew Park, Nitin Pasari, Chetan Rane, Karmini Sampath, Maya Krishnan, Srivatsa Kundurthy, Sean Hendryx, Zifan Wang, Chen Bo Calvin Zhang, Noah Jacobson, Bing Liu, Brad Kenstler","sourceUrl":"https://arxiv.org/abs/2509.16941","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"nova63","benchmarkName":"NOVA-63","category":"multilingual","score":55.1,"unit":"percent","family":"nova63","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"qwenClawBench","benchmarkName":"QwenClawBench","category":"agentic","score":54.1,"unit":"percent","family":"qwenClawBench","index":52.42291109321461,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gertLabs","benchmarkName":"Gert Labs Composite Game Benchmark","category":"agentic","score":50.99,"unit":"percent","family":"gertLabs","index":56.83421583342507,"aboveFit":false,"run":null,"source":"Gert Labs","sourceUrl":"https://gertlabs.com/rankings","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hleWithTools","benchmarkName":"Humanity's Last Exam with tools","category":"agentic","score":50.4,"unit":"percent","family":"hleWithTools","index":60.382309114978526,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"arc-prize:v1-semi-private:overall","benchmarkName":"ARC-AGI-1 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":44.67,"unit":"percent","family":"arc-prize:v1-semi-private:overall","index":47.52377058005709,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.17,"latency":null},{"benchmarkId":"cyberGym","benchmarkName":"CyberGym","category":"agentic","score":43.2,"unit":"percent","family":"cyberGym","index":43.95590069751661,"aboveFit":false,"run":null,"source":"Zhun Wang, Tianneng Shi, Jingxuan He, Matthew Cai, Jialin Zhang, Dawn Song","sourceUrl":"https://www.cybergym.io/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"terminalBenchHard","benchmarkName":"Terminal-Bench Hard","category":"agentic","score":43.2,"unit":"percent","family":"terminalBenchHard","index":54.5377851211709,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/terminalbench-hard","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"toolathlon","benchmarkName":"Toolathlon","category":"agentic","score":38,"unit":"percent","family":"toolathlon","index":53.01322386407534,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"nl2Repo","benchmarkName":"NL2Repo","category":"coding","score":35.9,"unit":"percent","family":"nl2Repo","index":52.09383951953491,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mcpAtlas","benchmarkName":"MCP Atlas","category":"agentic","score":31.1,"unit":"percent","family":"mcpAtlas","index":32.004886349441406,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hle","benchmarkName":"Humanity's Last Exam","category":"knowledge","score":30.5,"unit":"percent","family":"hle","index":54.98707303672221,"aboveFit":false,"run":null,"source":"Center for AI Safety, Scale AI, and thousands of expert contributors","sourceUrl":"https://lastexam.ai/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":29.3,"unit":"percent","family":"hle","index":53.783121742686454,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"omniscienceAccuracy","benchmarkName":"Artificial Analysis Omniscience Accuracy","category":"knowledge","score":26.3,"unit":"percent","family":"omniscienceAccuracy","index":43.83468284739039,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-2025-02-28-private:overall","benchmarkName":"FrontierMath-2025-02-28-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":16.433566433566433,"unit":"percent","family":"epoch-ai:epoch-frontiermath-2025-02-28-private:overall","index":44.42135812611245,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.1951258107494094,"costPerTest":null,"latency":null},{"benchmarkId":"deepPlanning","benchmarkName":"DeepPlanning","category":"agentic","score":14.6,"unit":"percent","family":"deepPlanning","index":null,"aboveFit":false,"run":null,"source":"DeepPlanning authors","sourceUrl":"https://arxiv.org/abs/2601.18137","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"apexAgentsAa","benchmarkName":"APEX-Agents-AA","category":"agentic","score":14.5,"unit":"percent","family":"apexAgentsAa","index":52.04035476310308,"aboveFit":false,"run":null,"source":"Artificial Analysis / Mercor","sourceUrl":"https://artificialanalysis.ai/evaluations/apex-agents-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"tau2-bench:banking-knowledge:overall","benchmarkName":"τ²-bench Banking","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":9.79,"unit":"percent","family":"tau2-bench","index":9.285820911051395,"aboveFit":false,"run":"enabled effort · Sierra","source":"Sierra Research","sourceUrl":"https://github.com/sierra-research/tau2-bench","measuredAt":"2026-10-20","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"arc-prize:v2-public-eval:overall","benchmarkName":"ARC-AGI-2 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":5.42,"unit":"percent","family":"arc-prize:v2-public-eval:overall","index":null,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.29,"latency":null},{"benchmarkId":"arc-prize:v2-semi-private:overall","benchmarkName":"ARC-AGI-2 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":4.859999999999999,"unit":"percent","family":"arc-prize:v2-semi-private:overall","index":41.419910048986665,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.27,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-private:overall","benchmarkName":"FrontierMath-Tier-4-2025-07-01-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":2.1,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-private:overall","index":45.33529135560964,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.1,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-2025-02-28-public:overall","benchmarkName":"FrontierMath-2025-02-28-Public","category":"math","domain":null,"eligible":false,"ranked":true,"score":2.08,"unit":"percent","family":"epoch-ai:epoch-frontiermath-2025-02-28-public:overall","index":null,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.1,"costPerTest":null,"latency":null},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":2,"unit":"percent","family":"critpt","index":40.72088264617068,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-public:overall","benchmarkName":"FrontierMath-Tier-4-2025-07-01-Public","category":"math","domain":null,"eligible":false,"ranked":true,"score":0,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-public:overall","index":null,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":1,"outputPerMillion":3.1999999999999997,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":45,"timeToFirstToken":1.4845,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":{"elo":1446.1220835069007,"eloUpper":1450.3484358060514,"eloLower":1441.89573120775,"votes":29020,"rank":61,"rankUpper":null,"rankLower":null,"arenaName":"GLM-5","organization":"zai","updatedAt":"2026-11-09"},"quality":{"coveragePercent":90,"benchmarkCount":38,"categoryCount":7,"confidence":"medium","state":"available"},"hosts":[{"host":"vultr","name":"Vultr","kind":"cloud","url":"https://api.vultrinference.com/","inputPerMillion":0.4,"outputPerMillion":1.75,"cacheReadPerMillion":null,"contextTokens":202752,"maxOutputTokens":131072,"deprecated":false},{"host":"gmi-cloud","name":"GMI Cloud","kind":"host","url":"https://docs.gmicloud.ai/inference-engine/api-reference/llm-api-reference","inputPerMillion":0.6,"outputPerMillion":1.92,"cacheReadPerMillion":0.12,"contextTokens":202752,"maxOutputTokens":131072,"deprecated":false},{"host":"deepinfra","name":"Deep Infra","kind":"host","url":"https://deepinfra.com/models","inputPerMillion":0.6,"outputPerMillion":2.08,"cacheReadPerMillion":0.12,"contextTokens":202752,"maxOutputTokens":16384,"deprecated":true},{"host":"siliconflow","name":"SiliconFlow","kind":"host","url":"https://cloud.siliconflow.com/models","inputPerMillion":0.95,"outputPerMillion":2.55,"cacheReadPerMillion":0.2,"contextTokens":205000,"maxOutputTokens":131072,"deprecated":false},{"host":"zhipu-ai","name":"Zhipu AI","kind":"maker","url":"https://docs.bigmodel.cn/cn/guide/start/pricing","inputPerMillion":0.59758,"outputPerMillion":2.68911,"cacheReadPerMillion":0.1494,"contextTokens":204800,"maxOutputTokens":131072,"deprecated":false},{"host":"amazon-bedrock","name":"Amazon Bedrock","kind":"cloud","url":"https://docs.aws.amazon.com/bedrock/latest/userguide/models-supported.html","inputPerMillion":1,"outputPerMillion":3.2,"cacheReadPerMillion":null,"contextTokens":202752,"maxOutputTokens":131072,"deprecated":false},{"host":"google-vertex","name":"Google Vertex AI","kind":"cloud","url":"https://cloud.google.com/vertex-ai/generative-ai/docs/models","inputPerMillion":1,"outputPerMillion":3.2,"cacheReadPerMillion":0.1,"contextTokens":200000,"maxOutputTokens":128000,"deprecated":true},{"host":"novita-ai","name":"Novita AI","kind":"host","url":"https://novita.ai/docs/guides/introduction","inputPerMillion":1,"outputPerMillion":3.2,"cacheReadPerMillion":0.2,"contextTokens":202800,"maxOutputTokens":131072,"deprecated":false},{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":1,"outputPerMillion":3.2,"cacheReadPerMillion":0.2,"contextTokens":204800,"maxOutputTokens":131072,"deprecated":false},{"host":"together-ai","name":"Together AI","kind":"host","url":"https://docs.together.ai/docs/serverless-models","inputPerMillion":1,"outputPerMillion":3.2,"cacheReadPerMillion":null,"contextTokens":202752,"maxOutputTokens":131072,"deprecated":true},{"host":"z-ai","name":"Z.AI","kind":"maker","url":"https://docs.z.ai/guides/overview/pricing","inputPerMillion":1,"outputPerMillion":3.2,"cacheReadPerMillion":0.2,"contextTokens":204800,"maxOutputTokens":131072,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:sierra-research","name":"Sierra Research","url":"https://github.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:swe-bench-team","name":"SWE-bench team","url":"https://github.com","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:arc-prize-foundation","name":"ARC Prize Foundation","url":"https://arcprize.org","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"qwen3-5-27b","providerId":"alibaba","slug":"qwen3-5-27b","name":"Qwen3.5-27B","provider":"Alibaba","description":"Qwen3.5-27B is a reasoning model from Alibaba. 24 benchmarks count toward its index, in 7 categories.","logoUrl":null,"releaseDate":"2026-03-04","contextWindowTokens":262144,"modalities":["text","image","video"],"openWeights":true,"details":{"apiId":"qwen/qwen3.5-27b","maxOutputTokens":81920,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":null,"weightsUrl":"https://huggingface.co/Qwen/Qwen3.5-27B"},"reasoning":true,"overallScore":51.33192825660454,"overallScoreRange":{"low":47.03124082820421,"high":55.63261568500487},"badges":[],"capabilityScores":{"agentic":50.325832025364406,"coding":52.65173951901354,"reasoning":50.46527535240369,"multimodal":53.90260378937772,"knowledge":48.968190577006155,"multilingual":51.58997478212999,"instruction":57.088452307574926,"math":null},"capabilityCounts":{"instruction":2,"agentic":6,"multimodal":5,"knowledge":5,"multilingual":1,"coding":2,"reasoning":3},"benchmarks":[{"benchmarkId":"ifeval","benchmarkName":"Instruction-Following Eval","category":"instruction","score":95,"unit":"percent","family":"ifeval","index":54.50427274197452,"aboveFit":false,"run":null,"source":"Jeffrey Zhou, Tianjian Lu, Swaroop Mishra, Siddhartha Brahma, Sujoy Basu, Yi Luan, Denny Zhou, Le Hou","sourceUrl":"https://arxiv.org/abs/2311.07911","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"tau2Bench","benchmarkName":"τ²-Bench Tool-Agent-User Evaluation","category":"agentic","score":93.9,"unit":"percent","family":"tau2-bench","index":65.09895535477767,"aboveFit":false,"run":null,"source":"Victor Barres, Honghua Dong, Soham Ray, Xujie Si, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2506.07982","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vStar","benchmarkName":"V*","category":"multimodal","score":93.7,"unit":"percent","family":"vStar","index":53.812571017226354,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaGlobalMmluLite","benchmarkName":"Artificial Analysis Global-MMLU-Lite","category":"multilingual","score":86.8,"unit":"percent","family":"aaGlobalMmluLite","index":null,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/global-mmlu-lite","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmluPro","benchmarkName":"Massive Multitask Language Understanding Professional","category":"knowledge","score":86.1,"unit":"percent","family":"mmlu-pro","index":52.956422521866564,"aboveFit":false,"run":null,"source":"Yubo Wang, Xueguang Ma, Ge Zhang, Yuansheng Ni, Abhranil Chandra, Shiguang Guo, Weiming Ren, Aaran Arulraj, Xuan He, Ziyan Jiang, Tianle Li, Max Ku, Kai Wang, Alex Zhuang, Rongqi Fan, Xiang Yue, Wenhu Chen","sourceUrl":"https://arxiv.org/abs/2406.01574","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mathVision","benchmarkName":"MathVision","category":"multimodal","score":86,"unit":"percent","family":"mathVision","index":57.078840437170825,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":85.8,"unit":"percent","family":"gpqa-diamond","index":55.902516245967085,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gpqa","benchmarkName":"Graduate-Level Google-Proof Q&A","category":"knowledge","score":85.5,"unit":"percent","family":"gpqa-diamond","index":55.62736444349115,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmmu","benchmarkName":"Massive Multi-discipline Multimodal Understanding","category":"multimodal","score":82.3,"unit":"percent","family":"mmmu","index":51.16804671676939,"aboveFit":false,"run":null,"source":"MMMU authors","sourceUrl":"https://arxiv.org/abs/2401.05508","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmluProX","benchmarkName":"MMLU-ProX","category":"multilingual","score":82.2,"unit":"percent","family":"mmluProX","index":51.58997478212999,"aboveFit":false,"run":null,"source":"MMLU-ProX authors","sourceUrl":"https://arxiv.org/abs/2503.10497","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"liveCodeBenchV6","benchmarkName":"LiveCodeBench v6","category":"coding","score":80.7,"unit":"percent","family":"liveCodeBenchV6","index":50.67638287701613,"aboveFit":false,"run":null,"source":"LiveCodeBench maintainers","sourceUrl":"https://github.com/LiveCodeBench/LiveCodeBench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"charxiv","benchmarkName":"CharXiv Reasoning","category":"multimodal","score":79.5,"unit":"percent","family":"charxiv","index":54.694825863948964,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":77.7,"unit":"percent","family":"lcr","index":60.6065062971765,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaIfBench","benchmarkName":"Artificial Analysis IFBench","category":"instruction","score":75.6,"unit":"percent","family":"ifbench-family","index":59.67263187317533,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmmuPro","benchmarkName":"Massive Multi-discipline Multimodal Understanding Pro","category":"multimodal","score":75,"unit":"percent","family":"mmmu-pro","index":52.75873491177307,"aboveFit":false,"run":null,"source":"MMMU-Pro authors","sourceUrl":"https://arxiv.org/abs/2409.02813","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaMmmuPro","benchmarkName":"Artificial Analysis MMMU-Pro","category":"multimodal","score":75,"unit":"percent","family":"mmmu-pro","index":52.75873491177307,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmmu-pro","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmvu","benchmarkName":"Multimodal Multi-disciplinary Video Understanding","category":"multimodal","score":73.3,"unit":"percent","family":"mmvu","index":null,"aboveFit":false,"run":null,"source":"MMVU benchmark maintainers","sourceUrl":"https://www.kimi.com/blog/kimi-k2-5.html","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"sweVerified","benchmarkName":"Software Engineering Benchmark Verified","category":"coding","score":72.4,"unit":"percent","family":"swe-bench-verified","index":54.62709616101095,"aboveFit":false,"run":null,"source":"Carlos E. Jimenez, John Yang, Alexander Wettig, Shunyu Yao, Kexin Pei, Ofir Press, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2310.06770","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"superGpqa","benchmarkName":"SuperGPQA: Scaling LLM Evaluation Across 285 Graduate Disciplines","category":"knowledge","score":65.6,"unit":"percent","family":"superGpqa","index":50.976999762252554,"aboveFit":false,"run":null,"source":"Xiaoxuan Du, Yao Yao, Kexin Ma, Bowen Wang, Tianyu Zheng, Kaiyan Zhu, Yiming Zhang, Yutao Zhu, Jiawei Zhou, Jingren Zhou","sourceUrl":"https://arxiv.org/abs/2502.14739","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"browseComp","benchmarkName":"BrowseComp","category":"agentic","score":61,"unit":"percent","family":"browseComp","index":48.10020221655606,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"longBenchV2","benchmarkName":"LongBench v2","category":"reasoning","score":60.6,"unit":"percent","family":"longBenchV2","index":52.19790876223265,"aboveFit":false,"run":null,"source":"LongBench v2 authors","sourceUrl":"https://arxiv.org/abs/2412.15204","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"sweRebench","benchmarkName":"SWE-Rebench","category":"coding","score":58.9,"unit":"percent","family":"sweRebench","index":null,"aboveFit":false,"run":null,"source":"Nebius","sourceUrl":"https://swe-rebench.com","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"osWorldVerified","benchmarkName":"OSWorld-Verified","category":"agentic","score":56.2,"unit":"percent","family":"osWorldVerified","index":45.84885746309052,"aboveFit":false,"run":null,"source":"Tianbao Xie, Danyang Zhang, Jixuan Chen, Xiaochuan Li, Siheng Zhao, Ruisheng Cao, Toh Jing Hua, Zhoujun Cheng, Dongchan Shin, Fangyu Lei, Yitao Liu, Yiheng Xu, Shuyan Zhou, Silvio Savarese, Caiming Xiong, Victor Zhong, Tao Yu","sourceUrl":"https://os-world.github.io/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"terminalBench2","benchmarkName":"Terminal-Bench 2.0","category":"agentic","score":41.6,"unit":"percent","family":"terminalBench2","index":43.48961109881995,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gertLabs","benchmarkName":"Gert Labs Composite Game Benchmark","category":"agentic","score":39.41,"unit":"percent","family":"gertLabs","index":47.45769544631012,"aboveFit":false,"run":null,"source":"Gert Labs","sourceUrl":"https://gertlabs.com/rankings","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaItbench","benchmarkName":"Artificial Analysis ITBench-AA","category":"agentic","score":35.5,"unit":"percent","family":"aaItbench","index":51.959670572632135,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/itbench-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":23.9,"unit":"percent","family":"hle","index":48.36534091952556,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"omniscienceAccuracy","benchmarkName":"Artificial Analysis Omniscience Accuracy","category":"knowledge","score":20.7,"unit":"percent","family":"omniscienceAccuracy","index":36.777249336656986,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":0.9,"unit":"percent","family":"critpt","index":38.5914109978019,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":0.26,"outputPerMillion":2.6,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":69,"timeToFirstToken":1.344,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":{"elo":1408.8173537842285,"eloUpper":1413.098814069332,"eloLower":1404.5358934991255,"votes":28424,"rank":144,"rankUpper":null,"rankLower":null,"arenaName":"Qwen3.5-27B","organization":"alibaba","updatedAt":"2026-11-09"},"quality":{"coveragePercent":90,"benchmarkCount":24,"categoryCount":7,"confidence":"medium","state":"available"},"hosts":[{"host":"chutes","name":"Chutes","kind":"host","url":"https://llm.chutes.ai/v1/models","inputPerMillion":0.2,"outputPerMillion":1.75,"cacheReadPerMillion":0.02,"contextTokens":262144,"maxOutputTokens":262144,"deprecated":false},{"host":"siliconflow","name":"SiliconFlow","kind":"host","url":"https://cloud.siliconflow.com/models","inputPerMillion":0.25,"outputPerMillion":2,"cacheReadPerMillion":null,"contextTokens":262144,"maxOutputTokens":262144,"deprecated":false},{"host":"novita-ai","name":"Novita AI","kind":"host","url":"https://novita.ai/docs/guides/introduction","inputPerMillion":0.3,"outputPerMillion":2.4,"cacheReadPerMillion":null,"contextTokens":262144,"maxOutputTokens":65536,"deprecated":false},{"host":"deepinfra","name":"Deep Infra","kind":"host","url":"https://deepinfra.com/models","inputPerMillion":0.26,"outputPerMillion":2.6,"cacheReadPerMillion":null,"contextTokens":262144,"maxOutputTokens":65536,"deprecated":false},{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":0.26,"outputPerMillion":2.6,"cacheReadPerMillion":null,"contextTokens":262144,"maxOutputTokens":81920,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"minimax-m2-7","providerId":"minimax","slug":"minimax-m2-7","name":"MiniMax M2.7","provider":"MiniMax","description":"MiniMax M2.7 is a standard model from MiniMax. 30 benchmarks count toward its index, in 6 categories.","logoUrl":null,"releaseDate":"2026-03-18","contextWindowTokens":196608,"modalities":["text"],"openWeights":true,"details":{"apiId":"minimax/minimax-m2.7","maxOutputTokens":176947,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":false,"cacheReadPerMillion":0.041999999999999996,"weightsUrl":"https://huggingface.co/MiniMaxAI/MiniMax-M2.7"},"reasoning":false,"overallScore":51.20610127513925,"overallScoreRange":{"low":46.7538766901864,"high":55.65832586009209},"badges":[],"capabilityScores":{"agentic":52.042358854031605,"coding":52.336921473546774,"reasoning":49.51440030766007,"multimodal":null,"knowledge":50.19837472612446,"multilingual":null,"instruction":59.766082934658286,"math":47.73995141744886},"capabilityCounts":{"math":4,"knowledge":4,"agentic":10,"coding":9,"reasoning":2,"instruction":1},"benchmarks":[{"benchmarkId":"vals-ai:aime:overall","benchmarkName":"AIME","category":"math","domain":null,"eligible":true,"ranked":true,"score":91.042,"unit":"percent","family":"vals-ai:aime:overall","index":55.29208611169544,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/aime","measuredAt":"2026-04-16","stderr":0.734,"costPerTest":0.022959,"latency":236.637},{"benchmarkId":"aime2026","benchmarkName":"AIME 2026","category":"math","score":89.8,"unit":"percent","family":"aime2026","index":49.94163519643721,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":87.4,"unit":"percent","family":"gpqa-diamond","index":57.36999252583875,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gpqaDiamond","benchmarkName":"GPQA Diamond","category":"knowledge","score":87,"unit":"percent","family":"gpqa-diamond","index":57.00312345587083,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:gpqa:overall","benchmarkName":"GPQA Diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":86.616,"unit":"percent","family":"gpqa-diamond","index":56.65092914870163,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","measuredAt":"2026-09-01","stderr":1.953,"costPerTest":0.014051,"latency":171.727},{"benchmarkId":"tau2Bench","benchmarkName":"τ²-Bench Tool-Agent-User Evaluation","category":"agentic","score":84.8,"unit":"percent","family":"tau2-bench","index":59.06044003629105,"aboveFit":false,"run":null,"source":"Victor Barres, Honghua Dong, Soham Ray, Xujie Si, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2506.07982","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:legal_bench:overall","benchmarkName":"LegalBench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":83.98,"unit":"percent","family":"vals-ai:legal_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","measuredAt":"2026-10-13","stderr":0.411,"costPerTest":0.000626,"latency":8.916},{"benchmarkId":"hmmtNov2025","benchmarkName":"Harvard-MIT Mathematics Tournament November 2025","category":"math","score":81,"unit":"percent","family":"hmmtNov2025","index":41.44856222322146,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmluProArcee","benchmarkName":"MMLU-Pro first-party comparison snapshot","category":"knowledge","score":80.8,"unit":"percent","family":"mmlu-pro","index":46.279056915559146,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:mmlu_pro:overall","benchmarkName":"MMLU Pro","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":80.427,"unit":"percent","family":"mmlu-pro","index":45.80912156251148,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","measuredAt":"2026-09-01","stderr":0.39,"costPerTest":0.003947,"latency":50.005},{"benchmarkId":"aime2025Arcee","benchmarkName":"AIME25 first-party comparison snapshot","category":"math","score":80,"unit":"percent","family":"aime2025Arcee","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:lcb:overall","benchmarkName":"LiveCodeBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":79.926,"unit":"percent","family":"livecodebench","index":55.661663972771706,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","measuredAt":"2026-09-01","stderr":1.096,"costPerTest":0.02142,"latency":318.934},{"benchmarkId":"vals-ai:medscribe:overall","benchmarkName":"MedScribe","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":79.867,"unit":"percent","family":"vals-ai:medscribe:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medscribe","measuredAt":"2026-10-29","stderr":1.86,"costPerTest":0.005124,"latency":27.25},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":78.3,"unit":"percent","family":"lcr","index":61.018154612527894,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaIfBench","benchmarkName":"Artificial Analysis IFBench","category":"instruction","score":75.7,"unit":"percent","family":"ifbench-family","index":59.766082934658286,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"sweVerifiedArcee","benchmarkName":"SWE-bench Verified (mini-swe-agent-v2)","category":"coding","score":75.4,"unit":"percent","family":"swe-bench-verified","index":56.839768894362024,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:swebench:overall","benchmarkName":"SWE-bench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":73.8,"unit":"percent","family":"swe-bench-verified","index":55.65967676990811,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/swebench","measuredAt":"2026-09-01","stderr":1.968,"costPerTest":0.466052,"latency":885.69},{"benchmarkId":"hmmtFeb2026","benchmarkName":"Harvard-MIT Mathematics Tournament February 2026","category":"math","score":72.7,"unit":"percent","family":"hmmtFeb2026","index":44.27752213844132,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"reactNativeEvals","benchmarkName":"React Native Evals","category":"coding","score":71.4,"unit":"percent","family":"reactNativeEvals","index":47.48090056425324,"aboveFit":false,"run":null,"source":"Callstack","sourceUrl":"https://rn-evals.vercel.app/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"tau3Bench","benchmarkName":"τ³-Bench Tool-Agent-User Evaluation","category":"agentic","score":67.6,"unit":"percent","family":"tau3Bench","index":57.23294379140429,"aboveFit":false,"run":null,"source":"Sierra Research","sourceUrl":"https://github.com/sierra-research/tau2-bench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mleBenchLite","benchmarkName":"MLE-Bench Lite","category":"agentic","score":66.6,"unit":"percent","family":"mleBenchLite","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:tax_eval_v2:overall","benchmarkName":"TaxEval v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":66.558,"unit":"percent","family":"vals-ai:tax_eval_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","measuredAt":"2026-09-01","stderr":0.92,"costPerTest":0.003474,"latency":28.694},{"benchmarkId":"mmClawBench","benchmarkName":"MM-ClawBench","category":"agentic","score":62.7,"unit":"percent","family":"mmClawBench","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:corp_fin_v2:overall","benchmarkName":"CorpFin v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":61.189,"unit":"percent","family":"vals-ai:corp_fin_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","measuredAt":"2026-08-12","stderr":0.96,"costPerTest":0.022663,"latency":10.973},{"benchmarkId":"vals-ai:case_law_v2:overall","benchmarkName":"CaseLaw v2","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":60.879,"unit":"percent","family":"vals-ai:case_law_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/case_law_v2","measuredAt":"2026-05-04","stderr":0.773,"costPerTest":0.003091,"latency":21.639},{"benchmarkId":"terminalBench2","benchmarkName":"Terminal-Bench 2.0","category":"agentic","score":57,"unit":"percent","family":"terminalBench2","index":54.444331804530655,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"swePro","benchmarkName":"SWE-bench Pro","category":"coding","score":56.2,"unit":"percent","family":"swePro","index":56.69986461667817,"aboveFit":false,"run":null,"source":"Xiang Deng, Jeff Da, Edwin Pan, Yannis Yiming He, Charles Ide, Kanak Garg, Niklas Lauffer, Andrew Park, Nitin Pasari, Chetan Rane, Karmini Sampath, Maya Krishnan, Srivatsa Kundurthy, Sean Hendryx, Zifan Wang, Chen Bo Calvin Zhang, Noah Jacobson, Bing Liu, Brad Kenstler","sourceUrl":"https://arxiv.org/abs/2509.16941","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vibePro","benchmarkName":"VIBE-Pro","category":"coding","score":55.6,"unit":"percent","family":"vibePro","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"multiSweBench","benchmarkName":"Multi-SWE Bench","category":"coding","score":52.7,"unit":"percent","family":"multiSweBench","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"sweRebench","benchmarkName":"SWE-Rebench","category":"coding","score":51.9,"unit":"percent","family":"sweRebench","index":null,"aboveFit":false,"run":null,"source":"Nebius","sourceUrl":"https://swe-rebench.com","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaSciCode","benchmarkName":"Artificial Analysis SciCode","category":"coding","score":50.1,"unit":"percent","family":"scicode-family","index":61.15472487788939,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/scicode","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"clawEval","benchmarkName":"Claw-Eval","category":"agentic","score":48.7,"unit":"percent","family":"clawEval","index":35.525746153279314,"aboveFit":false,"run":null,"source":"Bowen Ye, Rang Li, Qibin Yang, Yuanxin Liu, Linli Yao, Hanglong Lv, Zhihui Xie, Chenxin An, Lei Li, Lingpeng Kong, Qi Liu, Zhifang Sui, Tong Yang","sourceUrl":"https://arxiv.org/abs/2604.06132","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:terminal-bench-2-1:overall","benchmarkName":"Terminal-Bench 2.1","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":48.689,"unit":"percent","family":"terminal-bench-2-1","index":49.61714404817983,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2-1","measuredAt":"2026-10-01","stderr":3.066,"costPerTest":0.066012,"latency":812.638},{"benchmarkId":"vals-ai:finance_agent:overall","benchmarkName":"Finance Agent v1.1","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":48.402,"unit":"percent","family":"vals-ai:finance_agent:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/finance_agent","measuredAt":"2026-06-04","stderr":2.791,"costPerTest":0.160333,"latency":1292.755},{"benchmarkId":"vals-ai:terminal-bench-2:overall","benchmarkName":"Terminal-Bench 2.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":47.191,"unit":"percent","family":"vals-ai:terminal-bench-2:overall","index":54.96549677926841,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2","measuredAt":"2026-06-04","stderr":5.322,"costPerTest":0.073995,"latency":839.048},{"benchmarkId":"toolathlon","benchmarkName":"Toolathlon","category":"agentic","score":46.3,"unit":"percent","family":"toolathlon","index":59.69225517976294,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gertLabs","benchmarkName":"Gert Labs Composite Game Benchmark","category":"agentic","score":40.4,"unit":"percent","family":"gertLabs","index":48.25931506489766,"aboveFit":false,"run":null,"source":"Gert Labs","sourceUrl":"https://gertlabs.com/rankings","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"nl2Repo","benchmarkName":"NL2Repo","category":"coding","score":39.8,"unit":"percent","family":"nl2Repo","index":54.84347062117131,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:medcode:overall","benchmarkName":"MedCode","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":34.44,"unit":"percent","family":"vals-ai:medcode:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medcode","measuredAt":"2026-10-29","stderr":1.985,"costPerTest":0.007424,"latency":30.252},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":29.6,"unit":"percent","family":"hle","index":54.084109566195394,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:emb:overall","benchmarkName":"EMB","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":28.451,"unit":"percent","family":"vals-ai:emb:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/emb","measuredAt":"2026-10-20","stderr":2.431,"costPerTest":1.658686,"latency":1207.259},{"benchmarkId":"hle","benchmarkName":"Humanity's Last Exam","category":"knowledge","score":28,"unit":"percent","family":"hle","index":52.47884117414772,"aboveFit":false,"run":null,"source":"Center for AI Safety, Scale AI, and thousands of expert contributors","sourceUrl":"https://lastexam.ai/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:fabv2:overall","benchmarkName":"Finance Agent (v2)","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":27.887,"unit":"percent","family":"vals-ai:fabv2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/fabv2","measuredAt":"2026-10-29","stderr":0.508,"costPerTest":0.18348,"latency":803.528},{"benchmarkId":"omniscienceAccuracy","benchmarkName":"Artificial Analysis Omniscience Accuracy","category":"knowledge","score":26.8,"unit":"percent","family":"omniscienceAccuracy","index":44.46481083942015,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gdpvalAaNormalized","benchmarkName":"GDPval-AA normalized","category":"agentic","score":25.5,"unit":"percent","family":"gdpvalAaNormalized","index":52.2656823403345,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:vibe-code:overall","benchmarkName":"Vibe Code Bench v1.1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":11.926,"unit":"percent","family":"vals-ai:vibe-code:overall","index":46.552610762474245,"aboveFit":false,"run":"OpenHands","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","measuredAt":"2026-11-09","stderr":2.857,"costPerTest":2.819109,"latency":1377.326},{"benchmarkId":"apexAgentsAa","benchmarkName":"APEX-Agents-AA","category":"agentic","score":10.6,"unit":"percent","family":"apexAgentsAa","index":49.36023334236739,"aboveFit":false,"run":null,"source":"Artificial Analysis / Mercor","sourceUrl":"https://artificialanalysis.ai/evaluations/apex-agents-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:legal_research:overall","benchmarkName":"Legal Research Bench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":10.577,"unit":"percent","family":"vals-ai:legal_research:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_research","measuredAt":"2026-11-09","stderr":2.138,"costPerTest":0.219167,"latency":420.647},{"benchmarkId":"vals-ai:code-migration:overall","benchmarkName":"Code Migration","category":"coding","domain":null,"eligible":true,"ranked":true,"score":8.685,"unit":"percent","family":"vals-ai:code-migration:overall","index":49.802957658431616,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/code-migration","measuredAt":"2026-10-29","stderr":1.786,"costPerTest":0.587624,"latency":1259.318},{"benchmarkId":"vals-ai:tax_agent_bench:overall","benchmarkName":"Tax Agent Bench","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":6.872,"unit":"percent","family":"vals-ai:tax_agent_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench","measuredAt":"2026-11-09","stderr":1.677,"costPerTest":0.120058,"latency":302.854},{"benchmarkId":"vals-ai:ioi-v1:overall","benchmarkName":"IOI v1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":4.916,"unit":"percent","family":"vals-ai:ioi-v1:overall","index":42.586377356116245,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/ioi-v1","measuredAt":"2026-08-09","stderr":2.379,"costPerTest":1.154876,"latency":3242.758},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":0.6,"unit":"percent","family":"critpt","index":38.010646002792235,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:programbench:overall","benchmarkName":"ProgramBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":0,"unit":"percent","family":"vals-ai:programbench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/programbench","measuredAt":"2026-11-09","stderr":0,"costPerTest":null,"latency":1460.176}],"price":{"inputPerMillion":0.21,"outputPerMillion":0.84,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":35,"timeToFirstToken":1.226,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":{"elo":1404.718127762354,"eloUpper":1408.002510675392,"eloLower":1401.433744849316,"votes":89504,"rank":151,"rankUpper":null,"rankLower":null,"arenaName":"MiniMax-M2.7","organization":"minimax","updatedAt":"2026-11-09"},"quality":{"coveragePercent":85,"benchmarkCount":30,"categoryCount":6,"confidence":"medium","state":"available"},"hosts":[{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":0.21,"outputPerMillion":0.84,"cacheReadPerMillion":0.042,"contextTokens":196608,"maxOutputTokens":176947,"deprecated":false},{"host":"deepinfra","name":"Deep Infra","kind":"host","url":"https://deepinfra.com/models","inputPerMillion":0.25,"outputPerMillion":1,"cacheReadPerMillion":0.05,"contextTokens":196608,"maxOutputTokens":131072,"deprecated":true},{"host":"gmi-cloud","name":"GMI Cloud","kind":"host","url":"https://docs.gmicloud.ai/inference-engine/api-reference/llm-api-reference","inputPerMillion":0.3,"outputPerMillion":1.2,"cacheReadPerMillion":0.06,"contextTokens":196608,"maxOutputTokens":131072,"deprecated":false},{"host":"novita-ai","name":"Novita AI","kind":"host","url":"https://novita.ai/docs/guides/introduction","inputPerMillion":0.3,"outputPerMillion":1.2,"cacheReadPerMillion":0.06,"contextTokens":204800,"maxOutputTokens":131072,"deprecated":false},{"host":"together-ai","name":"Together AI","kind":"host","url":"https://docs.together.ai/docs/serverless-models","inputPerMillion":0.3,"outputPerMillion":1.2,"cacheReadPerMillion":0.06,"contextTokens":196608,"maxOutputTokens":131072,"deprecated":true}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"nemotron-3-5-super-vl-120b-a12b","providerId":"nvidia","slug":"nemotron-3-5-super-vl-120b-a12b","name":"Nemotron 3.5 Super VL 120B A12B","provider":"NVIDIA","description":"Nemotron 3.5 Super VL 120B A12B is a reasoning model from NVIDIA in the Nemotron 3.5 Super VL family. 20 benchmarks count toward its index, in 6 categories.","logoUrl":null,"releaseDate":"2026-10-15","contextWindowTokens":1000000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":true,"overallScore":51.04274101079333,"overallScoreRange":{"low":45.92701909117617,"high":56.15846293041049},"badges":[],"capabilityScores":{"agentic":47.779406426259065,"coding":53.039827310629654,"reasoning":56.12485197849077,"multimodal":47.39624933172859,"knowledge":52.73692914379376,"multilingual":null,"instruction":55.24305155888349,"math":null},"capabilityCounts":{"knowledge":3,"coding":5,"multimodal":4,"agentic":5,"instruction":1,"reasoning":2},"benchmarks":[{"benchmarkId":"gpqaDiamond","benchmarkName":"GPQA Diamond","category":"knowledge","score":89.52,"unit":"percent","family":"gpqa-diamond","index":59.31439859666868,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"apexShortlist","benchmarkName":"Apex Shortlist","category":"math","score":88.36,"unit":"percent","family":"apexShortlist","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"liveCodeBenchV6","benchmarkName":"LiveCodeBench v6","category":"coding","score":88.22,"unit":"percent","family":"liveCodeBenchV6","index":56.85073262286025,"aboveFit":false,"run":null,"source":"LiveCodeBench maintainers","sourceUrl":"https://github.com/LiveCodeBench/LiveCodeBench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vStar","benchmarkName":"V*","category":"multimodal","score":85.6,"unit":"percent","family":"vStar","index":44.20704423077334,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"charxiv","benchmarkName":"CharXiv Reasoning","category":"multimodal","score":80.82,"unit":"percent","family":"charxiv","index":56.19764378569715,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mathVision","benchmarkName":"MathVision","category":"multimodal","score":78.16,"unit":"percent","family":"mathVision","index":49.60990254493547,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"browseComp","benchmarkName":"BrowseComp","category":"agentic","score":78,"unit":"percent","family":"browseComp","index":61.53053891854455,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"tau2Airline","benchmarkName":"τ²-Bench Airline Domain","category":"agentic","score":76.5,"unit":"percent","family":"tau2Airline","index":null,"aboveFit":false,"run":null,"source":"Victor Barres, Honghua Dong, Soham Ray, Xujie Si, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2506.07982","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"sweVerified","benchmarkName":"Software Engineering Benchmark Verified","category":"coding","score":71.1,"unit":"percent","family":"swe-bench-verified","index":53.66827130989214,"aboveFit":false,"run":null,"source":"Carlos E. Jimenez, John Yang, Alexander Wettig, Shunyu Yao, Kexin Pei, Ofir Press, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2310.06770","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"ifBench","benchmarkName":"Instruction Following Benchmark","category":"instruction","score":70.86,"unit":"percent","family":"ifbench-family","index":55.24305155888349,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":68.62,"unit":"percent","family":"lcr","index":54.37689512485871,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"terminalBench21","benchmarkName":"Terminal-Bench 2.1","category":"agentic","score":66.7,"unit":"percent","family":"terminal-bench-2-1","index":60.15097401770262,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmLongBenchDoc","benchmarkName":"MMLongBench-Doc","category":"multimodal","score":62.82,"unit":"percent","family":"mmLongBenchDoc","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"swePro","benchmarkName":"SWE-bench Pro","category":"coding","score":51.78,"unit":"percent","family":"swePro","index":52.935748499827284,"aboveFit":false,"run":null,"source":"Xiang Deng, Jeff Da, Edwin Pan, Yannis Yiming He, Charles Ide, Kanak Garg, Niklas Lauffer, Andrew Park, Nitin Pasari, Chetan Rane, Karmini Sampath, Maya Krishnan, Srivatsa Kundurthy, Sean Hendryx, Zifan Wang, Chen Bo Calvin Zhang, Noah Jacobson, Bing Liu, Brad Kenstler","sourceUrl":"https://arxiv.org/abs/2509.16941","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaSciCode","benchmarkName":"Artificial Analysis SciCode","category":"coding","score":47.8,"unit":"percent","family":"scicode-family","index":58.156235241135505,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/scicode","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gdpvalAaNormalized","benchmarkName":"GDPval-AA normalized","category":"agentic","score":33.1,"unit":"percent","family":"gdpvalAaNormalized","index":57.699505729192424,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hleNoTools","benchmarkName":"Humanity's Last Exam without tools","category":"knowledge","score":31.79,"unit":"percent","family":"hle","index":56.28132067781064,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":30.7,"unit":"percent","family":"hle","index":55.18773158572816,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"babyVision","benchmarkName":"BabyVision","category":"multimodal","score":29.6,"unit":"percent","family":"babyVision","index":39.570406765508395,"aboveFit":false,"run":null,"source":"Meta AI","sourceUrl":"https://ai.meta.com/static-resource/muse-spark-1-1-evaluation-report","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hle","benchmarkName":"Humanity's Last Exam","category":"knowledge","score":29.2,"unit":"percent","family":"hle","index":53.68279246818347,"aboveFit":false,"run":null,"source":"Center for AI Safety, Scale AI, and thousands of expert contributors","sourceUrl":"https://lastexam.ai/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"omniscienceAccuracy","benchmarkName":"Artificial Analysis Omniscience Accuracy","category":"knowledge","score":26.2,"unit":"percent","family":"omniscienceAccuracy","index":43.70865724898444,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"tau3Bench","benchmarkName":"τ³-Bench Tool-Agent-User Evaluation","category":"agentic","score":18.88,"unit":"percent","family":"tau3Bench","index":5,"aboveFit":false,"run":null,"source":"Sierra Research","sourceUrl":"https://github.com/sierra-research/tau2-bench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:tax_agent_bench:overall","benchmarkName":"Tax Agent Bench","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":11.554,"unit":"percent","family":"vals-ai:tax_agent_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench","measuredAt":"2026-11-09","stderr":2.194,"costPerTest":0.046335,"latency":72.661},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":10.86,"unit":"percent","family":"critpt","index":57.87280883212282,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:legal_research:overall","benchmarkName":"Legal Research Bench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":10.096,"unit":"percent","family":"vals-ai:legal_research:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_research","measuredAt":"2026-11-09","stderr":2.094,"costPerTest":0.063761,"latency":82.971},{"benchmarkId":"vals-ai:hlab:overall","benchmarkName":"HLAB-Vals-1.1","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":3.75,"unit":"percent","family":"vals-ai:hlab:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/hlab","measuredAt":"2026-11-09","stderr":1.344,"costPerTest":0.373217,"latency":172.109},{"benchmarkId":"vals-ai:vibe-code:overall","benchmarkName":"Vibe Code Bench v1.1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":3.71,"unit":"percent","family":"vals-ai:vibe-code:overall","index":43.588148879433085,"aboveFit":false,"run":"OpenHands","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","measuredAt":"2026-11-09","stderr":1.926,"costPerTest":0.539233,"latency":262.905},{"benchmarkId":"vals-ai:terminal-bench-4:overall","benchmarkName":"Terminal-Bench 4.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":0,"unit":"percent","family":"terminal-bench-4","index":54.516013465855735,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-4","measuredAt":"2026-11-09","stderr":0,"costPerTest":0.862751,"latency":895.649}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":85,"benchmarkCount":20,"categoryCount":6,"confidence":"medium","state":"available"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gpt-5-4-nano","providerId":"openai","slug":"gpt-5-4-nano","name":"GPT-5.4 nano","provider":"OpenAI","description":"GPT-5.4 nano is a reasoning model from OpenAI in the GPT-5.4 family. 43 benchmarks count toward its index, in 8 categories.","logoUrl":null,"releaseDate":"2026-03-17","contextWindowTokens":400000,"modalities":["file","image","text"],"openWeights":false,"details":{"apiId":"openai/gpt-5.4-nano","maxOutputTokens":128000,"knowledgeCutoff":"2025-08-31","retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.02,"weightsUrl":null},"reasoning":true,"overallScore":50.94203320766292,"overallScoreRange":{"low":48.4761313253847,"high":53.407935089941134},"badges":[],"capabilityScores":{"agentic":50.546174729586205,"coding":53.693382901200295,"reasoning":53.69891150849923,"multimodal":41.72810362497825,"knowledge":44.522647855303134,"multilingual":44.419238513730285,"instruction":61.26213974237932,"math":55.851037407163275},"capabilityCounts":{"agentic":10,"math":7,"coding":7,"knowledge":6,"reasoning":7,"instruction":2,"multimodal":3,"multilingual":1},"benchmarks":[{"benchmarkId":"tau2Bench","benchmarkName":"τ²-Bench Tool-Agent-User Evaluation","category":"agentic","score":92.5,"unit":"percent","family":"tau2-bench","index":64.16995299808741,"aboveFit":false,"run":null,"source":"Victor Barres, Honghua Dong, Soham Ray, Xujie Si, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2506.07982","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-math:overall","benchmarkName":"LiveBench Mathematics","category":"math","domain":null,"eligible":true,"ranked":true,"score":90.977,"unit":"percent","family":"livebench:livebench-math:overall","index":66.57122441687595,"aboveFit":false,"run":"xhigh effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Mathematics","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.025875862068965517,"latency":null},{"benchmarkId":"vals-ai:aime:overall","benchmarkName":"AIME","category":"math","domain":null,"eligible":true,"ranked":true,"score":88.75,"unit":"percent","family":"vals-ai:aime:overall","index":54.290377228779015,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/aime","measuredAt":"2026-04-16","stderr":0.99,"costPerTest":0.003432,"latency":65.876},{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":87.78,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":59.98897221101503,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":3.94,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:lcb:overall","benchmarkName":"LiveCodeBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":84.009,"unit":"percent","family":"livecodebench","index":59.0494974107899,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","measuredAt":"2026-09-01","stderr":1.044,"costPerTest":0.002468,"latency":60.089},{"benchmarkId":"gpqa","benchmarkName":"Graduate-Level Google-Proof Q&A","category":"knowledge","score":82.8,"unit":"percent","family":"gpqa-diamond","index":53.15099822120773,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":81.7,"unit":"percent","family":"gpqa-diamond","index":52.14210827879597,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-reasoning:overall","benchmarkName":"LiveBench Reasoning","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":81.09700000000001,"unit":"percent","family":"livebench","index":66.67553212091671,"aboveFit":false,"run":"xhigh effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Reasoning","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.044406701030927836,"latency":null},{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":78.47,"unit":"percent","family":"gpqa-diamond","index":49.17964053880506,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.44,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:legal_bench:overall","benchmarkName":"LegalBench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":77.92,"unit":"percent","family":"vals-ai:legal_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","measuredAt":"2026-10-13","stderr":0.427,"costPerTest":0.000114,"latency":2.697},{"benchmarkId":"vals-ai:gpqa:overall","benchmarkName":"GPQA Diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":77.526,"unit":"percent","family":"gpqa-diamond","index":48.31382953368078,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","measuredAt":"2026-09-01","stderr":2.417,"costPerTest":0.001565,"latency":29.498},{"benchmarkId":"vals-ai:mmlu_pro:overall","benchmarkName":"MMLU Pro","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":77.172,"unit":"percent","family":"mmlu-pro","index":41.708211175996254,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","measuredAt":"2026-09-01","stderr":0.433,"costPerTest":0.000396,"latency":8.21},{"benchmarkId":"vals-ai:medscribe:overall","benchmarkName":"MedScribe","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":77.09,"unit":"percent","family":"vals-ai:medscribe:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medscribe","measuredAt":"2026-10-29","stderr":1.891,"costPerTest":0.0018,"latency":20.527},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":76.7,"unit":"percent","family":"lcr","index":59.92042577159084,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaIfBench","benchmarkName":"Artificial Analysis IFBench","category":"instruction","score":75.9,"unit":"percent","family":"ifbench-family","index":59.95298505762419,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"extractBenchLlamaIndex","benchmarkName":"ExtractBench: schema-guided enterprise document extraction (LlamaIndex leaderboard)","category":"multimodal","score":74.9,"unit":"percent","family":"extractBenchLlamaIndex","index":51.3562500691128,"aboveFit":false,"run":null,"source":"Boyang Zhang, Adrian Lyjak, Eli Stewart, Zhaoqi Li and Simon Suo (LlamaIndex)","sourceUrl":"https://arxiv.org/abs/2607.29677","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:mmmu:overall","benchmarkName":"MMMU Pro","category":"multimodal","domain":null,"eligible":true,"ranked":true,"score":73.584,"unit":"percent","family":"mmmu-pro","index":50.91593514436433,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmmu","measuredAt":"2026-09-01","stderr":1.06,"costPerTest":0.000844,"latency":21.907},{"benchmarkId":"livebench:livebench-coding:overall","benchmarkName":"LiveBench Coding","category":"coding","domain":null,"eligible":true,"ranked":true,"score":70.83850000000001,"unit":"percent","family":"livebench:livebench-coding:overall","index":54.45226522005266,"aboveFit":false,"run":"xhigh effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Coding","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.014964655172413794,"latency":null},{"benchmarkId":"vals-ai:swebench:overall","benchmarkName":"SWE-bench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":69.8,"unit":"percent","family":"swe-bench-verified","index":52.70944645877334,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/swebench","measuredAt":"2026-09-01","stderr":2.055,"costPerTest":0.097712,"latency":262.552},{"benchmarkId":"mmmuProPython","benchmarkName":"MMMU-Pro with Python","category":"multimodal","score":69.5,"unit":"percent","family":"mmmuProPython","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-data-analysis:overall","benchmarkName":"LiveBench Data Analysis","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":67.64266666666667,"unit":"percent","family":"livebench","index":49.24633132542888,"aboveFit":false,"run":"xhigh effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Data%20Analysis","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.04483150684931507,"latency":null},{"benchmarkId":"vals-ai:tax_eval_v2:overall","benchmarkName":"TaxEval v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":67.416,"unit":"percent","family":"vals-ai:tax_eval_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","measuredAt":"2026-09-01","stderr":0.926,"costPerTest":0.000638,"latency":15.898},{"benchmarkId":"livebench:livebench-instruction-following:overall","benchmarkName":"LiveBench Instruction Following","category":"instruction","domain":null,"eligible":true,"ranked":true,"score":67.20475,"unit":"percent","family":"livebench:livebench-instruction-following:overall","index":62.57129442713445,"aboveFit":false,"run":"xhigh effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=IF","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.045197461928934014,"latency":null},{"benchmarkId":"mmmuPro","benchmarkName":"Massive Multi-discipline Multimodal Understanding Pro","category":"multimodal","score":66.1,"unit":"percent","family":"mmmu-pro","index":41.17616575221249,"aboveFit":false,"run":null,"source":"MMMU-Pro authors","sourceUrl":"https://arxiv.org/abs/2409.02813","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaMmmuPro","benchmarkName":"Artificial Analysis MMMU-Pro","category":"multimodal","score":65.4,"unit":"percent","family":"mmmu-pro","index":40.26517716662909,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmmu-pro","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-language:overall","benchmarkName":"LiveBench Language","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":62.507000000000005,"unit":"percent","family":"livebench:livebench-language:overall","index":45.24225812890793,"aboveFit":false,"run":"xhigh effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Language","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.03685277777777777,"latency":null},{"benchmarkId":"vals-ai:corp_fin_v2:overall","benchmarkName":"CorpFin v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":61.189,"unit":"percent","family":"vals-ai:corp_fin_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","measuredAt":"2026-08-12","stderr":0.961,"costPerTest":0.01612,"latency":9.397},{"benchmarkId":"epoch-ai:epoch-frontiermath-2025-02-28-public:overall","benchmarkName":"FrontierMath-2025-02-28-Public","category":"math","domain":null,"eligible":false,"ranked":true,"score":60,"unit":"percent","family":"epoch-ai:epoch-frontiermath-2025-02-28-public:overall","index":null,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":16.33,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":59.1125,"unit":"percent","family":"euroeval","index":52.42229655975848,"aboveFit":false,"run":"high effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":0.6148361062104274,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:mortgage_tax:overall","benchmarkName":"MortgageTax","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":59.102,"unit":"percent","family":"vals-ai:mortgage_tax:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mortgage_tax","measuredAt":"2026-09-01","stderr":0.964,"costPerTest":0.002585,"latency":14.526},{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":57.7575,"unit":"percent","family":"euroeval","index":50.583873336376065,"aboveFit":false,"run":"medium effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":0.5498337817013429,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-portuguese:overall","benchmarkName":"EuroEval Portuguese","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":56.74999999999999,"unit":"percent","family":"euroeval","index":49.21692765552161,"aboveFit":false,"run":"high effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/portuguese","measuredAt":null,"stderr":0.46911652337132614,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":56.404999999999994,"unit":"percent","family":"euroeval","index":48.748842037760035,"aboveFit":false,"run":"low effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":0.677205886344766,"costPerTest":null,"latency":null},{"benchmarkId":"mcpAtlas","benchmarkName":"MCP Atlas","category":"agentic","score":56.1,"unit":"percent","family":"mcpAtlas","index":48.83977625123595,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-portuguese:overall","benchmarkName":"EuroEval Portuguese","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":55.82875000000001,"unit":"percent","family":"euroeval","index":47.96700337910757,"aboveFit":false,"run":"medium effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/portuguese","measuredAt":null,"stderr":0.5115097139840064,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-italian:overall","benchmarkName":"EuroEval Italian","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":55.743750000000006,"unit":"percent","family":"euroeval","index":47.85167793705037,"aboveFit":false,"run":"medium effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/italian","measuredAt":null,"stderr":0.49392243824309096,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":55.24714285714286,"unit":"percent","family":"euroeval","index":47.177896310241394,"aboveFit":false,"run":"high effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.46035478775857747,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-italian:overall","benchmarkName":"EuroEval Italian","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":55.05499999999999,"unit":"percent","family":"euroeval","index":46.917202663910395,"aboveFit":false,"run":"high effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/italian","measuredAt":null,"stderr":0.4825825706550123,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-french:overall","benchmarkName":"EuroEval French","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":54.885,"unit":"percent","family":"euroeval","index":46.686551779796005,"aboveFit":false,"run":"medium effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/french","measuredAt":null,"stderr":0.4753945729601885,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-french:overall","benchmarkName":"EuroEval French","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":54.821250000000006,"unit":"percent","family":"euroeval","index":46.60005769825312,"aboveFit":false,"run":"high effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/french","measuredAt":null,"stderr":0.4548248289176834,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-spanish:overall","benchmarkName":"EuroEval Spanish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":53.36875,"unit":"percent","family":"euroeval","index":44.629349408981554,"aboveFit":false,"run":"medium effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/spanish","measuredAt":null,"stderr":0.4454158871212386,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-spanish:overall","benchmarkName":"EuroEval Spanish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":53.29,"unit":"percent","family":"euroeval","index":44.522503778840324,"aboveFit":false,"run":"high effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/spanish","measuredAt":null,"stderr":0.5325264077958951,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-dutch:overall","benchmarkName":"EuroEval Dutch","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":53.13777777777778,"unit":"percent","family":"euroeval","index":44.31597324862024,"aboveFit":false,"run":"medium effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/dutch","measuredAt":null,"stderr":0.4172114956455124,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-french:overall","benchmarkName":"EuroEval French","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":53.035,"unit":"percent","family":"euroeval","index":44.17652745266872,"aboveFit":false,"run":"low effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/french","measuredAt":null,"stderr":0.47277934863951065,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-dutch:overall","benchmarkName":"EuroEval Dutch","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":52.986666666666665,"unit":"percent","family":"euroeval","index":44.11095024051855,"aboveFit":false,"run":"high effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/dutch","measuredAt":null,"stderr":0.45354297984634406,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-portuguese:overall","benchmarkName":"EuroEval Portuguese","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":52.71875,"unit":"percent","family":"euroeval","index":43.747448969720615,"aboveFit":false,"run":"low effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/portuguese","measuredAt":null,"stderr":0.535696100881834,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-italian:overall","benchmarkName":"EuroEval Italian","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":52.52749999999999,"unit":"percent","family":"euroeval","index":43.4879667250919,"aboveFit":false,"run":"low effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/italian","measuredAt":null,"stderr":0.4994184117551134,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":52.43,"unit":"percent","family":"euroeval","index":43.35568165920277,"aboveFit":false,"run":"medium effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.5739871292022113,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:case_law_v2:overall","benchmarkName":"CaseLaw v2","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":51.875,"unit":"percent","family":"vals-ai:case_law_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/case_law_v2","measuredAt":"2026-05-04","stderr":1.088,"costPerTest":0.001108,"latency":7.62},{"benchmarkId":"euroeval:euroeval-dutch:overall","benchmarkName":"EuroEval Dutch","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":51.57111111111111,"unit":"percent","family":"euroeval","index":42.190367061683624,"aboveFit":false,"run":"low effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/dutch","measuredAt":null,"stderr":0.4434586289109228,"costPerTest":null,"latency":null},{"benchmarkId":"arc-prize:v1-semi-private:overall","benchmarkName":"ARC-AGI-1 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":51.5,"unit":"percent","family":"arc-prize:v1-semi-private:overall","index":50.66376816337222,"aboveFit":false,"run":"xhigh effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.13,"latency":null},{"benchmarkId":"euroeval:euroeval-spanish:overall","benchmarkName":"EuroEval Spanish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":50.82124999999999,"unit":"percent","family":"euroeval","index":41.17297807203195,"aboveFit":false,"run":"low effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/spanish","measuredAt":null,"stderr":0.5445123965530996,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":50.675714285714285,"unit":"percent","family":"euroeval","index":40.975519594560076,"aboveFit":false,"run":"low effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.43086765807555133,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":49.160000000000004,"unit":"percent","family":"euroeval","index":38.91904406476699,"aboveFit":false,"run":"medium effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.5961215794617739,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":48.688750000000006,"unit":"percent","family":"euroeval","index":38.27966624630282,"aboveFit":false,"run":"high effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.5693115359800818,"costPerTest":null,"latency":null},{"benchmarkId":"arc-prize:v1-public-eval:overall","benchmarkName":"ARC-AGI-1 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":47.88,"unit":"percent","family":"arc-prize:v1-public-eval:overall","index":null,"aboveFit":false,"run":"xhigh effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.1,"latency":null},{"benchmarkId":"vals-ai:finance_agent:overall","benchmarkName":"Finance Agent v1.1","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":47.801,"unit":"percent","family":"vals-ai:finance_agent:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/finance_agent","measuredAt":"2026-06-04","stderr":2.828,"costPerTest":0.012919,"latency":283.802},{"benchmarkId":"aaSciCode","benchmarkName":"Artificial Analysis SciCode","category":"coding","score":47.2,"unit":"percent","family":"scicode-family","index":57.37402055328667,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/scicode","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-agentic-coding:overall","benchmarkName":"LiveBench Agentic Coding","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":46.76766666666666,"unit":"percent","family":"livebench:livebench-agentic-coding:overall","index":57.39338846400117,"aboveFit":false,"run":"xhigh effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Agentic%20Coding","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.5288902777777778,"latency":null},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":46.47750000000001,"unit":"percent","family":"euroeval","index":35.27950879043245,"aboveFit":false,"run":"low effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.6303991790921051,"costPerTest":null,"latency":null},{"benchmarkId":"terminalBench2","benchmarkName":"Terminal-Bench 2.0","category":"agentic","score":46.3,"unit":"percent","family":"terminalBench2","index":46.83293495056282,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-chess-puzzles:overall","benchmarkName":"Chess Puzzles","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":45,"unit":"percent","family":"epoch-ai:epoch-chess-puzzles:overall","index":68.14035950689328,"aboveFit":false,"run":"xhigh effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":5,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tiers-1-3-v2-private:overall","benchmarkName":"FrontierMath-Tiers-1-3-v2-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":44.91228070175438,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tiers-1-3-v2-private:overall","index":54.53435211101963,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.9515543245521676,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:emb:overall","benchmarkName":"EMB","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":44.751,"unit":"percent","family":"vals-ai:emb:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/emb","measuredAt":"2026-10-20","stderr":3.121,"costPerTest":0.57312,"latency":1767.043},{"benchmarkId":"parseBenchLlamaIndex","benchmarkName":"ParseBench: document parsing for AI agents (LlamaIndex leaderboard)","category":"multimodal","score":42.24,"unit":"percent","family":"parseBenchLlamaIndex","index":32.65189505360947,"aboveFit":false,"run":null,"source":"Boyang Zhang, Sebastián G. Acosta, Preston Carlson, Sacha Bron, Pierre-Loïc Doulcet, Daniel B. Ospina and Simon Suo (LlamaIndex)","sourceUrl":"https://arxiv.org/abs/2604.08538","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:terminal-bench-2-1:overall","benchmarkName":"Terminal-Bench 2.1","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":41.573,"unit":"percent","family":"terminal-bench-2-1","index":45.455313274590125,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2-1","measuredAt":"2026-10-01","stderr":2.973,"costPerTest":0.072321,"latency":757.485},{"benchmarkId":"vals-ai:medcode:overall","benchmarkName":"MedCode","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":41.029,"unit":"percent","family":"vals-ai:medcode:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medcode","measuredAt":"2026-10-29","stderr":2.256,"costPerTest":0.000844,"latency":8.043},{"benchmarkId":"vals-ai:terminal-bench-2:overall","benchmarkName":"Terminal-Bench 2.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":39.888,"unit":"percent","family":"vals-ai:terminal-bench-2:overall","index":50.568278279554775,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2","measuredAt":"2026-06-04","stderr":4.454,"costPerTest":0.075077,"latency":845.812},{"benchmarkId":"osWorldVerified","benchmarkName":"OSWorld-Verified","category":"agentic","score":39,"unit":"percent","family":"osWorldVerified","index":31.964036036654182,"aboveFit":false,"run":null,"source":"Tianbao Xie, Danyang Zhang, Jixuan Chen, Xiaochuan Li, Siheng Zhao, Ruisheng Cao, Toh Jing Hua, Zhoujun Cheng, Dongchan Shin, Fangyu Lei, Yitao Liu, Yiheng Xu, Shuyan Zhou, Silvio Savarese, Caiming Xiong, Victor Zhong, Tao Yu","sourceUrl":"https://os-world.github.io/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:fabv2:overall","benchmarkName":"Finance Agent (v2)","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":38.217,"unit":"percent","family":"vals-ai:fabv2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/fabv2","measuredAt":"2026-10-29","stderr":1.185,"costPerTest":0.15925,"latency":334.538},{"benchmarkId":"hle","benchmarkName":"Humanity's Last Exam","category":"knowledge","score":37.7,"unit":"percent","family":"hle","index":62.210780800936725,"aboveFit":false,"run":null,"source":"Center for AI Safety, Scale AI, and thousands of expert contributors","sourceUrl":"https://lastexam.ai/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"toolathlon","benchmarkName":"Toolathlon","category":"agentic","score":35.5,"unit":"percent","family":"toolathlon","index":51.00146744368751,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":28.3,"unit":"percent","family":"hle","index":52.77982899765666,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:vibe-code:overall","benchmarkName":"Vibe Code Bench v1.1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":26.097,"unit":"percent","family":"vals-ai:vibe-code:overall","index":51.665730205582356,"aboveFit":false,"run":"OpenHands","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","measuredAt":"2026-11-09","stderr":5.075,"costPerTest":1.277298,"latency":3276.224},{"benchmarkId":"epoch-ai:epoch-frontiermath-2025-02-28-private:overall","benchmarkName":"FrontierMath-2025-02-28-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":25.86,"unit":"percent","family":"epoch-ai:epoch-frontiermath-2025-02-28-private:overall","index":52.742802775099925,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.58,"costPerTest":null,"latency":null},{"benchmarkId":"omniscienceAccuracy","benchmarkName":"Artificial Analysis Omniscience Accuracy","category":"knowledge","score":25.7,"unit":"percent","family":"omniscienceAccuracy","index":43.07852925695467,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"apexAgentsAa","benchmarkName":"APEX-Agents-AA","category":"agentic","score":24.9,"unit":"percent","family":"apexAgentsAa","index":59.18734521839825,"aboveFit":false,"run":null,"source":"Artificial Analysis / Mercor","sourceUrl":"https://artificialanalysis.ai/evaluations/apex-agents-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hleNoTools","benchmarkName":"Humanity's Last Exam without tools","category":"knowledge","score":24.3,"unit":"percent","family":"hle","index":48.76665801753748,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gdpvalAaNormalized","benchmarkName":"GDPval-AA normalized","category":"agentic","score":22.4,"unit":"percent","family":"gdpvalAaNormalized","index":50.049254379089824,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:ioi-v1:overall","benchmarkName":"IOI v1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":15.25,"unit":"percent","family":"vals-ai:ioi-v1:overall","index":47.72813257970397,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/ioi-v1","measuredAt":"2026-08-09","stderr":6.438,"costPerTest":0.042833,"latency":1170.492},{"benchmarkId":"vals-ai:code-migration:overall","benchmarkName":"Code Migration","category":"coding","domain":null,"eligible":true,"ranked":true,"score":14.467,"unit":"percent","family":"vals-ai:code-migration:overall","index":52.874587880213156,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/code-migration","measuredAt":"2026-10-29","stderr":4.025,"costPerTest":0.416225,"latency":1594.162},{"benchmarkId":"epoch-ai:epoch-frontiermath-tier-4-v2-private:overall","benchmarkName":"FrontierMath-Tier-4-v2-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":12.195121951219512,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tier-4-v2-private:overall","index":53.27915862886624,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":5.173952057462543,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-simpleqa-verified:overall","benchmarkName":"SimpleQA Verified","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":11.700000000000001,"unit":"percent","family":"epoch-ai:epoch-simpleqa-verified:overall","index":33.66618516350279,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.0169287802713345,"costPerTest":null,"latency":null},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":9.3,"unit":"percent","family":"critpt","index":54.852830858072565,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:legal_research:overall","benchmarkName":"Legal Research Bench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":6.25,"unit":"percent","family":"vals-ai:legal_research:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_research","measuredAt":"2026-11-09","stderr":1.682,"costPerTest":0.141253,"latency":526.838},{"benchmarkId":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-private:overall","benchmarkName":"FrontierMath-Tier-4-2025-07-01-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":6.25,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-private:overall","index":49.55037447848712,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":3.53,"costPerTest":null,"latency":null},{"benchmarkId":"arc-prize:v2-semi-private:overall","benchmarkName":"ARC-AGI-2 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":5.6899999999999995,"unit":"percent","family":"arc-prize:v2-semi-private:overall","index":41.800991850796876,"aboveFit":false,"run":"xhigh effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.16,"latency":null},{"benchmarkId":"epoch-ai:epoch-mystery-game-puzzles:overall","benchmarkName":"Mystery Game Puzzles","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":5,"unit":"percent","family":"epoch-ai:epoch-mystery-game-puzzles:overall","index":42.55307268559606,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.1904291355759034,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:tax_agent_bench:overall","benchmarkName":"Tax Agent Bench","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":4.974,"unit":"percent","family":"vals-ai:tax_agent_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench","measuredAt":"2026-11-09","stderr":1.417,"costPerTest":0.069662,"latency":220.306},{"benchmarkId":"arc-prize:v2-public-eval:overall","benchmarkName":"ARC-AGI-2 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":4.17,"unit":"percent","family":"arc-prize:v2-public-eval:overall","index":null,"aboveFit":false,"run":"xhigh effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.21,"latency":null}],"price":{"inputPerMillion":0.19999999999999998,"outputPerMillion":1.25,"batch":{"inputPerMillion":0.09999999999999999,"outputPerMillion":0.625},"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":91,"timeToFirstToken":0.882,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":{"elo":1371.7828100109446,"eloUpper":1375.5521492217797,"eloLower":1368.0134708001096,"votes":61636,"rank":187,"rankUpper":null,"rankLower":null,"arenaName":"GPT-5.4 nano-high","organization":"openai","updatedAt":"2026-11-09"},"quality":{"coveragePercent":100,"benchmarkCount":43,"categoryCount":8,"confidence":"high","state":"available"},"hosts":[{"host":"databricks","name":"Databricks","kind":"cloud","url":"https://docs.databricks.com/aws/en/machine-learning/foundation-models/","inputPerMillion":0.2,"outputPerMillion":1.25,"cacheReadPerMillion":0.02,"contextTokens":400000,"maxOutputTokens":128000,"deprecated":false},{"host":"digitalocean","name":"DigitalOcean","kind":"cloud","url":"https://docs.digitalocean.com/products/gradient-ai-platform/details/models/","inputPerMillion":0.2,"outputPerMillion":1.25,"cacheReadPerMillion":0.02,"contextTokens":400000,"maxOutputTokens":128000,"deprecated":false},{"host":"azure","name":"Microsoft Azure","kind":"cloud","url":"https://learn.microsoft.com/en-us/azure/ai-services/openai/concepts/models","inputPerMillion":0.2,"outputPerMillion":1.25,"cacheReadPerMillion":0.02,"contextTokens":400000,"maxOutputTokens":128000,"deprecated":false},{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":0.2,"outputPerMillion":1.25,"cacheReadPerMillion":0.02,"contextTokens":400000,"maxOutputTokens":128000,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:livebench","name":"LiveBench","url":"https://livebench.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:arc-prize-foundation","name":"ARC Prize Foundation","url":"https://arcprize.org","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"ling-3-0-flash","providerId":"inclusionai","slug":"ling-3-0-flash","name":"Ling 3.0 Flash","provider":"InclusionAI","description":"Ling 3.0 Flash is a reasoning model from InclusionAI in the Ling 3.0 family. 25 benchmarks count toward its index, in 6 categories.","logoUrl":null,"releaseDate":"2026-07-23","contextWindowTokens":262144,"modalities":["text"],"openWeights":true,"details":{"apiId":"inclusionai/ling-3.0-flash","maxOutputTokens":32768,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":false,"cacheReadPerMillion":0.004200000000000001,"weightsUrl":"https://huggingface.co/inclusionAI/Ling-3.0-flash"},"reasoning":true,"overallScore":50.92464633859923,"overallScoreRange":{"low":46.19064633859923,"high":55.65864633859923},"badges":[],"capabilityScores":{"agentic":53.505275293276476,"coding":49.98904161068073,"reasoning":48.76102273904245,"multimodal":null,"knowledge":46.057975155493125,"multilingual":null,"instruction":58.644670196862876,"math":54.480181877470436},"capabilityCounts":{"math":3,"knowledge":4,"coding":7,"instruction":1,"agentic":8,"reasoning":2},"benchmarks":[{"benchmarkId":"aime2026","benchmarkName":"AIME 2026","category":"math","score":93.2,"unit":"percent","family":"aime2026","index":52.38943295033079,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hmmtFeb2026","benchmarkName":"Harvard-MIT Mathematics Tournament February 2026","category":"math","score":87,"unit":"percent","family":"hmmtFeb2026","index":54.28494261404696,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":85.5,"unit":"percent","family":"gpqa-diamond","index":55.62736444349115,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gpqa","benchmarkName":"Graduate-Level Google-Proof Q&A","category":"knowledge","score":84.97,"unit":"percent","family":"gpqa-diamond","index":55.141262925783664,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gpqaDiamond","benchmarkName":"GPQA Diamond","category":"knowledge","score":84.97,"unit":"percent","family":"gpqa-diamond","index":55.141262925783664,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:gpqa:overall","benchmarkName":"GPQA Diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":84.848,"unit":"percent","family":"gpqa-diamond","index":55.02936785944345,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","measuredAt":"2026-09-01","stderr":2.197,"costPerTest":0.003327,"latency":43.513},{"benchmarkId":"vals-ai:lcb:overall","benchmarkName":"LiveCodeBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":83.988,"unit":"percent","family":"livecodebench","index":59.03207284498083,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","measuredAt":"2026-09-01","stderr":1.038,"costPerTest":0.003019,"latency":60.397},{"benchmarkId":"imoAnswerBench","benchmarkName":"IMOAnswerBench","category":"math","score":83.7,"unit":"percent","family":"imoAnswerBench","index":56.76617006803355,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"liveCodeBenchV5","benchmarkName":"LiveCodeBench v5","category":"coding","score":82.8,"unit":"percent","family":"liveCodeBenchV5","index":null,"aboveFit":false,"run":null,"source":"LiveCodeBench maintainers","sourceUrl":"https://github.com/LiveCodeBench/LiveCodeBench","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:mmlu_pro:overall","benchmarkName":"MMLU Pro","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":82.008,"unit":"percent","family":"mmlu-pro","index":47.800992321676006,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","measuredAt":"2026-09-01","stderr":0.375,"costPerTest":0.000862,"latency":20.073},{"benchmarkId":"vals-ai:medscribe:overall","benchmarkName":"MedScribe","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":80.904,"unit":"percent","family":"vals-ai:medscribe:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medscribe","measuredAt":"2026-10-29","stderr":2.039,"costPerTest":0.001358,"latency":12.121},{"benchmarkId":"vals-ai:legal_bench:overall","benchmarkName":"LegalBench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":79.69,"unit":"percent","family":"vals-ai:legal_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","measuredAt":"2026-10-13","stderr":0.515,"costPerTest":0.000172,"latency":3.896},{"benchmarkId":"ifBench","benchmarkName":"Instruction Following Benchmark","category":"instruction","score":74.5,"unit":"percent","family":"ifbench-family","index":58.644670196862876,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"wideResearch","benchmarkName":"WideResearch","category":"agentic","score":73.6,"unit":"percent","family":"wideResearch","index":56.41823576910139,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"bfclV4","benchmarkName":"Berkeley Function Calling Leaderboard v4","category":"agentic","score":73,"unit":"percent","family":"bfclV4","index":52.944782175804896,"aboveFit":false,"run":null,"source":"Shishir G. Patil et al.","sourceUrl":"https://gorilla.cs.berkeley.edu/leaderboard.html","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":73,"unit":"percent","family":"lcr","index":57.3819278269239,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"browseComp","benchmarkName":"BrowseComp","category":"agentic","score":72.2,"unit":"percent","family":"browseComp","index":56.94842404374848,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:tax_eval_v2:overall","benchmarkName":"TaxEval v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":70.646,"unit":"percent","family":"vals-ai:tax_eval_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","measuredAt":"2026-09-01","stderr":0.895,"costPerTest":0.000664,"latency":7.54},{"benchmarkId":"draco","benchmarkName":"Data Research and Analysis with Complex Operations","category":"agentic","score":70.4,"unit":"percent","family":"draco","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mcpAtlas","benchmarkName":"MCP Atlas","category":"agentic","score":65.5,"unit":"percent","family":"mcpAtlas","index":55.169694854310706,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:swebench:overall","benchmarkName":"SWE-bench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":65.2,"unit":"percent","family":"swe-bench-verified","index":49.316681600968366,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/swebench","measuredAt":"2026-09-01","stderr":2.132,"costPerTest":0.031792,"latency":195.978},{"benchmarkId":"vals-ai:corp_fin_v2:overall","benchmarkName":"CorpFin v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":61.927,"unit":"percent","family":"vals-ai:corp_fin_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","measuredAt":"2026-08-12","stderr":0.957,"costPerTest":0.006376,"latency":12.451},{"benchmarkId":"terminalBench21","benchmarkName":"Terminal-Bench 2.1","category":"agentic","score":57,"unit":"percent","family":"terminal-bench-2-1","index":54.47787698231474,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"swePro","benchmarkName":"SWE-bench Pro","category":"coding","score":56.6,"unit":"percent","family":"swePro","index":57.04050860915336,"aboveFit":false,"run":null,"source":"Xiang Deng, Jeff Da, Edwin Pan, Yannis Yiming He, Charles Ide, Kanak Garg, Niklas Lauffer, Andrew Park, Nitin Pasari, Chetan Rane, Karmini Sampath, Maya Krishnan, Srivatsa Kundurthy, Sean Hendryx, Zifan Wang, Chen Bo Calvin Zhang, Noah Jacobson, Bing Liu, Brad Kenstler","sourceUrl":"https://arxiv.org/abs/2509.16941","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hermesBench","benchmarkName":"Hermes Bench: 150 agent tasks graded on the files and state the agent leaves behind","category":"agentic","score":53.83,"unit":"percent","family":"hermesBench","index":null,"aboveFit":false,"run":null,"source":"Nous Research","sourceUrl":"https://portal.nousresearch.com/bench","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:public-benefits-bench:overall","benchmarkName":"Public Benefits Bench v1.1","category":null,"domain":"Public benefits","eligible":false,"ranked":true,"score":50.744,"unit":"percent","family":"vals-ai:public-benefits-bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/public-benefits-bench","measuredAt":"2026-10-12","stderr":1.3,"costPerTest":0.049736,"latency":275.254},{"benchmarkId":"vals-ai:terminal-bench-2-1:overall","benchmarkName":"Terminal-Bench 2.1","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":50.187,"unit":"percent","family":"terminal-bench-2-1","index":50.493257384160344,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2-1","measuredAt":"2026-10-01","stderr":2.456,"costPerTest":0.067399,"latency":593.826},{"benchmarkId":"aaSciCode","benchmarkName":"Artificial Analysis SciCode","category":"coding","score":42,"unit":"percent","family":"scicode-family","index":50.59482659193006,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/scicode","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"sciCode","benchmarkName":"Scientific Code Benchmark","category":"coding","score":41.24,"unit":"percent","family":"scicode-family","index":49.604021320654866,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:medcode:overall","benchmarkName":"MedCode","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":32.275,"unit":"percent","family":"vals-ai:medcode:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medcode","measuredAt":"2026-10-29","stderr":1.908,"costPerTest":0.001645,"latency":9.647},{"benchmarkId":"vals-ai:fabv2:overall","benchmarkName":"Finance Agent (v2)","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":30.314,"unit":"percent","family":"vals-ai:fabv2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/fabv2","measuredAt":"2026-10-29","stderr":0.275,"costPerTest":0.05047,"latency":157.02},{"benchmarkId":"aaTau3Banking","benchmarkName":"Artificial Analysis Tau3-Banking","category":"agentic","score":28,"unit":"percent","family":"aaTau3Banking","index":49.15740595052108,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/tau3-banking","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:skillsbench:overall","benchmarkName":"SkillsBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":27.946,"unit":"percent","family":"vals-ai:skillsbench:overall","index":45.94452724540008,"aboveFit":false,"run":"OpenHands","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/skillsbench","measuredAt":"2026-10-29","stderr":3.76,"costPerTest":0.060905,"latency":292.431},{"benchmarkId":"vals-ai:emb:overall","benchmarkName":"EMB","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":25.006,"unit":"percent","family":"vals-ai:emb:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/emb","measuredAt":"2026-10-20","stderr":2.462,"costPerTest":0.179925,"latency":735.485},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":23.7,"unit":"percent","family":"hle","index":48.1646823705196,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hle","benchmarkName":"Humanity's Last Exam","category":"knowledge","score":22.7,"unit":"percent","family":"hle","index":47.16138962548981,"aboveFit":false,"run":null,"source":"Center for AI Safety, Scale AI, and thousands of expert contributors","sourceUrl":"https://lastexam.ai/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gdpvalAaNormalized","benchmarkName":"GDPval-AA normalized","category":"agentic","score":22.4,"unit":"percent","family":"gdpvalAaNormalized","index":50.049254379089824,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hermesIndex","benchmarkName":"Hermes Index: agentic capability and cost per task inside Hermes Agent","category":"agentic","score":21.56,"unit":"percent","family":"hermesIndex","index":54.86883799039789,"aboveFit":false,"run":null,"source":"Nous Research","sourceUrl":"https://portal.nousresearch.com/bench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"omniscienceAccuracy","benchmarkName":"Artificial Analysis Omniscience Accuracy","category":"knowledge","score":18.2,"unit":"percent","family":"omniscienceAccuracy","index":33.62660937650814,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:vibe-code:overall","benchmarkName":"Vibe Code Bench v1.1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":2.914,"unit":"percent","family":"vals-ai:vibe-code:overall","index":43.300939573335135,"aboveFit":false,"run":"OpenHands","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","measuredAt":"2026-11-09","stderr":2.072,"costPerTest":0.292199,"latency":1005.598},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":1.7,"unit":"percent","family":"critpt","index":40.14011765116101,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:code-migration:overall","benchmarkName":"Code Migration","category":"coding","domain":null,"eligible":true,"ranked":true,"score":0,"unit":"percent","family":"vals-ai:code-migration:overall","index":45.18913744463489,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/code-migration","measuredAt":"2026-10-29","stderr":0,"costPerTest":0.060112,"latency":17934.764},{"benchmarkId":"vals-ai:legal_research:overall","benchmarkName":"Legal Research Bench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":0,"unit":"percent","family":"vals-ai:legal_research:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_research","measuredAt":"2026-11-09","stderr":0,"costPerTest":0.036023,"latency":211.833}],"price":{"inputPerMillion":0.020999999999999998,"outputPerMillion":0.063,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":73,"timeToFirstToken":0.911,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":85,"benchmarkCount":25,"categoryCount":6,"confidence":"medium","state":"available"},"hosts":[{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":0.021,"outputPerMillion":0.063,"cacheReadPerMillion":0.0042,"contextTokens":262144,"maxOutputTokens":32768,"deprecated":false},{"host":"novita-ai","name":"Novita AI","kind":"host","url":"https://novita.ai/docs/guides/introduction","inputPerMillion":0.06,"outputPerMillion":0.18,"cacheReadPerMillion":0.012,"contextTokens":262144,"maxOutputTokens":32768,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"nemotron-3-5-super-vl-120b-a12b-nvfp4","providerId":"nvidia","slug":"nemotron-3-5-super-vl-120b-a12b-nvfp4","name":"Nemotron 3.5 Super VL 120B A12B NVFP4","provider":"NVIDIA","description":"Nemotron 3.5 Super VL 120B A12B NVFP4 is a reasoning model from NVIDIA in the Nemotron 3.5 Super VL family. 18 benchmarks count toward its index, in 6 categories.","logoUrl":null,"releaseDate":"2026-10-15","contextWindowTokens":1000000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":true,"overallScore":50.83410272791536,"overallScoreRange":{"low":45.522797177319866,"high":56.14540827851085},"badges":[],"capabilityScores":{"agentic":45.13384052801234,"coding":55.128757966574476,"reasoning":55.554159702922576,"multimodal":47.131258521932565,"knowledge":52.710275337756855,"multilingual":null,"instruction":55.84113835237437,"math":null},"capabilityCounts":{"knowledge":3,"coding":4,"multimodal":4,"agentic":4,"instruction":1,"reasoning":2},"benchmarks":[{"benchmarkId":"gpqaDiamond","benchmarkName":"GPQA Diamond","category":"knowledge","score":89.9,"unit":"percent","family":"gpqa-diamond","index":59.66292421313821,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"apexShortlist","benchmarkName":"Apex Shortlist","category":"math","score":89.36,"unit":"percent","family":"apexShortlist","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"liveCodeBenchV6","benchmarkName":"LiveCodeBench v6","category":"coding","score":87.41,"unit":"percent","family":"liveCodeBenchV6","index":56.18567633374672,"aboveFit":false,"run":null,"source":"LiveCodeBench maintainers","sourceUrl":"https://github.com/LiveCodeBench/LiveCodeBench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vStar","benchmarkName":"V*","category":"multimodal","score":85.95,"unit":"percent","family":"vStar","index":44.62209785734848,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"charxiv","benchmarkName":"CharXiv Reasoning","category":"multimodal","score":79.88,"unit":"percent","family":"charxiv","index":55.12745526566435,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mathVision","benchmarkName":"MathVision","category":"multimodal","score":77.79,"unit":"percent","family":"mathVision","index":49.25741440461315,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"browseComp","benchmarkName":"BrowseComp","category":"agentic","score":77.5,"unit":"percent","family":"browseComp","index":61.13552901554489,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"tau2Airline","benchmarkName":"τ²-Bench Airline Domain","category":"agentic","score":75.75,"unit":"percent","family":"tau2Airline","index":null,"aboveFit":false,"run":null,"source":"Victor Barres, Honghua Dong, Soham Ray, Xujie Si, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2506.07982","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"ifBench","benchmarkName":"Instruction Following Benchmark","category":"instruction","score":71.5,"unit":"percent","family":"ifbench-family","index":55.84113835237437,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"sweVerified","benchmarkName":"Software Engineering Benchmark Verified","category":"coding","score":71.07,"unit":"percent","family":"swe-bench-verified","index":53.64614458255863,"aboveFit":false,"run":null,"source":"Carlos E. Jimenez, John Yang, Alexander Wettig, Shunyu Yao, Kexin Pei, Ofir Press, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2310.06770","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":67.69,"unit":"percent","family":"lcr","index":53.73884023606404,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmLongBenchDoc","benchmarkName":"MMLongBench-Doc","category":"multimodal","score":62.82,"unit":"percent","family":"mmLongBenchDoc","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"terminalBench21","benchmarkName":"Terminal-Bench 2.1","category":"agentic","score":60.8,"unit":"percent","family":"terminal-bench-2-1","index":56.700327367312056,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"swePro","benchmarkName":"SWE-bench Pro","category":"coding","score":51.3,"unit":"percent","family":"swePro","index":52.52697570885705,"aboveFit":false,"run":null,"source":"Xiang Deng, Jeff Da, Edwin Pan, Yannis Yiming He, Charles Ide, Kanak Garg, Niklas Lauffer, Andrew Park, Nitin Pasari, Chetan Rane, Karmini Sampath, Maya Krishnan, Srivatsa Kundurthy, Sean Hendryx, Zifan Wang, Chen Bo Calvin Zhang, Noah Jacobson, Bing Liu, Brad Kenstler","sourceUrl":"https://arxiv.org/abs/2509.16941","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaSciCode","benchmarkName":"Artificial Analysis SciCode","category":"coding","score":47.8,"unit":"percent","family":"scicode-family","index":58.156235241135505,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/scicode","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gdpvalAaNormalized","benchmarkName":"GDPval-AA normalized","category":"agentic","score":33.1,"unit":"percent","family":"gdpvalAaNormalized","index":57.699505729192424,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hleNoTools","benchmarkName":"Humanity's Last Exam without tools","category":"knowledge","score":31.44,"unit":"percent","family":"hle","index":55.93016821705021,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":30.7,"unit":"percent","family":"hle","index":55.18773158572816,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"babyVision","benchmarkName":"BabyVision","category":"multimodal","score":29.51,"unit":"percent","family":"babyVision","index":39.51806656010427,"aboveFit":false,"run":null,"source":"Meta AI","sourceUrl":"https://ai.meta.com/static-resource/muse-spark-1-1-evaluation-report","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hle","benchmarkName":"Humanity's Last Exam","category":"knowledge","score":29,"unit":"percent","family":"hle","index":53.482133919177514,"aboveFit":false,"run":null,"source":"Center for AI Safety, Scale AI, and thousands of expert contributors","sourceUrl":"https://lastexam.ai/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"omniscienceAccuracy","benchmarkName":"Artificial Analysis Omniscience Accuracy","category":"knowledge","score":25.86,"unit":"percent","family":"omniscienceAccuracy","index":43.280170214404194,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"tau3Bench","benchmarkName":"τ³-Bench Tool-Agent-User Evaluation","category":"agentic","score":17.78,"unit":"percent","family":"tau3Bench","index":5,"aboveFit":false,"run":null,"source":"Sierra Research","sourceUrl":"https://github.com/sierra-research/tau2-bench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":10.6,"unit":"percent","family":"critpt","index":57.369479169781116,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":85,"benchmarkCount":18,"categoryCount":6,"confidence":"medium","state":"available"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"nemotron-3-ultra-500b","providerId":"nvidia","slug":"nemotron-3-ultra","name":"Nemotron 3 Ultra","provider":"NVIDIA","description":"Nemotron 3 Ultra is a reasoning model from NVIDIA. 24 benchmarks count toward its index, in 6 categories.","logoUrl":null,"releaseDate":"2026-06-04","contextWindowTokens":1000000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":true,"overallScore":50.83208651294625,"overallScoreRange":{"low":45.53139908454592,"high":56.13277394134658},"badges":[],"capabilityScores":{"agentic":48.934582837958985,"coding":54.31032072583265,"reasoning":49.95127243545085,"multimodal":null,"knowledge":49.98187562945269,"multilingual":52.378041573015366,"instruction":65.23297003141089,"math":null},"capabilityCounts":{"agentic":12,"coding":3,"knowledge":4,"multilingual":1,"instruction":1,"reasoning":3},"benchmarks":[{"benchmarkId":"pinchBench","benchmarkName":"PinchBench","category":"agentic","score":90,"unit":"percent","family":"pinchBench","index":49.847313385590816,"aboveFit":false,"run":null,"source":"Kilo Code","sourceUrl":"https://pinchbench.com/about","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"liveCodeBenchV6","benchmarkName":"LiveCodeBench v6","category":"coding","score":89,"unit":"percent","family":"liveCodeBenchV6","index":57.49115719756217,"aboveFit":false,"run":null,"source":"LiveCodeBench maintainers","sourceUrl":"https://github.com/LiveCodeBench/LiveCodeBench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gpqa","benchmarkName":"Graduate-Level Google-Proof Q&A","category":"knowledge","score":87,"unit":"percent","family":"gpqa-diamond","index":57.00312345587083,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gpqaDiamond","benchmarkName":"GPQA Diamond","category":"knowledge","score":87,"unit":"percent","family":"gpqa-diamond","index":57.00312345587083,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmluPro","benchmarkName":"Massive Multitask Language Understanding Professional","category":"knowledge","score":86.8,"unit":"percent","family":"mmlu-pro","index":53.838338734020375,"aboveFit":false,"run":null,"source":"Yubo Wang, Xueguang Ma, Ge Zhang, Yuansheng Ni, Abhranil Chandra, Shiguang Guo, Weiming Ren, Aaran Arulraj, Xuan He, Ziyan Jiang, Tianle Li, Max Ku, Kai Wang, Alex Zhuang, Rongqi Fan, Xiang Yue, Wenhu Chen","sourceUrl":"https://arxiv.org/abs/2406.01574","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":86.7,"unit":"percent","family":"gpqa-diamond","index":56.727971653394896,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"tau2Bench","benchmarkName":"τ²-Bench Tool-Agent-User Evaluation","category":"agentic","score":83.3,"unit":"percent","family":"tau2-bench","index":58.06508036840864,"aboveFit":false,"run":null,"source":"Victor Barres, Honghua Dong, Soham Ray, Xujie Si, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2506.07982","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaOpennessIndex","benchmarkName":"Artificial Analysis Openness Index","category":"knowledge","score":83.3,"unit":"percent","family":"aaOpennessIndex","index":null,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-openness-index","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmluProX","benchmarkName":"MMLU-ProX","category":"multilingual","score":83,"unit":"percent","family":"mmluProX","index":52.378041573015366,"aboveFit":false,"run":null,"source":"MMLU-ProX authors","sourceUrl":"https://arxiv.org/abs/2503.10497","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaHarveyLab","benchmarkName":"Artificial Analysis Harvey LAB-AA v1.0","category":"agentic","score":81.7,"unit":"percent","family":"aaHarveyLab","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"ifBench","benchmarkName":"Instruction Following Benchmark","category":"instruction","score":81.7,"unit":"percent","family":"ifbench-family","index":65.37314662363531,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaIfBench","benchmarkName":"Artificial Analysis IFBench","category":"instruction","score":81.4,"unit":"percent","family":"ifbench-family","index":65.09279343918645,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"sweVerified","benchmarkName":"Software Engineering Benchmark Verified","category":"coding","score":71.9,"unit":"percent","family":"swe-bench-verified","index":54.2583173721191,"aboveFit":false,"run":null,"source":"Carlos E. Jimenez, John Yang, Alexander Wettig, Shunyu Yao, Kexin Pei, Ofir Press, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2310.06770","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"tau3Bench","benchmarkName":"τ³-Bench Tool-Agent-User Evaluation","category":"agentic","score":70.9,"unit":"percent","family":"tau3Bench","index":61.02887308471328,"aboveFit":false,"run":null,"source":"Sierra Research","sourceUrl":"https://github.com/sierra-research/tau2-bench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":67,"unit":"percent","family":"lcr","index":53.26544467340994,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"longBenchV2","benchmarkName":"LongBench v2","category":"reasoning","score":61.9,"unit":"percent","family":"longBenchV2","index":53.73801833840317,"aboveFit":false,"run":null,"source":"LongBench v2 authors","sourceUrl":"https://arxiv.org/abs/2412.15204","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"terminalBench21","benchmarkName":"Terminal-Bench 2.1","category":"agentic","score":56.4,"unit":"percent","family":"terminal-bench-2-1","index":54.126963763630954,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"sciCode","benchmarkName":"Scientific Code Benchmark","category":"coding","score":44.6,"unit":"percent","family":"scicode-family","index":53.984423572608364,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"browseComp","benchmarkName":"BrowseComp","category":"agentic","score":44.4,"unit":"percent","family":"browseComp","index":34.9858734369673,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaSciCode","benchmarkName":"Artificial Analysis SciCode","category":"coding","score":40.3,"unit":"percent","family":"scicode-family","index":48.37855164302501,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/scicode","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hleWithTools","benchmarkName":"Humanity's Last Exam with tools","category":"agentic","score":37.4,"unit":"percent","family":"hleWithTools","index":49.30516756727356,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gdpvalAaNormalized","benchmarkName":"GDPval-AA normalized","category":"agentic","score":33.1,"unit":"percent","family":"gdpvalAaNormalized","index":57.699505729192424,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":28.4,"unit":"percent","family":"hle","index":52.88015827215963,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaCweBench","benchmarkName":"Artificial Analysis CWE-Bench-AA","category":"agentic","score":27.5,"unit":"percent","family":"aaCweBench","index":44.31125373919793,"aboveFit":false,"run":null,"source":"Artificial Analysis; dataset by Collinear AI","sourceUrl":"https://artificialanalysis.ai/evaluations/cwe-bench-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hle","benchmarkName":"Humanity's Last Exam","category":"knowledge","score":26.7,"unit":"percent","family":"hle","index":51.17456060560899,"aboveFit":false,"run":null,"source":"Center for AI Safety, Scale AI, and thousands of expert contributors","sourceUrl":"https://lastexam.ai/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hleNoTools","benchmarkName":"Humanity's Last Exam without tools","category":"knowledge","score":26.7,"unit":"percent","family":"hle","index":51.17456060560899,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"omniscienceAccuracy","benchmarkName":"Artificial Analysis Omniscience Accuracy","category":"knowledge","score":21.6,"unit":"percent","family":"omniscienceAccuracy","index":37.91147972231057,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaCyberIndex","benchmarkName":"Artificial Analysis Cyber Index","category":"agentic","score":13.1,"unit":"percent","family":"aaCyberIndex","index":40.703897348726414,"aboveFit":false,"run":null,"source":"Artificial Analysis with the Cyber Index Alliance (Collinear AI, IBM, NVIDIA, Pasta Lab, Vercel)","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-cyber-index","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaCyberGymE2e","benchmarkName":"Artificial Analysis CyberGym-E2E-AA","category":"agentic","score":10.7,"unit":"percent","family":"aaCyberGymE2e","index":24.109249133593934,"aboveFit":false,"run":null,"source":"Artificial Analysis; dataset by Berkeley RDI","sourceUrl":"https://artificialanalysis.ai/evaluations/cybergym-e2e-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":3.1,"unit":"percent","family":"critpt","index":42.85035429453946,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaDeepsecBench","benchmarkName":"Artificial Analysis DeepsecBench-AA","category":"agentic","score":1,"unit":"percent","family":"aaDeepsecBench","index":51.05314914986474,"aboveFit":false,"run":null,"source":"Artificial Analysis; dataset by Vercel","sourceUrl":"https://artificialanalysis.ai/evaluations/deepsecbench-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mazeBench","benchmarkName":"MazeBench: visual spatial reasoning in a 3D open world","category":"agentic","score":0,"unit":"percent","family":"mazeBench","index":61.97866734834783,"aboveFit":false,"run":null,"source":"Jonathan Pappas and David Pappas","sourceUrl":"https://mazebench.com/blog?post=maze-bench-results","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":0,"outputPerMillion":0,"batch":null,"freeTier":true,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":80,"benchmarkCount":24,"categoryCount":6,"confidence":"medium","state":"available"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"qwen3-6-35b-a3b","providerId":"alibaba","slug":"qwen3-6-35b-a3b","name":"Qwen3.6-35B-A3B","provider":"Alibaba","description":"Qwen3.6-35B-A3B is a reasoning model from Alibaba. 39 benchmarks count toward its index, in 7 categories.","logoUrl":null,"releaseDate":"2026-04-15","contextWindowTokens":262144,"modalities":["text","image","video"],"openWeights":true,"details":{"apiId":"qwen/qwen3.6-35b-a3b","maxOutputTokens":235929,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.024999999999999998,"weightsUrl":"https://huggingface.co/Qwen/Qwen3.6-35B-A3B"},"reasoning":true,"overallScore":50.58061461958152,"overallScoreRange":{"low":47.491342372553255,"high":53.66988686660978},"badges":[],"capabilityScores":{"agentic":52.702202869861004,"coding":49.570962526502285,"reasoning":49.87934336034303,"multimodal":51.51374278962838,"knowledge":49.83251431683478,"multilingual":null,"instruction":49.2061129870849,"math":48.98169979579278},"capabilityCounts":{"agentic":11,"math":6,"multimodal":8,"knowledge":5,"coding":5,"reasoning":3,"instruction":1},"benchmarks":[{"benchmarkId":"tau2Bench","benchmarkName":"τ²-Bench Tool-Agent-User Evaluation","category":"agentic","score":95.3,"unit":"percent","family":"tau2-bench","index":66.0279577114679,"aboveFit":false,"run":null,"source":"Victor Barres, Honghua Dong, Soham Ray, Xujie Si, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2506.07982","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"ai2dTest","benchmarkName":"AI2D test split","category":"multimodal","score":92.7,"unit":"percent","family":"ai2dTest","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aime2026","benchmarkName":"AIME 2026","category":"math","score":92.7,"unit":"percent","family":"aime2026","index":52.02946269240527,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"refcocoAvg","benchmarkName":"RefCOCO average","category":"multimodal","score":92,"unit":"percent","family":"refcocoAvg","index":51.22339051504448,"aboveFit":false,"run":null,"source":"RefCOCO dataset authors","sourceUrl":"https://github.com/lichengunc/refer","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hmmtFeb2025","benchmarkName":"Harvard-MIT Mathematics Tournament February 2025","category":"math","score":90.7,"unit":"percent","family":"hmmtFeb2025","index":49.004514206810256,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"cEval","benchmarkName":"C-Eval","category":"knowledge","score":90,"unit":"percent","family":"cEval","index":45.7992791517471,"aboveFit":false,"run":null,"source":"C-Eval authors","sourceUrl":"https://arxiv.org/abs/2305.08322","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"omniDocBench15","benchmarkName":"OmniDocBench 1.5","category":"multimodal","score":89.9,"unit":"percent","family":"omniDocBench15","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hmmtNov2025","benchmarkName":"Harvard-MIT Mathematics Tournament November 2025","category":"math","score":89.1,"unit":"percent","family":"hmmtNov2025","index":50.34601435643425,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"extractBenchLlamaIndex","benchmarkName":"ExtractBench: schema-guided enterprise document extraction (LlamaIndex leaderboard)","category":"multimodal","score":88.11,"unit":"percent","family":"extractBenchLlamaIndex","index":68.97316019149898,"aboveFit":true,"run":null,"source":"Boyang Zhang, Adrian Lyjak, Eli Stewart, Zhaoqi Li and Simon Suo (LlamaIndex)","sourceUrl":"https://arxiv.org/abs/2607.29677","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"videoMmeWithSub","benchmarkName":"Video-MME with subtitle","category":"multimodal","score":86.6,"unit":"percent","family":"videoMmeWithSub","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mlvuAvg","benchmarkName":"MLVU mean average","category":"multimodal","score":86.2,"unit":"percent","family":"mlvuAvg","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gpqa","benchmarkName":"Graduate-Level Google-Proof Q&A","category":"knowledge","score":86,"unit":"percent","family":"gpqa-diamond","index":56.085950780951045,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"realWorldQa","benchmarkName":"RealWorldQA","category":"multimodal","score":85.3,"unit":"percent","family":"realWorldQa","index":59.66115961432302,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmluPro","benchmarkName":"Massive Multitask Language Understanding Professional","category":"knowledge","score":85.2,"unit":"percent","family":"mmlu-pro","index":51.82253024909739,"aboveFit":false,"run":null,"source":"Yubo Wang, Xueguang Ma, Ge Zhang, Yuansheng Ni, Abhranil Chandra, Shiguang Guo, Weiming Ren, Aaran Arulraj, Xuan He, Ziyan Jiang, Tianle Li, Max Ku, Kai Wang, Alex Zhuang, Rongqi Fan, Xiang Yue, Wenhu Chen","sourceUrl":"https://arxiv.org/abs/2406.01574","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":84.84848484848484,"unit":"percent","family":"gpqa-diamond","index":55.02981254922523,"aboveFit":false,"run":"none effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.554565042660364,"costPerTest":null,"latency":null},{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":84.1,"unit":"percent","family":"gpqa-diamond","index":54.343322698603444,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"videoMmmu","benchmarkName":"VideoMMMU","category":"multimodal","score":83.7,"unit":"percent","family":"videoMmmu","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hmmtFeb2026","benchmarkName":"Harvard-MIT Mathematics Tournament February 2026","category":"math","score":83.6,"unit":"percent","family":"hmmtFeb2026","index":51.90555592753932,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"videoMmeNoSub","benchmarkName":"Video-MME without subtitle","category":"multimodal","score":82.5,"unit":"percent","family":"videoMmeNoSub","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"ccOcr","benchmarkName":"CC-OCR","category":"multimodal","score":81.9,"unit":"percent","family":"ccOcr","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmmu","benchmarkName":"Massive Multi-discipline Multimodal Understanding","category":"multimodal","score":81.7,"unit":"percent","family":"mmmu","index":50.51508377803903,"aboveFit":false,"run":null,"source":"MMMU authors","sourceUrl":"https://arxiv.org/abs/2401.05508","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"liveCodeBenchV6","benchmarkName":"LiveCodeBench v6","category":"coding","score":80.4,"unit":"percent","family":"liveCodeBenchV6","index":50.43006573290001,"aboveFit":false,"run":null,"source":"LiveCodeBench maintainers","sourceUrl":"https://github.com/LiveCodeBench/LiveCodeBench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmAnswerBench","benchmarkName":"MMAnswerBench","category":"math","score":78.9,"unit":"percent","family":"mmAnswerBench","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"charxiv","benchmarkName":"CharXiv Reasoning","category":"multimodal","score":78,"unit":"percent","family":"charxiv","index":52.987078225598744,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmmuPro","benchmarkName":"Massive Multi-discipline Multimodal Understanding Pro","category":"multimodal","score":75.3,"unit":"percent","family":"mmmu-pro","index":53.14915859130881,"aboveFit":false,"run":null,"source":"MMMU-Pro authors","sourceUrl":"https://arxiv.org/abs/2409.02813","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaMmmuPro","benchmarkName":"Artificial Analysis MMMU-Pro","category":"multimodal","score":75,"unit":"percent","family":"mmmu-pro","index":52.75873491177307,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmmu-pro","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"sweVerified","benchmarkName":"Software Engineering Benchmark Verified","category":"coding","score":73.4,"unit":"percent","family":"swe-bench-verified","index":55.36465373879464,"aboveFit":false,"run":null,"source":"Carlos E. Jimenez, John Yang, Alexander Wettig, Shunyu Yao, Kexin Pei, Ofir Press, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2310.06770","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":71.7,"unit":"percent","family":"lcr","index":56.49002314366254,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":68.88888888888889,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":48.99823624928844,"aboveFit":false,"run":"none effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":6.979205927323111,"costPerTest":null,"latency":null},{"benchmarkId":"clawEval","benchmarkName":"Claw-Eval","category":"agentic","score":68.7,"unit":"percent","family":"clawEval","index":64.46809185278812,"aboveFit":false,"run":null,"source":"Bowen Ye, Rang Li, Qibin Yang, Yuanxin Liu, Linli Yao, Hanglong Lv, Zhihui Xie, Chenxin An, Lei Li, Lingpeng Kong, Qi Liu, Zhifang Sui, Tong Yang","sourceUrl":"https://arxiv.org/abs/2604.06132","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"tau3Bench","benchmarkName":"τ³-Bench Tool-Agent-User Evaluation","category":"agentic","score":67.2,"unit":"percent","family":"tau3Bench","index":56.77283114979109,"aboveFit":false,"run":null,"source":"Sierra Research","sourceUrl":"https://github.com/sierra-research/tau2-bench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"superGpqa","benchmarkName":"SuperGPQA: Scaling LLM Evaluation Across 285 Graduate Disciplines","category":"knowledge","score":64.7,"unit":"percent","family":"superGpqa","index":50.25252347914121,"aboveFit":false,"run":null,"source":"Xiaoxuan Du, Yao Yao, Kexin Ma, Bowen Wang, Tianyu Zheng, Kaiyan Zhu, Yiming Zhang, Yutao Zhu, Jiawei Zhou, Jingren Zhou","sourceUrl":"https://arxiv.org/abs/2502.14739","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaIfBench","benchmarkName":"Artificial Analysis IFBench","category":"instruction","score":64.4,"unit":"percent","family":"ifbench-family","index":49.2061129870849,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mcpAtlas","benchmarkName":"MCP Atlas","category":"agentic","score":62.8,"unit":"percent","family":"mcpAtlas","index":53.351526744916896,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"wideResearch","benchmarkName":"WideResearch","category":"agentic","score":60.1,"unit":"percent","family":"wideResearch","index":43.91515925526581,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"simpleVqa","benchmarkName":"SimpleVQA","category":"multimodal","score":58.9,"unit":"percent","family":"simpleVqa","index":49.96748666252854,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"qwenClawBench","benchmarkName":"QwenClawBench","category":"agentic","score":52.6,"unit":"percent","family":"qwenClawBench","index":51.19235128664267,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"terminalBench2","benchmarkName":"Terminal-Bench 2.0","category":"agentic","score":51.5,"unit":"percent","family":"terminalBench2","index":50.531931552491116,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"odinw13","benchmarkName":"ODINW13","category":"multimodal","score":50.8,"unit":"percent","family":"odinw13","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"swePro","benchmarkName":"SWE-bench Pro","category":"coding","score":49.5,"unit":"percent","family":"swePro","index":50.99407774271868,"aboveFit":false,"run":null,"source":"Xiang Deng, Jeff Da, Edwin Pan, Yannis Yiming He, Charles Ide, Kanak Garg, Niklas Lauffer, Andrew Park, Nitin Pasari, Chetan Rane, Karmini Sampath, Maya Krishnan, Srivatsa Kundurthy, Sean Hendryx, Zifan Wang, Chen Bo Calvin Zhang, Noah Jacobson, Bing Liu, Brad Kenstler","sourceUrl":"https://arxiv.org/abs/2509.16941","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gertLabs","benchmarkName":"Gert Labs Composite Game Benchmark","category":"agentic","score":42.65,"unit":"percent","family":"gertLabs","index":50.08117783441482,"aboveFit":false,"run":null,"source":"Gert Labs","sourceUrl":"https://gertlabs.com/rankings","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"parseBenchLlamaIndex","benchmarkName":"ParseBench: document parsing for AI agents (LlamaIndex leaderboard)","category":"multimodal","score":37.35,"unit":"percent","family":"parseBenchLlamaIndex","index":25.828636578453303,"aboveFit":false,"run":null,"source":"Boyang Zhang, Sebastián G. Acosta, Preston Carlson, Sacha Bron, Pierre-Loïc Doulcet, Daniel B. Ospina and Simon Suo (LlamaIndex)","sourceUrl":"https://arxiv.org/abs/2604.08538","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaSciCode","benchmarkName":"Artificial Analysis SciCode","category":"coding","score":36.6,"unit":"percent","family":"scicode-family","index":43.554894401290504,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/scicode","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vitaBench","benchmarkName":"VITA-Bench","category":"agentic","score":35.6,"unit":"percent","family":"vitaBench","index":51.254859376786584,"aboveFit":false,"run":null,"source":"Meituan LongCat Team","sourceUrl":"https://vitabench.github.io/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"nl2Repo","benchmarkName":"NL2Repo","category":"coding","score":29.4,"unit":"percent","family":"nl2Repo","index":47.51112101680757,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"toolathlon","benchmarkName":"Toolathlon","category":"agentic","score":26.9,"unit":"percent","family":"toolathlon","index":44.08102535755337,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"deepPlanning","benchmarkName":"DeepPlanning","category":"agentic","score":25.9,"unit":"percent","family":"deepPlanning","index":null,"aboveFit":false,"run":null,"source":"DeepPlanning authors","sourceUrl":"https://arxiv.org/abs/2601.18137","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":22.2,"unit":"percent","family":"hle","index":46.65974325297491,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-mystery-game-puzzles:overall","benchmarkName":"Mystery Game Puzzles","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":22,"unit":"percent","family":"epoch-ai:epoch-mystery-game-puzzles:overall","index":55.71812592958398,"aboveFit":false,"run":"none effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":4.163331998932265,"costPerTest":null,"latency":null},{"benchmarkId":"hle","benchmarkName":"Humanity's Last Exam","category":"knowledge","score":21.4,"unit":"percent","family":"hle","index":45.857109056951074,"aboveFit":false,"run":null,"source":"Center for AI Safety, Scale AI, and thousands of expert contributors","sourceUrl":"https://lastexam.ai/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tiers-1-3-v2-private:overall","benchmarkName":"FrontierMath-Tiers-1-3-v2-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":20.350877192982455,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tiers-1-3-v2-private:overall","index":41.60641534227914,"aboveFit":false,"run":"none effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.389037106974561,"costPerTest":null,"latency":null},{"benchmarkId":"gdpvalAaNormalized","benchmarkName":"GDPval-AA normalized","category":"agentic","score":19.6,"unit":"percent","family":"gdpvalAaNormalized","index":48.047319446352695,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":0.3,"unit":"percent","family":"critpt","index":37.42988100778257,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":0.09999999999999999,"outputPerMillion":0.8999999999999999,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":65,"timeToFirstToken":0.601,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":95,"benchmarkCount":39,"categoryCount":7,"confidence":"high","state":"available"},"hosts":[{"host":"coreweave","name":"CoreWeave","kind":"host","url":"https://docs.wandb.ai/inference","inputPerMillion":0.1,"outputPerMillion":0.89,"cacheReadPerMillion":0.035,"contextTokens":262144,"maxOutputTokens":262144,"deprecated":false},{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":0.1,"outputPerMillion":0.9,"cacheReadPerMillion":0.025,"contextTokens":262144,"maxOutputTokens":235929,"deprecated":false},{"host":"deepinfra","name":"Deep Infra","kind":"host","url":"https://deepinfra.com/models","inputPerMillion":0.1,"outputPerMillion":0.95,"cacheReadPerMillion":0.1,"contextTokens":262144,"maxOutputTokens":81920,"deprecated":false},{"host":"novita-ai","name":"Novita AI","kind":"host","url":"https://novita.ai/docs/guides/introduction","inputPerMillion":0.248,"outputPerMillion":1.485,"cacheReadPerMillion":null,"contextTokens":262144,"maxOutputTokens":65536,"deprecated":false},{"host":"scaleway","name":"Scaleway","kind":"cloud","url":"https://www.scaleway.com/en/docs/generative-apis/","inputPerMillion":0.280375,"outputPerMillion":1.68225,"cacheReadPerMillion":null,"contextTokens":128000,"maxOutputTokens":16384,"deprecated":false},{"host":"siliconflow","name":"SiliconFlow","kind":"host","url":"https://cloud.siliconflow.com/models","inputPerMillion":0.24,"outputPerMillion":1.8,"cacheReadPerMillion":0.15,"contextTokens":262144,"maxOutputTokens":262144,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"mistral-medium-3-5-128b","providerId":"mistral","slug":"mistral-medium-3-5-128b","name":"Mistral Medium 3.5 128B","provider":"Mistral","description":"Mistral Medium 3.5 128B is a reasoning model from Mistral in the Mistral Medium 3.5 family. 13 benchmarks count toward its index, in 6 categories.","logoUrl":null,"releaseDate":"2026-04-29","contextWindowTokens":256000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":true,"overallScore":50.31474501197546,"overallScoreRange":{"low":44.329993925494485,"high":56.29949609845644},"badges":[],"capabilityScores":{"agentic":60.11070732936025,"coding":53.35528904693484,"reasoning":45.846272947514926,"multimodal":39.61447103406951,"knowledge":41.240513038856,"multilingual":null,"instruction":53.31795969233471,"math":null},"capabilityCounts":{"agentic":4,"coding":2,"knowledge":3,"reasoning":2,"instruction":1,"multimodal":1},"benchmarks":[{"benchmarkId":"tau2Bench","benchmarkName":"τ²-Bench Tool-Agent-User Evaluation","category":"agentic","score":94.2,"unit":"percent","family":"tau2-bench","index":65.29802728835415,"aboveFit":false,"run":null,"source":"Victor Barres, Honghua Dong, Soham Ray, Xujie Si, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2506.07982","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"tau3Bench","benchmarkName":"τ³-Bench Tool-Agent-User Evaluation","category":"agentic","score":91.4,"unit":"percent","family":"tau3Bench","index":84.60964596739029,"aboveFit":true,"run":null,"source":"Sierra Research","sourceUrl":"https://github.com/sierra-research/tau2-bench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"sweVerified","benchmarkName":"Software Engineering Benchmark Verified","category":"coding","score":77.6,"unit":"percent","family":"swe-bench-verified","index":58.46239556548613,"aboveFit":false,"run":null,"source":"Carlos E. Jimenez, John Yang, Alexander Wettig, Shunyu Yao, Kexin Pei, Ofir Press, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2310.06770","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":74.8,"unit":"percent","family":"gpqa-diamond","index":45.81361682184945,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":69.3,"unit":"percent","family":"lcr","index":54.843429882256956,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaIfBench","benchmarkName":"Artificial Analysis IFBench","category":"instruction","score":68.8,"unit":"percent","family":"ifbench-family","index":53.31795969233471,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaMmmuPro","benchmarkName":"Artificial Analysis MMMU-Pro","category":"multimodal","score":64.9,"unit":"percent","family":"mmmu-pro","index":39.61447103406951,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmmu-pro","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaSciCode","benchmarkName":"Artificial Analysis SciCode","category":"coding","score":40.2,"unit":"percent","family":"scicode-family","index":48.248182528383545,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/scicode","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gertLabs","benchmarkName":"Gert Labs Composite Game Benchmark","category":"agentic","score":39.1,"unit":"percent","family":"gertLabs","index":47.20668324250998,"aboveFit":false,"run":null,"source":"Gert Labs","sourceUrl":"https://gertlabs.com/rankings","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"omniscienceAccuracy","benchmarkName":"Artificial Analysis Omniscience Accuracy","category":"knowledge","score":23,"unit":"percent","family":"omniscienceAccuracy","index":39.67583809999392,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":13.8,"unit":"percent","family":"hle","index":38.232084194724635,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gdpvalAaNormalized","benchmarkName":"GDPval-AA normalized","category":"agentic","score":13,"unit":"percent","family":"gdpvalAaNormalized","index":43.3284728191866,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":0,"unit":"percent","family":"critpt","index":36.8491160127729,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":1.5,"outputPerMillion":7.5,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":85,"benchmarkCount":13,"categoryCount":6,"confidence":"medium","state":"available"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"minimax-m2-5","providerId":"minimax","slug":"minimax-m2-5","name":"MiniMax M2.5","provider":"MiniMax","description":"MiniMax M2.5 is a standard model from MiniMax. 10 benchmarks count toward its index, in 5 categories.","logoUrl":null,"releaseDate":"2026-02-12","contextWindowTokens":200000,"modalities":["text"],"openWeights":true,"details":{"apiId":"minimax/minimax-m2.5","maxOutputTokens":128000,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.027,"weightsUrl":"https://huggingface.co/MiniMaxAI/MiniMax-M2.5"},"reasoning":false,"overallScore":50.16457248543621,"overallScoreRange":{"low":43.05116950894394,"high":57.27797546192848},"badges":[],"capabilityScores":{"agentic":51.5828357452562,"coding":50.66911907239445,"reasoning":48.83933329277864,"multimodal":null,"knowledge":48.934891854900044,"multilingual":null,"instruction":null,"math":54.290377228779015},"capabilityCounts":{"math":1,"knowledge":2,"coding":4,"reasoning":2,"agentic":1},"benchmarks":[{"benchmarkId":"vals-ai:medqa:overall","benchmarkName":"MedQA","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":92.525,"unit":"percent","family":"vals-ai:medqa:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medqa","measuredAt":"2026-04-16","stderr":0.24,"costPerTest":0.002928,"latency":33.54},{"benchmarkId":"vals-ai:aime:overall","benchmarkName":"AIME","category":"math","domain":null,"eligible":true,"ranked":true,"score":88.75,"unit":"percent","family":"vals-ai:aime:overall","index":54.290377228779015,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/aime","measuredAt":"2026-04-16","stderr":0.877,"costPerTest":0.041379,"latency":373.487},{"benchmarkId":"vals-ai:gpqa:overall","benchmarkName":"GPQA Diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":82.07,"unit":"percent","family":"gpqa-diamond","index":52.48146216851628,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","measuredAt":"2026-09-01","stderr":2.258,"costPerTest":0.017869,"latency":171.962},{"benchmarkId":"vals-ai:mmlu_pro:overall","benchmarkName":"MMLU Pro","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":80.093,"unit":"percent","family":"mmlu-pro","index":45.3883215412838,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","measuredAt":"2026-09-01","stderr":0.39,"costPerTest":0.005362,"latency":48.271},{"benchmarkId":"vals-ai:legal_bench:overall","benchmarkName":"LegalBench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":79.963,"unit":"percent","family":"vals-ai:legal_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","measuredAt":"2026-10-13","stderr":0.473,"costPerTest":0.001287,"latency":15.819},{"benchmarkId":"vals-ai:lcb:overall","benchmarkName":"LiveCodeBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":79.208,"unit":"percent","family":"livecodebench","index":55.06590977034774,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","measuredAt":"2026-09-01","stderr":1.119,"costPerTest":0.033052,"latency":312.011},{"benchmarkId":"swebench-experiments:swe-bench-verified:overall","benchmarkName":"SWE-bench Verified","category":"coding","domain":null,"eligible":true,"ranked":true,"score":75.8,"unit":"percent","family":"swe-bench-verified","index":57.1347919254755,"aboveFit":false,"run":"high effort · mini-SWE-agent","source":"SWE-bench team","sourceUrl":"https://github.com/SWE-bench/experiments/tree/main/evaluation/verified","measuredAt":"2026-09-01","stderr":null,"costPerTest":0.073289987844,"latency":null},{"benchmarkId":"vals-ai:swebench:overall","benchmarkName":"SWE-bench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":74.2,"unit":"percent","family":"swe-bench-verified","index":55.95469980102159,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/swebench","measuredAt":"2026-09-01","stderr":1.959,"costPerTest":0.456928,"latency":402.941},{"benchmarkId":"swe-bench:swe-bench-multilingual:overall","benchmarkName":"SWE-bench Multilingual","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":68.3,"unit":"percent","family":"swe-bench-multilingual","index":null,"aboveFit":false,"run":"mini-SWE-agent","source":"SWE-bench team","sourceUrl":"https://www.swebench.com/","measuredAt":"2026-02-20","stderr":null,"costPerTest":0.10121049956228956,"latency":null},{"benchmarkId":"vals-ai:tax_eval_v2:overall","benchmarkName":"TaxEval v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":68.152,"unit":"percent","family":"vals-ai:tax_eval_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","measuredAt":"2026-09-01","stderr":0.906,"costPerTest":0.008629,"latency":73.243},{"benchmarkId":"arc-prize:v1-semi-private:overall","benchmarkName":"ARC-AGI-1 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":63.67,"unit":"percent","family":"arc-prize:v1-semi-private:overall","index":56.258756536570615,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.07,"latency":null},{"benchmarkId":"vals-ai:corp_fin_v2:overall","benchmarkName":"CorpFin v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":59.596,"unit":"percent","family":"vals-ai:corp_fin_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","measuredAt":"2026-08-12","stderr":0.966,"costPerTest":0.024284,"latency":16.599},{"benchmarkId":"arc-prize:v1-public-eval:overall","benchmarkName":"ARC-AGI-1 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":59.13,"unit":"percent","family":"arc-prize:v1-public-eval:overall","index":null,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.09,"latency":null},{"benchmarkId":"vals-ai:case_law_v2:overall","benchmarkName":"CaseLaw v2","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":53.483,"unit":"percent","family":"vals-ai:case_law_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/case_law_v2","measuredAt":"2026-05-04","stderr":0.467,"costPerTest":0.012109,"latency":30.875},{"benchmarkId":"vals-ai:terminal-bench-2:overall","benchmarkName":"Terminal-Bench 2.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":41.573,"unit":"percent","family":"vals-ai:terminal-bench-2:overall","index":51.5828357452562,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2","measuredAt":"2026-06-04","stderr":5.254,"costPerTest":0.085163,"latency":754.513},{"benchmarkId":"vals-ai:finance_agent:overall","benchmarkName":"Finance Agent v1.1","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":38.579,"unit":"percent","family":"vals-ai:finance_agent:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/finance_agent","measuredAt":"2026-06-04","stderr":2.646,"costPerTest":0.164656,"latency":1355.194},{"benchmarkId":"vals-ai:vibe-code:overall","benchmarkName":"Vibe Code Bench v1.1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":14.853,"unit":"percent","family":"vals-ai:vibe-code:overall","index":47.608718349093216,"aboveFit":false,"run":"OpenHands","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","measuredAt":"2026-11-09","stderr":2.953,"costPerTest":2.197729,"latency":3065.622},{"benchmarkId":"vals-ai:ioi-v1:overall","benchmarkName":"IOI v1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":6.666,"unit":"percent","family":"vals-ai:ioi-v1:overall","index":43.45710230688831,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/ioi-v1","measuredAt":"2026-08-09","stderr":2.641,"costPerTest":1.501416,"latency":22887.998},{"benchmarkId":"arc-prize:v2-public-eval:overall","benchmarkName":"ARC-AGI-2 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":5.42,"unit":"percent","family":"arc-prize:v2-public-eval:overall","index":null,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.19,"latency":null},{"benchmarkId":"arc-prize:v2-semi-private:overall","benchmarkName":"ARC-AGI-2 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":4.859999999999999,"unit":"percent","family":"arc-prize:v2-semi-private:overall","index":41.419910048986665,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.17,"latency":null}],"price":{"inputPerMillion":0.27,"outputPerMillion":1.08,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":44,"timeToFirstToken":0.651,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":{"elo":1359.3004164775448,"eloUpper":1363.1802213149645,"eloLower":1355.420611640125,"votes":42459,"rank":203,"rankUpper":null,"rankLower":null,"arenaName":"MiniMax-M2.5","organization":"minimax","updatedAt":"2026-11-09"},"quality":{"coveragePercent":80,"benchmarkCount":10,"categoryCount":5,"confidence":"low","state":"available"},"hosts":[{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":0.27,"outputPerMillion":1.08,"cacheReadPerMillion":0.027,"contextTokens":200000,"maxOutputTokens":128000,"deprecated":false},{"host":"deepinfra","name":"Deep Infra","kind":"host","url":"https://deepinfra.com/models","inputPerMillion":0.15,"outputPerMillion":1.15,"cacheReadPerMillion":0.03,"contextTokens":196608,"maxOutputTokens":131072,"deprecated":true},{"host":"amazon-bedrock","name":"Amazon Bedrock","kind":"cloud","url":"https://docs.aws.amazon.com/bedrock/latest/userguide/models-supported.html","inputPerMillion":0.3,"outputPerMillion":1.2,"cacheReadPerMillion":null,"contextTokens":196608,"maxOutputTokens":98304,"deprecated":false},{"host":"novita-ai","name":"Novita AI","kind":"host","url":"https://novita.ai/docs/guides/introduction","inputPerMillion":0.3,"outputPerMillion":1.2,"cacheReadPerMillion":0.03,"contextTokens":204800,"maxOutputTokens":131100,"deprecated":false},{"host":"siliconflow","name":"SiliconFlow","kind":"host","url":"https://cloud.siliconflow.com/models","inputPerMillion":0.3,"outputPerMillion":1.2,"cacheReadPerMillion":0.03,"contextTokens":197000,"maxOutputTokens":131000,"deprecated":true},{"host":"together-ai","name":"Together AI","kind":"host","url":"https://docs.together.ai/docs/serverless-models","inputPerMillion":0.3,"outputPerMillion":1.2,"cacheReadPerMillion":0.06,"contextTokens":204800,"maxOutputTokens":131072,"deprecated":true}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:swe-bench-team","name":"SWE-bench team","url":"https://github.com","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:arc-prize-foundation","name":"ARC Prize Foundation","url":"https://arcprize.org","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"glm-4-7","providerId":"z-ai","slug":"glm-4-7","name":"GLM-4.7","provider":"Z.AI","description":"GLM-4.7 is a reasoning model from Z.AI. 27 benchmarks count toward its index, in 6 categories.","logoUrl":null,"releaseDate":"2025-12-22","contextWindowTokens":204800,"modalities":["text"],"openWeights":true,"details":{"apiId":"z-ai/glm-4.7","maxOutputTokens":131072,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.11,"weightsUrl":"https://huggingface.co/zai-org/GLM-4.7"},"reasoning":true,"overallScore":50.049265808073606,"overallScoreRange":{"low":45.437347857141525,"high":54.66118375900569},"badges":[],"capabilityScores":{"agentic":48.803101535775326,"coding":53.15635057111507,"reasoning":48.69129308340151,"multimodal":null,"knowledge":50.86932931668776,"multilingual":null,"instruction":52.476900138988164,"math":49.1203002607173},"capabilityCounts":{"math":6,"agentic":8,"coding":4,"knowledge":5,"reasoning":3,"instruction":1},"benchmarks":[{"benchmarkId":"hmmtFeb2025","benchmarkName":"Harvard-MIT Mathematics Tournament February 2025","category":"math","score":97.1,"unit":"percent","family":"hmmtFeb2025","index":54.820355965316104,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"tau2Bench","benchmarkName":"τ²-Bench Tool-Agent-User Evaluation","category":"agentic","score":95.9,"unit":"percent","family":"tau2-bench","index":66.42610157862087,"aboveFit":false,"run":null,"source":"Victor Barres, Honghua Dong, Soham Ray, Xujie Si, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2506.07982","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aime2025","benchmarkName":"American Invitational Mathematics Examination 2025","category":"math","score":95.7,"unit":"percent","family":"aime2025","index":50.93651238312302,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaAime2025","benchmarkName":"Artificial Analysis AIME 2025","category":"math","score":95,"unit":"percent","family":"aaAime2025","index":null,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/aime-2025","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:medqa:overall","benchmarkName":"MedQA","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":93.742,"unit":"percent","family":"vals-ai:medqa:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medqa","measuredAt":"2026-04-16","stderr":0.221,"costPerTest":0.00487,"latency":66.008},{"benchmarkId":"vals-ai:aime:overall","benchmarkName":"AIME","category":"math","domain":null,"eligible":true,"ranked":true,"score":93.333,"unit":"percent","family":"vals-ai:aime:overall","index":56.29335794885142,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/aime","measuredAt":"2026-04-16","stderr":0.445,"costPerTest":0.043308,"latency":529.192},{"benchmarkId":"aaLiveCodeBench","benchmarkName":"Artificial Analysis LiveCodeBench","category":"coding","score":89.4,"unit":"percent","family":"livecodebench","index":63.522632376343665,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/livecodebench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:mgsm:overall","benchmarkName":"MGSM","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":88.182,"unit":"percent","family":"vals-ai:mgsm:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mgsm","measuredAt":"2026-01-09","stderr":1.828,"costPerTest":0.004315,"latency":63.175},{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":85.9,"unit":"percent","family":"gpqa-diamond","index":55.99423351345907,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gpqa","benchmarkName":"Graduate-Level Google-Proof Q&A","category":"knowledge","score":85.7,"unit":"percent","family":"gpqa-diamond","index":55.81079897847511,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaMmluPro","benchmarkName":"Artificial Analysis MMLU-Pro","category":"knowledge","score":85.6,"unit":"percent","family":"mmlu-pro","index":52.32648237032813,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmlu-pro","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"liveCodeBenchV6","benchmarkName":"LiveCodeBench v6","category":"coding","score":84.9,"unit":"percent","family":"liveCodeBenchV6","index":54.12482289464184,"aboveFit":false,"run":null,"source":"LiveCodeBench maintainers","sourceUrl":"https://github.com/LiveCodeBench/LiveCodeBench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmluPro","benchmarkName":"Massive Multitask Language Understanding Professional","category":"knowledge","score":84.3,"unit":"percent","family":"mmlu-pro","index":50.688637976328195,"aboveFit":false,"run":null,"source":"Yubo Wang, Xueguang Ma, Ge Zhang, Yuansheng Ni, Abhranil Chandra, Shiguang Guo, Weiming Ren, Aaran Arulraj, Xuan He, Ziyan Jiang, Tianle Li, Max Ku, Kai Wang, Alex Zhuang, Rongqi Fan, Xiang Yue, Wenhu Chen","sourceUrl":"https://arxiv.org/abs/2406.01574","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:legal_bench:overall","benchmarkName":"LegalBench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":83.358,"unit":"percent","family":"vals-ai:legal_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","measuredAt":"2026-10-13","stderr":0.473,"costPerTest":0.002348,"latency":30.675},{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":83.33333333333334,"unit":"percent","family":"gpqa-diamond","index":53.64015698116496,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.3749016662636553,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":83.33333333333334,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":57.40192802098858,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":4.767312946227961,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:mmlu_pro:overall","benchmarkName":"MMLU Pro","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":82.736,"unit":"percent","family":"mmlu-pro","index":48.71818518231598,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","measuredAt":"2026-09-01","stderr":0.371,"costPerTest":0.00798,"latency":104.451},{"benchmarkId":"vals-ai:lcb:overall","benchmarkName":"LiveCodeBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":82.234,"unit":"percent","family":"livecodebench","index":57.576706729310025,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","measuredAt":"2026-09-01","stderr":0.993,"costPerTest":0.035469,"latency":393.276},{"benchmarkId":"vals-ai:gpqa:overall","benchmarkName":"GPQA Diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":80.05,"unit":"percent","family":"gpqa-diamond","index":50.62877336517832,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","measuredAt":"2026-09-01","stderr":2.006,"costPerTest":0.024866,"latency":211.079},{"benchmarkId":"sweVerified","benchmarkName":"Software Engineering Benchmark Verified","category":"coding","score":73.8,"unit":"percent","family":"swe-bench-verified","index":55.65967676990811,"aboveFit":false,"run":null,"source":"Carlos E. Jimenez, John Yang, Alexander Wettig, Shunyu Yao, Kexin Pei, Ofir Press, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2310.06770","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":71,"unit":"percent","family":"lcr","index":56.00976677575258,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:swebench:overall","benchmarkName":"SWE-bench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":69.4,"unit":"percent","family":"swe-bench-verified","index":52.41442342765987,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/swebench","measuredAt":"2026-09-01","stderr":2.063,"costPerTest":0.203695,"latency":365.799},{"benchmarkId":"vals-ai:tax_eval_v2:overall","benchmarkName":"TaxEval v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":68.766,"unit":"percent","family":"vals-ai:tax_eval_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","measuredAt":"2026-09-01","stderr":0.921,"costPerTest":0.013873,"latency":187.181},{"benchmarkId":"vals-ai:medscribe:overall","benchmarkName":"MedScribe","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":68.629,"unit":"percent","family":"vals-ai:medscribe:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medscribe","measuredAt":"2026-10-29","stderr":2.123,"costPerTest":0.019082,"latency":167.479},{"benchmarkId":"aaIfBench","benchmarkName":"Artificial Analysis IFBench","category":"instruction","score":67.9,"unit":"percent","family":"ifbench-family","index":52.476900138988164,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"sweRebench","benchmarkName":"SWE-Rebench","category":"coding","score":58.7,"unit":"percent","family":"sweRebench","index":null,"aboveFit":false,"run":null,"source":"Nebius","sourceUrl":"https://swe-rebench.com","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:case_law_v2:overall","benchmarkName":"CaseLaw v2","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":54.877,"unit":"percent","family":"vals-ai:case_law_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/case_law_v2","measuredAt":"2026-05-04","stderr":0.916,"costPerTest":0.021159,"latency":59.643},{"benchmarkId":"browseComp","benchmarkName":"BrowseComp","category":"agentic","score":52,"unit":"percent","family":"browseComp","index":40.99002396256216,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:terminal-bench:overall","benchmarkName":"Terminal-Bench 1.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":50,"unit":"percent","family":"vals-ai:terminal-bench:overall","index":54.402056932447785,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench","measuredAt":"2026-01-12","stderr":5.59,"costPerTest":0.223346,"latency":562.429},{"benchmarkId":"vals-ai:corp_fin_v2:overall","benchmarkName":"CorpFin v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":46.387,"unit":"percent","family":"vals-ai:corp_fin_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","measuredAt":"2026-08-12","stderr":0.899,"costPerTest":0.039082,"latency":66.101},{"benchmarkId":"vals-ai:finance_agent:overall","benchmarkName":"Finance Agent v1.1","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":45.977,"unit":"percent","family":"vals-ai:finance_agent:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/finance_agent","measuredAt":"2026-06-04","stderr":2.818,"costPerTest":0.244376,"latency":605.003},{"benchmarkId":"terminalBench2","benchmarkName":"Terminal-Bench 2.0","category":"agentic","score":41,"unit":"percent","family":"terminalBench2","index":43.06280379859746,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gertLabs","benchmarkName":"Gert Labs Composite Game Benchmark","category":"agentic","score":39.95,"unit":"percent","family":"gertLabs","index":47.89494251099424,"aboveFit":false,"run":null,"source":"Gert Labs","sourceUrl":"https://gertlabs.com/rankings","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:terminal-bench-2:overall","benchmarkName":"Terminal-Bench 2.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":38.202,"unit":"percent","family":"vals-ai:terminal-bench-2:overall","index":49.55311870259774,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2","measuredAt":"2026-06-04","stderr":5.18,"costPerTest":0.183983,"latency":777.512},{"benchmarkId":"vals-ai:medcode:overall","benchmarkName":"MedCode","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":32.772,"unit":"percent","family":"vals-ai:medcode:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medcode","measuredAt":"2026-10-29","stderr":1.996,"costPerTest":0.00671,"latency":123.441},{"benchmarkId":"epoch-ai:epoch-simpleqa-verified:overall","benchmarkName":"SimpleQA Verified","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":32.2,"unit":"percent","family":"epoch-ai:epoch-simpleqa-verified:overall","index":50.74449486913045,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.4782913600996743,"costPerTest":null,"latency":null},{"benchmarkId":"omniscienceAccuracy","benchmarkName":"Artificial Analysis Omniscience Accuracy","category":"knowledge","score":29.3,"unit":"percent","family":"omniscienceAccuracy","index":47.615450799569,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":27.4,"unit":"percent","family":"hle","index":51.87686552712984,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gdpvalAaNormalized","benchmarkName":"GDPval-AA normalized","category":"agentic","score":25.7,"unit":"percent","family":"gdpvalAaNormalized","index":52.40867769267287,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hle","benchmarkName":"Humanity's Last Exam","category":"knowledge","score":24.8,"unit":"percent","family":"hle","index":49.26830439005238,"aboveFit":false,"run":null,"source":"Center for AI Safety, Scale AI, and thousands of expert contributors","sourceUrl":"https://lastexam.ai/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-chess-puzzles:overall","benchmarkName":"Chess Puzzles","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":21,"unit":"percent","family":"epoch-ai:epoch-chess-puzzles:overall","index":49.923994823290954,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":4.093601807403323,"costPerTest":null,"latency":null},{"benchmarkId":"vitaBench","benchmarkName":"VITA-Bench","category":"agentic","score":15.5,"unit":"percent","family":"vitaBench","index":35.6870871077095,"aboveFit":false,"run":null,"source":"Meituan LongCat Team","sourceUrl":"https://vitabench.github.io/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:ioi-v1:overall","benchmarkName":"IOI v1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":7.584,"unit":"percent","family":"vals-ai:ioi-v1:overall","index":43.9138597382076,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/ioi-v1","measuredAt":"2026-08-09","stderr":3.404,"costPerTest":0.443515,"latency":5316.876},{"benchmarkId":"epoch-ai:epoch-frontiermath-2025-02-28-private:overall","benchmarkName":"FrontierMath-2025-02-28-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":2.4390243902439024,"unit":"percent","family":"epoch-ai:epoch-frontiermath-2025-02-28-private:overall","index":32.06728951885907,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0.9121434048862995,"costPerTest":null,"latency":null},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":1.7,"unit":"percent","family":"critpt","index":40.14011765116101,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-private:overall","benchmarkName":"FrontierMath-Tier-4-2025-07-01-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":0,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-private:overall","index":43.20235772716562,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":0.6,"outputPerMillion":2.2,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":28,"timeToFirstToken":1.079,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":{"elo":1435.4321783953014,"eloUpper":1441.4654206555474,"eloLower":1429.3989361350555,"votes":12167,"rank":89,"rankUpper":null,"rankLower":null,"arenaName":"GLM-4.7","organization":"zai","updatedAt":"2026-11-09"},"quality":{"coveragePercent":85,"benchmarkCount":27,"categoryCount":6,"confidence":"medium","state":"available"},"hosts":[{"host":"deepinfra","name":"Deep Infra","kind":"host","url":"https://deepinfra.com/models","inputPerMillion":0.4,"outputPerMillion":1.75,"cacheReadPerMillion":0.08,"contextTokens":202752,"maxOutputTokens":16384,"deprecated":false},{"host":"zhipu-ai","name":"Zhipu AI","kind":"maker","url":"https://docs.bigmodel.cn/cn/guide/start/pricing","inputPerMillion":0.44819,"outputPerMillion":2.09153,"cacheReadPerMillion":0.08964,"contextTokens":204800,"maxOutputTokens":131072,"deprecated":false},{"host":"amazon-bedrock","name":"Amazon Bedrock","kind":"cloud","url":"https://docs.aws.amazon.com/bedrock/latest/userguide/models-supported.html","inputPerMillion":0.6,"outputPerMillion":2.2,"cacheReadPerMillion":null,"contextTokens":202752,"maxOutputTokens":131072,"deprecated":true},{"host":"novita-ai","name":"Novita AI","kind":"host","url":"https://novita.ai/docs/guides/introduction","inputPerMillion":0.6,"outputPerMillion":2.2,"cacheReadPerMillion":0.11,"contextTokens":204800,"maxOutputTokens":131072,"deprecated":false},{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":0.6,"outputPerMillion":2.2,"cacheReadPerMillion":0.11,"contextTokens":204800,"maxOutputTokens":131072,"deprecated":false},{"host":"z-ai","name":"Z.AI","kind":"maker","url":"https://docs.z.ai/guides/overview/pricing","inputPerMillion":0.6,"outputPerMillion":2.2,"cacheReadPerMillion":0.11,"contextTokens":204800,"maxOutputTokens":131072,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"grok-4","providerId":"xai","slug":"grok-4","name":"Grok 4","provider":"xAI","description":"Grok 4 is a standard model from xAI. 21 benchmarks count toward its index, in 7 categories.","logoUrl":null,"releaseDate":"2025-07-09","contextWindowTokens":128000,"modalities":["text"],"openWeights":false,"details":null,"reasoning":false,"overallScore":50.044174270326906,"overallScoreRange":{"low":46.015590985210906,"high":54.072757555442905},"badges":[],"capabilityScores":{"agentic":48.01160544981344,"coding":50.91383829997783,"reasoning":47.33620392258314,"multimodal":49.55075367825432,"knowledge":55.82999574799266,"multilingual":null,"instruction":39.20684940840921,"math":50.54390809944074},"capabilityCounts":{"math":5,"knowledge":4,"coding":4,"multimodal":1,"agentic":4,"reasoning":2,"instruction":1},"benchmarks":[{"benchmarkId":"aaMath500","benchmarkName":"Artificial Analysis MATH-500","category":"math","score":99,"unit":"percent","family":"math-500","index":48.745846286935404,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/math-500","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:math500:overall","benchmarkName":"MATH 500","category":"math","domain":null,"eligible":true,"ranked":true,"score":96.2,"unit":"percent","family":"math-500","index":45.786923292131746,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/math500","measuredAt":"2026-01-09","stderr":0.855,"costPerTest":0.037425,"latency":27.261},{"benchmarkId":"aaAime2025","benchmarkName":"Artificial Analysis AIME 2025","category":"math","score":92.7,"unit":"percent","family":"aaAime2025","index":null,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/aime-2025","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:medqa:overall","benchmarkName":"MedQA","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":92.492,"unit":"percent","family":"vals-ai:medqa:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medqa","measuredAt":"2026-04-16","stderr":0.24,"costPerTest":0.025753,"latency":64.247},{"benchmarkId":"vals-ai:mgsm:overall","benchmarkName":"MGSM","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":90.909,"unit":"percent","family":"vals-ai:mgsm:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mgsm","measuredAt":"2026-01-09","stderr":1.778,"costPerTest":0.026131,"latency":116.619},{"benchmarkId":"vals-ai:aime:overall","benchmarkName":"AIME","category":"math","domain":null,"eligible":true,"ranked":true,"score":90.556,"unit":"percent","family":"vals-ai:aime:overall","index":55.079681872124155,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/aime","measuredAt":"2026-04-16","stderr":1.757,"costPerTest":0.174399,"latency":133.228},{"benchmarkId":"aaGlobalMmluLite","benchmarkName":"Artificial Analysis Global-MMLU-Lite","category":"multilingual","score":89.5,"unit":"percent","family":"aaGlobalMmluLite","index":null,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/global-mmlu-lite","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:gpqa:overall","benchmarkName":"GPQA Diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":88.132,"unit":"percent","family":"gpqa-diamond","index":58.04136292388003,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","measuredAt":"2026-09-01","stderr":1.625,"costPerTest":0.147971,"latency":115.516},{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":87.7,"unit":"percent","family":"gpqa-diamond","index":57.64514432831468,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":87,"unit":"percent","family":"gpqa-diamond","index":57.00312345587083,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2,"costPerTest":null,"latency":null},{"benchmarkId":"aaMmluPro","benchmarkName":"Artificial Analysis MMLU-Pro","category":"knowledge","score":86.6,"unit":"percent","family":"mmlu-pro","index":53.586362673405,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmlu-pro","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:mmlu_pro:overall","benchmarkName":"MMLU Pro","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":85.304,"unit":"percent","family":"mmlu-pro","index":51.95355780061738,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","measuredAt":"2026-09-01","stderr":0.349,"costPerTest":0.05025,"latency":89.255},{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":84,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":57.78979071814397,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":5,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:lcb:overall","benchmarkName":"LiveCodeBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":83.247,"unit":"percent","family":"livecodebench","index":58.41723459428981,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","measuredAt":"2026-09-01","stderr":1.026,"costPerTest":0.155988,"latency":229.402},{"benchmarkId":"vals-ai:legal_bench:overall","benchmarkName":"LegalBench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":83.192,"unit":"percent","family":"vals-ai:legal_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","measuredAt":"2026-10-13","stderr":0.469,"costPerTest":null,"latency":27.346},{"benchmarkId":"aaLiveCodeBench","benchmarkName":"Artificial Analysis LiveCodeBench","category":"coding","score":81.9,"unit":"percent","family":"livecodebench","index":57.29957315882312,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/livecodebench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:medscribe:overall","benchmarkName":"MedScribe","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":78.152,"unit":"percent","family":"vals-ai:medscribe:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medscribe","measuredAt":"2026-10-29","stderr":2.084,"costPerTest":0.063955,"latency":74.054},{"benchmarkId":"vals-ai:mmmu:overall","benchmarkName":"MMMU Pro","category":"multimodal","domain":null,"eligible":true,"ranked":true,"score":76.27,"unit":"percent","family":"mmmu-pro","index":54.4115284884744,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmmu","measuredAt":"2026-09-01","stderr":1.022,"costPerTest":0.097579,"latency":102.831},{"benchmarkId":"tau2Bench","benchmarkName":"τ²-Bench Tool-Agent-User Evaluation","category":"agentic","score":74.9,"unit":"percent","family":"tau2-bench","index":52.49106622826716,"aboveFit":false,"run":null,"source":"Victor Barres, Honghua Dong, Soham Ray, Xujie Si, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2506.07982","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"reactNativeEvals","benchmarkName":"React Native Evals","category":"coding","score":72.6,"unit":"percent","family":"reactNativeEvals","index":48.77823019680837,"aboveFit":false,"run":null,"source":"Callstack","sourceUrl":"https://rn-evals.vercel.app/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaMmmuPro","benchmarkName":"Artificial Analysis MMMU-Pro","category":"multimodal","score":68.8,"unit":"percent","family":"mmmu-pro","index":44.68997886803424,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmmu-pro","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":68,"unit":"percent","family":"lcr","index":53.9515251989956,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:corp_fin_v2:overall","benchmarkName":"CorpFin v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":66.045,"unit":"percent","family":"vals-ai:corp_fin_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","measuredAt":"2026-08-12","stderr":0.932,"costPerTest":0.190665,"latency":29.733},{"benchmarkId":"vals-ai:case_law_v2:overall","benchmarkName":"CaseLaw v2","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":65.809,"unit":"percent","family":"vals-ai:case_law_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/case_law_v2","measuredAt":"2026-05-04","stderr":0.214,"costPerTest":0.158935,"latency":32.335},{"benchmarkId":"vals-ai:tax_eval_v2:overall","benchmarkName":"TaxEval v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":65.086,"unit":"percent","family":"vals-ai:tax_eval_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","measuredAt":"2026-09-01","stderr":0.962,"costPerTest":0.077989,"latency":159.421},{"benchmarkId":"vals-ai:swebench:overall","benchmarkName":"SWE-bench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":57.8,"unit":"percent","family":"swe-bench-verified","index":43.85875552536904,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/swebench","measuredAt":"2026-09-01","stderr":2.211,"costPerTest":1.415174,"latency":587.945},{"benchmarkId":"aaIfBench","benchmarkName":"Artificial Analysis IFBench","category":"instruction","score":53.7,"unit":"percent","family":"ifbench-family","index":39.20684940840921,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:finance_agent:overall","benchmarkName":"Finance Agent v1.1","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":53.506,"unit":"percent","family":"vals-ai:finance_agent:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/finance_agent","measuredAt":"2026-06-04","stderr":2.854,"costPerTest":1.074806,"latency":321.035},{"benchmarkId":"vals-ai:mortgage_tax:overall","benchmarkName":"MortgageTax","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":44.475,"unit":"percent","family":"vals-ai:mortgage_tax:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mortgage_tax","measuredAt":"2026-09-01","stderr":0.963,"costPerTest":0.08575,"latency":65.022},{"benchmarkId":"gertLabs","benchmarkName":"Gert Labs Composite Game Benchmark","category":"agentic","score":42.34,"unit":"percent","family":"gertLabs","index":49.83016563061468,"aboveFit":false,"run":null,"source":"Gert Labs","sourceUrl":"https://gertlabs.com/rankings","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"omniscienceAccuracy","benchmarkName":"Artificial Analysis Omniscience Accuracy","category":"knowledge","score":40.5,"unit":"percent","family":"omniscienceAccuracy","index":61.730317821035804,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:terminal-bench:overall","benchmarkName":"Terminal-Bench 1.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":38.75,"unit":"percent","family":"vals-ai:terminal-bench:overall","index":46.26062025449392,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench","measuredAt":"2026-01-12","stderr":5.447,"costPerTest":7.31463,"latency":902.175},{"benchmarkId":"vals-ai:medcode:overall","benchmarkName":"MedCode","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":38.078,"unit":"percent","family":"vals-ai:medcode:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medcode","measuredAt":"2026-10-29","stderr":2.206,"costPerTest":0.034103,"latency":89.002},{"benchmarkId":"vals-ai:terminal-bench-2:overall","benchmarkName":"Terminal-Bench 2.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":28.09,"unit":"percent","family":"vals-ai:terminal-bench-2:overall","index":43.464569685878004,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2","measuredAt":"2026-06-04","stderr":4.791,"costPerTest":0.552755,"latency":885.185},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":26.7,"unit":"percent","family":"hle","index":51.17456060560899,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:ioi-v1:overall","benchmarkName":"IOI v1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":26.167,"unit":"percent","family":"vals-ai:ioi-v1:overall","index":53.159963601177466,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/ioi-v1","measuredAt":"2026-08-09","stderr":6.763,"costPerTest":2.693986,"latency":4265.882},{"benchmarkId":"epoch-ai:epoch-frontiermath-2025-02-28-private:overall","benchmarkName":"FrontierMath-2025-02-28-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":19.655172413793103,"unit":"percent","family":"epoch-ai:epoch-frontiermath-2025-02-28-private:overall","index":47.26531980646258,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.3375906908472146,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-private:overall","benchmarkName":"FrontierMath-Tier-4-2025-07-01-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":2.083333333333333,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-private:overall","index":45.31836331093945,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.083333333333333,"costPerTest":null,"latency":null},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":2,"unit":"percent","family":"critpt","index":40.72088264617068,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-public:overall","benchmarkName":"FrontierMath-Tier-4-2025-07-01-Public","category":"math","domain":null,"eligible":false,"ranked":true,"score":0,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-public:overall","index":null,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":78,"timeToFirstToken":11.6,"source":"BenchLM","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":{"elo":1411.4538602298062,"eloUpper":1415.3827291743319,"eloLower":1407.5249912852805,"votes":39584,"rank":141,"rankUpper":null,"rankLower":null,"arenaName":"Grok 4-0709","organization":"xai","updatedAt":"2026-11-09"},"quality":{"coveragePercent":95,"benchmarkCount":21,"categoryCount":7,"confidence":"medium","state":"available"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gemini31flashlitepreview","providerId":"google","slug":"google-gemini-3-1-flash-lite-preview","name":"Gemini 3.1 Flash Lite Preview","provider":"Google","description":"Gemini 3.1 Flash Lite Preview is a model from Google. 11 benchmarks count toward its index, in 6 categories.","logoUrl":null,"releaseDate":"2026-03-03T04:37:53.000Z","contextWindowTokens":1048576,"modalities":["text","image","video","file","audio"],"openWeights":null,"details":{"apiId":"google/gemini-3.1-flash-lite-preview","maxOutputTokens":65536,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.024999999999999998,"weightsUrl":null},"reasoning":null,"overallScore":49.55727963844518,"overallScoreRange":{"low":42.68184119662274,"high":56.43271808026762},"badges":[],"capabilityScores":{"agentic":41.2545071912713,"coding":48.31325096200897,"reasoning":null,"multimodal":62.50110712845514,"knowledge":52.34396176557232,"multilingual":57.261254119540865,"instruction":null,"math":51.92290034450403},"capabilityCounts":{"knowledge":2,"math":1,"multimodal":1,"coding":4,"multilingual":1,"agentic":2},"benchmarks":[{"benchmarkId":"vals-ai:mmlu_pro:overall","benchmarkName":"MMLU Pro","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":86.24,"unit":"percent","family":"mmlu-pro","index":53.13280576429732,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","measuredAt":"2026-09-01","stderr":0.34,"costPerTest":0.001052,"latency":14.511},{"benchmarkId":"vals-ai:legal_bench:overall","benchmarkName":"LegalBench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":83.764,"unit":"percent","family":"vals-ai:legal_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","measuredAt":"2026-10-13","stderr":0.406,"costPerTest":0.000295,"latency":2.704},{"benchmarkId":"vals-ai:aime:overall","benchmarkName":"AIME","category":"math","domain":null,"eligible":true,"ranked":true,"score":83.333,"unit":"percent","family":"vals-ai:aime:overall","index":51.92290034450403,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/aime","measuredAt":"2026-04-16","stderr":1.178,"costPerTest":0.006966,"latency":40.564},{"benchmarkId":"vals-ai:mmmu:overall","benchmarkName":"MMMU Pro","category":"multimodal","domain":null,"eligible":true,"ranked":true,"score":82.486,"unit":"percent","family":"mmmu-pro","index":62.50110712845514,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmmu","measuredAt":"2026-09-01","stderr":0.914,"costPerTest":0.003187,"latency":12.448},{"benchmarkId":"vals-ai:gpqa:overall","benchmarkName":"GPQA Diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":81.06,"unit":"percent","family":"gpqa-diamond","index":51.55511776684731,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","measuredAt":"2026-09-01","stderr":1.967,"costPerTest":0.002298,"latency":16.464},{"benchmarkId":"vals-ai:lcb:overall","benchmarkName":"LiveCodeBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":80.116,"unit":"percent","family":"livecodebench","index":55.81931480628223,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","measuredAt":"2026-09-01","stderr":1.083,"costPerTest":0.00435,"latency":30.749},{"benchmarkId":"vals-ai:tax_eval_v2:overall","benchmarkName":"TaxEval v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":71.79,"unit":"percent","family":"vals-ai:tax_eval_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","measuredAt":"2026-09-01","stderr":0.88,"costPerTest":0.00096,"latency":7.644},{"benchmarkId":"vals-ai:mortgage_tax:overall","benchmarkName":"MortgageTax","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":68.044,"unit":"percent","family":"vals-ai:mortgage_tax:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mortgage_tax","measuredAt":"2026-09-01","stderr":0.912,"costPerTest":0.00284,"latency":5.368},{"benchmarkId":"euroeval:euroeval-french:overall","benchmarkName":"EuroEval French","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":67.46875,"unit":"percent","family":"euroeval","index":63.759805091411096,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/french","measuredAt":null,"stderr":0.4075517605163791,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-italian:overall","benchmarkName":"EuroEval Italian","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":66.7275,"unit":"percent","family":"euroeval","index":62.754099398177,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/italian","measuredAt":null,"stderr":0.43461549098024566,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":65.47875,"unit":"percent","family":"euroeval","index":61.05983297736608,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":0.5624916666049373,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-portuguese:overall","benchmarkName":"EuroEval Portuguese","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":63.9825,"unit":"percent","family":"euroeval","index":59.02976600468273,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/portuguese","measuredAt":null,"stderr":0.5087914602270758,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:medscribe:overall","benchmarkName":"MedScribe","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":63.902,"unit":"percent","family":"vals-ai:medscribe:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medscribe","measuredAt":"2026-10-29","stderr":1.823,"costPerTest":0.002195,"latency":16.793},{"benchmarkId":"vals-ai:swebench:overall","benchmarkName":"SWE-bench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":62.8,"unit":"percent","family":"swe-bench-verified","index":47.5465434142875,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/swebench","measuredAt":"2026-09-01","stderr":2.164,"costPerTest":0.105389,"latency":132.936},{"benchmarkId":"euroeval:euroeval-dutch:overall","benchmarkName":"EuroEval Dutch","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":61.37555555555556,"unit":"percent","family":"euroeval","index":55.49274223439902,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/dutch","measuredAt":null,"stderr":0.3786216066043436,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":61.02285714285715,"unit":"percent","family":"euroeval","index":55.014211641959996,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.4256447690125595,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:corp_fin_v2:overall","benchmarkName":"CorpFin v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":59.363,"unit":"percent","family":"vals-ai:corp_fin_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","measuredAt":"2026-08-12","stderr":0.967,"costPerTest":0.022927,"latency":4.03},{"benchmarkId":"euroeval:euroeval-spanish:overall","benchmarkName":"EuroEval Spanish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":56.868750000000006,"unit":"percent","family":"euroeval","index":49.37804408192506,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/spanish","measuredAt":null,"stderr":0.4857854464678826,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":55.0075,"unit":"percent","family":"euroeval","index":46.85275609334903,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.4614735230758098,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:case_law_v2:overall","benchmarkName":"CaseLaw v2","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":54.984,"unit":"percent","family":"vals-ai:case_law_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/case_law_v2","measuredAt":"2026-05-04","stderr":0.742,"costPerTest":0.009312,"latency":4.641},{"benchmarkId":"vals-ai:medcode:overall","benchmarkName":"MedCode","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":47.602,"unit":"percent","family":"vals-ai:medcode:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medcode","measuredAt":"2026-10-29","stderr":2.071,"costPerTest":0.002029,"latency":8.55},{"benchmarkId":"vals-ai:finance_agent:overall","benchmarkName":"Finance Agent v1.1","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":46.123,"unit":"percent","family":"vals-ai:finance_agent:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/finance_agent","measuredAt":"2026-06-04","stderr":2.81,"costPerTest":0.071759,"latency":102.442},{"benchmarkId":"vals-ai:terminal-bench-2-1:overall","benchmarkName":"Terminal-Bench 2.1","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":34.082,"unit":"percent","family":"terminal-bench-2-1","index":41.074161739323046,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2-1","measuredAt":"2026-10-01","stderr":0.991,"costPerTest":0.084894,"latency":278.342},{"benchmarkId":"vals-ai:fabv2:overall","benchmarkName":"Finance Agent (v2)","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":29.988,"unit":"percent","family":"vals-ai:fabv2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/fabv2","measuredAt":"2026-10-29","stderr":0.643,"costPerTest":0.142482,"latency":54.778},{"benchmarkId":"vals-ai:terminal-bench-2:overall","benchmarkName":"Terminal-Bench 2.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":24.719,"unit":"percent","family":"vals-ai:terminal-bench-2:overall","index":41.43485264321955,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2","measuredAt":"2026-06-04","stderr":4.599,"costPerTest":0.065496,"latency":667.879},{"benchmarkId":"vals-ai:emb:overall","benchmarkName":"EMB","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":8.629,"unit":"percent","family":"vals-ai:emb:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/emb","measuredAt":"2026-10-20","stderr":1.088,"costPerTest":0.069424,"latency":67.007},{"benchmarkId":"vals-ai:code-migration:overall","benchmarkName":"Code Migration","category":"coding","domain":null,"eligible":true,"ranked":true,"score":4.609,"unit":"percent","family":"vals-ai:code-migration:overall","index":47.63762303650467,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/code-migration","measuredAt":"2026-10-29","stderr":1.069,"costPerTest":0.053028,"latency":92.63},{"benchmarkId":"vals-ai:legal_research:overall","benchmarkName":"Legal Research Bench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":3.365,"unit":"percent","family":"vals-ai:legal_research:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_research","measuredAt":"2026-11-09","stderr":1.253,"costPerTest":0.013607,"latency":20.403},{"benchmarkId":"vals-ai:tax_agent_bench:overall","benchmarkName":"Tax Agent Bench","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":3.109,"unit":"percent","family":"vals-ai:tax_agent_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench","measuredAt":"2026-11-09","stderr":1.253,"costPerTest":0.014308,"latency":18.744},{"benchmarkId":"vals-ai:programbench:overall","benchmarkName":"ProgramBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":0,"unit":"percent","family":"vals-ai:programbench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/programbench","measuredAt":"2026-11-09","stderr":0,"costPerTest":0.063906,"latency":176.246},{"benchmarkId":"vals-ai:vibe-code:overall","benchmarkName":"Vibe Code Bench v1.1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":0,"unit":"percent","family":"vals-ai:vibe-code:overall","index":42.24952259096148,"aboveFit":false,"run":"OpenHands","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","measuredAt":"2026-11-09","stderr":0,"costPerTest":0.51,"latency":572.711}],"price":{"inputPerMillion":0.25,"outputPerMillion":1.5,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":151,"timeToFirstToken":0.7025,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":{"elo":1415.9810521793756,"eloUpper":1419.603730561148,"eloLower":1412.3583737976028,"votes":63629,"rank":132,"rankUpper":null,"rankLower":null,"arenaName":"Gemini 3.1 Flash-Lite Preview","organization":"google","updatedAt":"2026-11-09"},"quality":{"coveragePercent":80,"benchmarkCount":11,"categoryCount":6,"confidence":"low","state":"available"},"hosts":[{"host":"databricks","name":"Databricks","kind":"cloud","url":"https://docs.databricks.com/aws/en/machine-learning/foundation-models/","inputPerMillion":0.25,"outputPerMillion":1.5,"cacheReadPerMillion":0.025,"contextTokens":1048576,"maxOutputTokens":65536,"deprecated":false},{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":0.25,"outputPerMillion":1.5,"cacheReadPerMillion":0.025,"contextTokens":1048576,"maxOutputTokens":65536,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"qwen3-5-35b-a3b","providerId":"alibaba","slug":"qwen3-5-35b-a3b","name":"Qwen3.5-35B-A3B","provider":"Alibaba","description":"Qwen3.5-35B-A3B is a reasoning model from Alibaba. 25 benchmarks count toward its index, in 8 categories.","logoUrl":null,"releaseDate":"2026-03-04","contextWindowTokens":262144,"modalities":["text","image","video"],"openWeights":true,"details":{"apiId":"qwen/qwen3.5-35b-a3b","maxOutputTokens":81920,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.049999999999999996,"weightsUrl":"https://huggingface.co/Qwen/Qwen3.5-35B-A3B"},"reasoning":true,"overallScore":49.43527980691109,"overallScoreRange":{"low":46.20127980691109,"high":52.66927980691109},"badges":[],"capabilityScores":{"agentic":47.25203409664813,"coding":48.96742309604572,"reasoning":48.529882527669685,"multimodal":55.077502011392816,"knowledge":47.413300737506034,"multilingual":50.40787459580192,"instruction":52.51841758741392,"math":49.644674077880765},"capabilityCounts":{"multimodal":6,"instruction":2,"agentic":5,"knowledge":5,"multilingual":1,"coding":2,"reasoning":3,"math":1},"benchmarks":[{"benchmarkId":"vStar","benchmarkName":"V*","category":"multimodal","score":92.7,"unit":"percent","family":"vStar","index":52.62670351272598,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"ifeval","benchmarkName":"Instruction-Following Eval","category":"instruction","score":91.9,"unit":"percent","family":"ifeval","index":48.26118620762397,"aboveFit":false,"run":null,"source":"Jeffrey Zhou, Tianjian Lu, Swaroop Mishra, Siddhartha Brahma, Sujoy Basu, Yi Luan, Denny Zhou, Le Hou","sourceUrl":"https://arxiv.org/abs/2311.07911","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"tau2Bench","benchmarkName":"τ²-Bench Tool-Agent-User Evaluation","category":"agentic","score":89.2,"unit":"percent","family":"tau2-bench","index":61.98016172874612,"aboveFit":false,"run":null,"source":"Victor Barres, Honghua Dong, Soham Ray, Xujie Si, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2506.07982","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"extractBenchLlamaIndex","benchmarkName":"ExtractBench: schema-guided enterprise document extraction (LlamaIndex leaderboard)","category":"multimodal","score":88,"unit":"percent","family":"extractBenchLlamaIndex","index":68.82646374082051,"aboveFit":true,"run":null,"source":"Boyang Zhang, Adrian Lyjak, Eli Stewart, Zhaoqi Li and Simon Suo (LlamaIndex)","sourceUrl":"https://arxiv.org/abs/2607.29677","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaGlobalMmluLite","benchmarkName":"Artificial Analysis Global-MMLU-Lite","category":"multilingual","score":86.4,"unit":"percent","family":"aaGlobalMmluLite","index":null,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/global-mmlu-lite","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmluPro","benchmarkName":"Massive Multitask Language Understanding Professional","category":"knowledge","score":85.3,"unit":"percent","family":"mmlu-pro","index":51.94851827940507,"aboveFit":false,"run":null,"source":"Yubo Wang, Xueguang Ma, Ge Zhang, Yuansheng Ni, Abhranil Chandra, Shiguang Guo, Weiming Ren, Aaran Arulraj, Xuan He, Ziyan Jiang, Tianle Li, Max Ku, Kai Wang, Alex Zhuang, Rongqi Fan, Xiang Yue, Wenhu Chen","sourceUrl":"https://arxiv.org/abs/2406.01574","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":84.5,"unit":"percent","family":"gpqa-diamond","index":54.71019176857136,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gpqa","benchmarkName":"Graduate-Level Google-Proof Q&A","category":"knowledge","score":84.2,"unit":"percent","family":"gpqa-diamond","index":54.43503996609543,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mathVision","benchmarkName":"MathVision","category":"multimodal","score":83.9,"unit":"percent","family":"mathVision","index":55.078232073179215,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmmu","benchmarkName":"Massive Multi-discipline Multimodal Understanding","category":"multimodal","score":81.4,"unit":"percent","family":"mmmu","index":50.18860230867384,"aboveFit":false,"run":null,"source":"MMMU authors","sourceUrl":"https://arxiv.org/abs/2401.05508","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":81.18686868686868,"unit":"percent","family":"gpqa-diamond","index":51.67147825974621,"aboveFit":false,"run":"none effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.319862922976808,"costPerTest":null,"latency":null},{"benchmarkId":"mmluProX","benchmarkName":"MMLU-ProX","category":"multilingual","score":81,"unit":"percent","family":"mmluProX","index":50.40787459580192,"aboveFit":false,"run":null,"source":"MMLU-ProX authors","sourceUrl":"https://arxiv.org/abs/2503.10497","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"charxiv","benchmarkName":"CharXiv Reasoning","category":"multimodal","score":77.5,"unit":"percent","family":"charxiv","index":52.41782901281534,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmmuPro","benchmarkName":"Massive Multi-discipline Multimodal Understanding Pro","category":"multimodal","score":75.1,"unit":"percent","family":"mmmu-pro","index":52.88887613828497,"aboveFit":false,"run":null,"source":"MMMU-Pro authors","sourceUrl":"https://arxiv.org/abs/2409.02813","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"liveCodeBenchV6","benchmarkName":"LiveCodeBench v6","category":"coding","score":74.6,"unit":"percent","family":"liveCodeBenchV6","index":45.667934279988316,"aboveFit":false,"run":null,"source":"LiveCodeBench maintainers","sourceUrl":"https://github.com/LiveCodeBench/LiveCodeBench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaMmmuPro","benchmarkName":"Artificial Analysis MMMU-Pro","category":"multimodal","score":72.7,"unit":"percent","family":"mmmu-pro","index":49.76548670199899,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmmu-pro","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaIfBench","benchmarkName":"Artificial Analysis IFBench","category":"instruction","score":72.5,"unit":"percent","family":"ifbench-family","index":56.77564896720387,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmvu","benchmarkName":"Multimodal Multi-disciplinary Video Understanding","category":"multimodal","score":72.3,"unit":"percent","family":"mmvu","index":null,"aboveFit":false,"run":null,"source":"MMVU benchmark maintainers","sourceUrl":"https://www.kimi.com/blog/kimi-k2-5.html","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":72,"unit":"percent","family":"lcr","index":56.69584730133824,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":70,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":49.644674077880765,"aboveFit":false,"run":"none effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":6.003996985174109,"costPerTest":null,"latency":null},{"benchmarkId":"sweVerified","benchmarkName":"Software Engineering Benchmark Verified","category":"coding","score":69.2,"unit":"percent","family":"swe-bench-verified","index":52.26691191210313,"aboveFit":false,"run":null,"source":"Carlos E. Jimenez, John Yang, Alexander Wettig, Shunyu Yao, Kexin Pei, Ofir Press, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2310.06770","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"superGpqa","benchmarkName":"SuperGPQA: Scaling LLM Evaluation Across 285 Graduate Disciplines","category":"knowledge","score":63.4,"unit":"percent","family":"superGpqa","index":49.20605773686926,"aboveFit":false,"run":null,"source":"Xiaoxuan Du, Yao Yao, Kexin Ma, Bowen Wang, Tianyu Zheng, Kaiyan Zhu, Yiming Zhang, Yutao Zhu, Jiawei Zhou, Jingren Zhou","sourceUrl":"https://arxiv.org/abs/2502.14739","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"browseComp","benchmarkName":"BrowseComp","category":"agentic","score":61,"unit":"percent","family":"browseComp","index":48.10020221655606,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"longBenchV2","benchmarkName":"LongBench v2","category":"reasoning","score":59,"unit":"percent","family":"longBenchV2","index":50.30238928386892,"aboveFit":false,"run":null,"source":"LongBench v2 authors","sourceUrl":"https://arxiv.org/abs/2412.15204","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"osWorldVerified","benchmarkName":"OSWorld-Verified","category":"agentic","score":54.5,"unit":"percent","family":"osWorldVerified","index":44.47652046164041,"aboveFit":false,"run":null,"source":"Tianbao Xie, Danyang Zhang, Jixuan Chen, Xiaochuan Li, Siheng Zhao, Ruisheng Cao, Toh Jing Hua, Zhoujun Cheng, Dongchan Shin, Fangyu Lei, Yitao Liu, Yiheng Xu, Shuyan Zhou, Silvio Savarese, Caiming Xiong, Victor Zhong, Tao Yu","sourceUrl":"https://os-world.github.io/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"sweRebench","benchmarkName":"SWE-Rebench","category":"coding","score":53.7,"unit":"percent","family":"sweRebench","index":null,"aboveFit":false,"run":null,"source":"Nebius","sourceUrl":"https://swe-rebench.com","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"terminalBench2","benchmarkName":"Terminal-Bench 2.0","category":"agentic","score":40.5,"unit":"percent","family":"terminalBench2","index":42.70713104841204,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gertLabs","benchmarkName":"Gert Labs Composite Game Benchmark","category":"agentic","score":28.96,"unit":"percent","family":"gertLabs","index":38.996155027886005,"aboveFit":false,"run":null,"source":"Gert Labs","sourceUrl":"https://gertlabs.com/rankings","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":21,"unit":"percent","family":"hle","index":45.45579195893916,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"omniscienceAccuracy","benchmarkName":"Artificial Analysis Omniscience Accuracy","category":"knowledge","score":20.1,"unit":"percent","family":"omniscienceAccuracy","index":36.02109574622126,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":0.9,"unit":"percent","family":"critpt","index":38.5914109978019,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":0.14,"outputPerMillion":1,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":43,"timeToFirstToken":0.489,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":{"elo":1394.7399080714358,"eloUpper":1398.9529335823108,"eloLower":1390.5268825605608,"votes":30206,"rank":166,"rankUpper":null,"rankLower":null,"arenaName":"Qwen3.5-35B-A3B","organization":"alibaba","updatedAt":"2026-11-09"},"quality":{"coveragePercent":100,"benchmarkCount":25,"categoryCount":8,"confidence":"high","state":"available"},"hosts":[{"host":"deepinfra","name":"Deep Infra","kind":"host","url":"https://deepinfra.com/models","inputPerMillion":0.14,"outputPerMillion":1,"cacheReadPerMillion":0.05,"contextTokens":262144,"maxOutputTokens":81920,"deprecated":false},{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":0.14,"outputPerMillion":1,"cacheReadPerMillion":0.05,"contextTokens":262144,"maxOutputTokens":81920,"deprecated":false},{"host":"siliconflow","name":"SiliconFlow","kind":"host","url":"https://cloud.siliconflow.com/models","inputPerMillion":0.24,"outputPerMillion":1.8,"cacheReadPerMillion":0.15,"contextTokens":262144,"maxOutputTokens":262144,"deprecated":false},{"host":"novita-ai","name":"Novita AI","kind":"host","url":"https://novita.ai/docs/guides/introduction","inputPerMillion":0.25,"outputPerMillion":2,"cacheReadPerMillion":null,"contextTokens":262144,"maxOutputTokens":65536,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"deepseek-v3-2-thinking","providerId":"deepseek","slug":"deepseek-v3-2-thinking","name":"DeepSeek V3.2 (Thinking)","provider":"DeepSeek","description":"DeepSeek V3.2 (Thinking) is a reasoning model from DeepSeek in the DeepSeek V3.2 family. 10 benchmarks count toward its index, in 4 categories.","logoUrl":null,"releaseDate":"2025-12-01","contextWindowTokens":128000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":true,"overallScore":49.35473604519898,"overallScoreRange":{"low":40.74133306870671,"high":57.96813902169126},"badges":[],"capabilityScores":{"agentic":47.682699520533056,"coding":49.23173808302376,"reasoning":null,"multimodal":null,"knowledge":51.16340108762983,"multilingual":null,"instruction":null,"math":50.997446402843956},"capabilityCounts":{"math":2,"knowledge":2,"coding":4,"agentic":2},"benchmarks":[{"benchmarkId":"vals-ai:medqa:overall","benchmarkName":"MedQA","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":93.917,"unit":"percent","family":"vals-ai:medqa:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medqa","measuredAt":"2026-04-16","stderr":0.218,"costPerTest":0.000772,"latency":53.69},{"benchmarkId":"aime2025","benchmarkName":"American Invitational Mathematics Examination 2025","category":"math","score":93.1,"unit":"percent","family":"aime2025","index":49.52568526064046,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:mgsm:overall","benchmarkName":"MGSM","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":86.036,"unit":"percent","family":"vals-ai:mgsm:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mgsm","measuredAt":"2026-01-09","stderr":2.037,"costPerTest":0.0004,"latency":28.224},{"benchmarkId":"vals-ai:mmlu_pro:overall","benchmarkName":"MMLU Pro","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":84.917,"unit":"percent","family":"mmlu-pro","index":51.465984123326635,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","measuredAt":"2026-09-01","stderr":0.353,"costPerTest":0.001102,"latency":90.148},{"benchmarkId":"vals-ai:aime:overall","benchmarkName":"AIME","category":"math","domain":null,"eligible":true,"ranked":true,"score":84.583,"unit":"percent","family":"vals-ai:aime:overall","index":52.46920754504745,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/aime","measuredAt":"2026-04-16","stderr":1.125,"costPerTest":0.00442,"latency":373.072},{"benchmarkId":"vals-ai:lcb:overall","benchmarkName":"LiveCodeBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":80.695,"unit":"percent","family":"livecodebench","index":56.299734977874806,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","measuredAt":"2026-09-01","stderr":1.099,"costPerTest":0.003266,"latency":254.276},{"benchmarkId":"vals-ai:gpqa:overall","benchmarkName":"GPQA Diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":80.303,"unit":"percent","family":"gpqa-diamond","index":50.86081805193302,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","measuredAt":"2026-09-01","stderr":1.997,"costPerTest":0.003465,"latency":260.248},{"benchmarkId":"vals-ai:legal_bench:overall","benchmarkName":"LegalBench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":76.076,"unit":"percent","family":"vals-ai:legal_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","measuredAt":"2026-10-13","stderr":0.493,"costPerTest":0.000454,"latency":31.468},{"benchmarkId":"vals-ai:tax_eval_v2:overall","benchmarkName":"TaxEval v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":68.152,"unit":"percent","family":"vals-ai:tax_eval_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","measuredAt":"2026-09-01","stderr":0.919,"costPerTest":0.002376,"latency":98.919},{"benchmarkId":"vals-ai:swebench:overall","benchmarkName":"SWE-bench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":67.6,"unit":"percent","family":"swe-bench-verified","index":51.08681978764922,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/swebench","measuredAt":"2026-09-01","stderr":2.095,"costPerTest":0.672838,"latency":533.036},{"benchmarkId":"vals-ai:case_law_v2:overall","benchmarkName":"CaseLaw v2","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":55.412,"unit":"percent","family":"vals-ai:case_law_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/case_law_v2","measuredAt":"2026-05-04","stderr":0.467,"costPerTest":0.017751,"latency":66.077},{"benchmarkId":"vals-ai:corp_fin_v2:overall","benchmarkName":"CorpFin v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":50.971,"unit":"percent","family":"vals-ai:corp_fin_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","measuredAt":"2026-08-12","stderr":0.979,"costPerTest":0.038215,"latency":99.485},{"benchmarkId":"vals-ai:terminal-bench:overall","benchmarkName":"Terminal-Bench 1.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":40,"unit":"percent","family":"vals-ai:terminal-bench:overall","index":47.165224329822124,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench","measuredAt":"2026-01-12","stderr":5.477,"costPerTest":0.047832,"latency":341.987},{"benchmarkId":"vals-ai:terminal-bench-2:overall","benchmarkName":"Terminal-Bench 2.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":35.955,"unit":"percent","family":"vals-ai:terminal-bench-2:overall","index":48.20017471124398,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2","measuredAt":"2026-06-04","stderr":5.115,"costPerTest":0.166362,"latency":1024.966},{"benchmarkId":"vals-ai:ioi-v1:overall","benchmarkName":"IOI v1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":10.667,"unit":"percent","family":"vals-ai:ioi-v1:overall","index":45.44782832291061,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/ioi-v1","measuredAt":"2026-08-09","stderr":3.089,"costPerTest":0.058713,"latency":1558.619},{"benchmarkId":"vals-ai:vibe-code:overall","benchmarkName":"Vibe Code Bench v1.1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":5.108,"unit":"percent","family":"vals-ai:vibe-code:overall","index":44.09256924366039,"aboveFit":false,"run":"OpenHands","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","measuredAt":"2026-11-09","stderr":2.127,"costPerTest":2.469325,"latency":3365.726},{"benchmarkId":"vals-ai:finance_agent:overall","benchmarkName":"Finance Agent v1.1","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":2.345,"unit":"percent","family":"vals-ai:finance_agent:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/finance_agent","measuredAt":"2026-06-04","stderr":0.771,"costPerTest":0.208831,"latency":1721.463}],"price":{"inputPerMillion":0.55,"outputPerMillion":2.19,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":{"elo":1420.0645802937074,"eloUpper":1423.6242567704103,"eloLower":1416.5049038170046,"votes":41782,"rank":116,"rankUpper":null,"rankLower":null,"arenaName":"DeepSeek-V3.2-thinking","organization":"deepseek","updatedAt":"2026-11-09"},"quality":{"coveragePercent":65,"benchmarkCount":10,"categoryCount":4,"confidence":"low","state":"available"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"minimaxm21","providerId":"minimax","slug":"minimax-minimax-m2-1","name":"MiniMax M2.1","provider":"MiniMax","description":"MiniMax M2.1 is a model from MiniMax. 9 benchmarks count toward its index, in 4 categories.","logoUrl":null,"releaseDate":"2025-12-23T01:56:37.000Z","contextWindowTokens":204800,"modalities":["text"],"openWeights":true,"details":{"apiId":"minimax/minimax-m2.1","maxOutputTokens":131072,"knowledgeCutoff":null,"retiresAt":"2026-11-05","tools":true,"structuredOutput":false,"cacheReadPerMillion":0.03,"weightsUrl":"https://huggingface.co/MiniMaxAI/MiniMax-M2.1"},"reasoning":null,"overallScore":49.14493242881733,"overallScoreRange":{"low":40.25493242881733,"high":58.03493242881733},"badges":[],"capabilityScores":{"agentic":48.4733880838495,"coding":51.62633655180998,"reasoning":null,"multimodal":null,"knowledge":51.69611143241815,"multilingual":null,"instruction":null,"math":43.86706233431305},"capabilityCounts":{"math":2,"knowledge":2,"coding":3,"agentic":2},"benchmarks":[{"benchmarkId":"vals-ai:medqa:overall","benchmarkName":"MedQA","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":91.158,"unit":"percent","family":"vals-ai:medqa:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medqa","measuredAt":"2026-04-16","stderr":0.259,"costPerTest":0.002383,"latency":35.938},{"benchmarkId":"vals-ai:math500:overall","benchmarkName":"MATH 500","category":"math","domain":null,"eligible":true,"ranked":true,"score":89,"unit":"percent","family":"math-500","index":38.1782641626366,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/math500","measuredAt":"2026-01-09","stderr":1.399,"costPerTest":0.004642,"latency":59.692},{"benchmarkId":"vals-ai:mgsm:overall","benchmarkName":"MGSM","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":87.854,"unit":"percent","family":"vals-ai:mgsm:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mgsm","measuredAt":"2026-01-09","stderr":1.827,"costPerTest":0.001335,"latency":13.862},{"benchmarkId":"vals-ai:mmlu_pro:overall","benchmarkName":"MMLU Pro","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":87.049,"unit":"percent","family":"mmlu-pro","index":54.152048929486526,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","measuredAt":"2026-09-01","stderr":0.333,"costPerTest":0.003225,"latency":40.957},{"benchmarkId":"vals-ai:lcb:overall","benchmarkName":"LiveCodeBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":81.756,"unit":"percent","family":"livecodebench","index":57.18009042184672,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","measuredAt":"2026-09-01","stderr":1.099,"costPerTest":0.018485,"latency":246.072},{"benchmarkId":"vals-ai:medscribe:overall","benchmarkName":"MedScribe","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":80.777,"unit":"percent","family":"vals-ai:medscribe:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medscribe","measuredAt":"2026-10-29","stderr":1.831,"costPerTest":0.005087,"latency":53.161},{"benchmarkId":"vals-ai:gpqa:overall","benchmarkName":"GPQA Diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":78.536,"unit":"percent","family":"gpqa-diamond","index":49.240173935349766,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","measuredAt":"2026-09-01","stderr":2.128,"costPerTest":0.011422,"latency":135.412},{"benchmarkId":"vals-ai:aime:overall","benchmarkName":"AIME","category":"math","domain":null,"eligible":true,"ranked":true,"score":77.917,"unit":"percent","family":"vals-ai:aime:overall","index":49.555860505989486,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/aime","measuredAt":"2026-04-16","stderr":1.289,"costPerTest":0.024152,"latency":237.635},{"benchmarkId":"vals-ai:legal_bench:overall","benchmarkName":"LegalBench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":75.448,"unit":"percent","family":"vals-ai:legal_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","measuredAt":"2026-10-13","stderr":0.539,"costPerTest":0.000937,"latency":11.491},{"benchmarkId":"vals-ai:swebench:overall","benchmarkName":"SWE-bench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":74.8,"unit":"percent","family":"swe-bench-verified","index":56.3972343476918,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/swebench","measuredAt":"2026-09-01","stderr":1.944,"costPerTest":0.439041,"latency":386.256},{"benchmarkId":"vals-ai:tax_eval_v2:overall","benchmarkName":"TaxEval v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":66.353,"unit":"percent","family":"vals-ai:tax_eval_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","measuredAt":"2026-09-01","stderr":0.925,"costPerTest":0.009137,"latency":113.258},{"benchmarkId":"vals-ai:corp_fin_v2:overall","benchmarkName":"CorpFin v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":58.896,"unit":"percent","family":"vals-ai:corp_fin_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","measuredAt":"2026-08-12","stderr":0.968,"costPerTest":0.022461,"latency":21.969},{"benchmarkId":"vals-ai:case_law_v2:overall","benchmarkName":"CaseLaw v2","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":55.842,"unit":"percent","family":"vals-ai:case_law_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/case_law_v2","measuredAt":"2026-05-04","stderr":0.467,"costPerTest":0.008008,"latency":37.357},{"benchmarkId":"vals-ai:terminal-bench:overall","benchmarkName":"Terminal-Bench 1.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":41.25,"unit":"percent","family":"vals-ai:terminal-bench:overall","index":48.06982840515033,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench","measuredAt":"2026-01-12","stderr":5.504,"costPerTest":0.062918,"latency":442.254},{"benchmarkId":"vals-ai:terminal-bench-2:overall","benchmarkName":"Terminal-Bench 2.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":37.079,"unit":"percent","family":"vals-ai:terminal-bench-2:overall","index":48.87694776254867,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2","measuredAt":"2026-06-04","stderr":5.149,"costPerTest":0.070371,"latency":782.07},{"benchmarkId":"vals-ai:medcode:overall","benchmarkName":"MedCode","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":34.083,"unit":"percent","family":"vals-ai:medcode:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medcode","measuredAt":"2026-10-29","stderr":1.943,"costPerTest":null,"latency":19.548},{"benchmarkId":"vals-ai:finance_agent:overall","benchmarkName":"Finance Agent v1.1","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":33.35,"unit":"percent","family":"vals-ai:finance_agent:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/finance_agent","measuredAt":"2026-06-04","stderr":2.58,"costPerTest":0.114177,"latency":1301.289},{"benchmarkId":"vals-ai:ioi-v1:overall","benchmarkName":"IOI v1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":2.334,"unit":"percent","family":"vals-ai:ioi-v1:overall","index":41.301684885891405,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/ioi-v1","measuredAt":"2026-08-09","stderr":1.3,"costPerTest":0.319345,"latency":5210.402}],"price":{"inputPerMillion":0.3,"outputPerMillion":1.2,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":34,"timeToFirstToken":1.654,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":65,"benchmarkCount":9,"categoryCount":4,"confidence":"low","state":"available"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"k-exaone-2-0","providerId":"lg-ai-research","slug":"k-exaone-2-0","name":"K-EXAONE 2.0","provider":"LG AI Research","description":"K-EXAONE 2.0 is a reasoning model from LG AI Research. 17 benchmarks count toward its index, in 6 categories.","logoUrl":null,"releaseDate":"2026-07-31","contextWindowTokens":262000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":true,"overallScore":48.82157240861064,"overallScoreRange":{"low":43.39977135177314,"high":54.243373465448144},"badges":[],"capabilityScores":{"agentic":56.43145021202851,"coding":49.56284564474781,"reasoning":42.223592997443355,"multimodal":null,"knowledge":43.713012942296565,"multilingual":null,"instruction":53.068617838667535,"math":50.95328664115504},"capabilityCounts":{"instruction":2,"math":3,"knowledge":5,"agentic":3,"coding":2,"reasoning":2},"benchmarks":[{"benchmarkId":"ifeval","benchmarkName":"Instruction-Following Eval","category":"instruction","score":92.4,"unit":"percent","family":"ifeval","index":49.268135648648254,"aboveFit":false,"run":null,"source":"Jeffrey Zhou, Tianjian Lu, Swaroop Mishra, Siddhartha Brahma, Sujoy Basu, Yi Luan, Denny Zhou, Le Hou","sourceUrl":"https://arxiv.org/abs/2311.07911","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aime2026","benchmarkName":"AIME 2026","category":"math","score":92.3,"unit":"percent","family":"aime2026","index":51.741486486064844,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmmlu","benchmarkName":"MMMLU","category":"knowledge","score":86.6,"unit":"percent","family":"mmmlu","index":44.101568230537275,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmluPro","benchmarkName":"Massive Multitask Language Understanding Professional","category":"knowledge","score":83.5,"unit":"percent","family":"mmlu-pro","index":49.6807337338667,"aboveFit":false,"run":null,"source":"Yubo Wang, Xueguang Ma, Ge Zhang, Yuansheng Ni, Abhranil Chandra, Shiguang Guo, Weiming Ren, Aaran Arulraj, Xuan He, Ziyan Jiang, Tianle Li, Max Ku, Kai Wang, Alex Zhuang, Rongqi Fan, Xiang Yue, Wenhu Chen","sourceUrl":"https://arxiv.org/abs/2406.01574","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":82.9,"unit":"percent","family":"gpqa-diamond","index":53.242715488699716,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gpqaDiamond","benchmarkName":"GPQA Diamond","category":"knowledge","score":82.2,"unit":"percent","family":"gpqa-diamond","index":52.600694616255865,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"imoAnswerBench","benchmarkName":"IMOAnswerBench","category":"math","score":78.6,"unit":"percent","family":"imoAnswerBench","index":52.85187950099027,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hmmtFeb2026","benchmarkName":"Harvard-MIT Mathematics Tournament February 2026","category":"math","score":78.4,"unit":"percent","family":"hmmtFeb2026","index":48.26649393641,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"clawEval","benchmarkName":"Claw-Eval","category":"agentic","score":77.7,"unit":"percent","family":"clawEval","index":77.49214741756708,"aboveFit":true,"run":null,"source":"Bowen Ye, Rang Li, Qibin Yang, Yuanxin Liu, Linli Yao, Hanglong Lv, Zhihui Xie, Chenxin An, Lei Li, Lingpeng Kong, Qi Liu, Zhifang Sui, Tong Yang","sourceUrl":"https://arxiv.org/abs/2604.06132","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"ifBench","benchmarkName":"Instruction Following Benchmark","category":"instruction","score":72.6,"unit":"percent","family":"ifbench-family","index":56.869100028686816,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"polyMath","benchmarkName":"PolyMath","category":"multilingual","score":71.3,"unit":"percent","family":"polyMath","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"sweVerified","benchmarkName":"Software Engineering Benchmark Verified","category":"coding","score":68.2,"unit":"percent","family":"swe-bench-verified","index":51.529354334319436,"aboveFit":false,"run":null,"source":"Carlos E. Jimenez, John Yang, Alexander Wettig, Shunyu Yao, Kexin Pei, Ofir Press, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2310.06770","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":56.2,"unit":"percent","family":"lcr","index":45.85577499708481,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"terminalBench21","benchmarkName":"Terminal-Bench 2.1","category":"agentic","score":43.8,"unit":"percent","family":"terminal-bench-2-1","index":46.75778617127144,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaSciCode","benchmarkName":"Artificial Analysis SciCode","category":"coding","score":42,"unit":"percent","family":"scicode-family","index":50.59482659193006,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/scicode","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"sciCode","benchmarkName":"Scientific Code Benchmark","category":"coding","score":37.4,"unit":"percent","family":"scicode-family","index":44.59784731842229,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":18.6,"unit":"percent","family":"hle","index":43.04788937086765,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hle","benchmarkName":"Humanity's Last Exam","category":"knowledge","score":18.3,"unit":"percent","family":"hle","index":42.74690154735872,"aboveFit":false,"run":null,"source":"Center for AI Safety, Scale AI, and thousands of expert contributors","sourceUrl":"https://lastexam.ai/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gdpvalAaNormalized","benchmarkName":"GDPval-AA normalized","category":"agentic","score":15.4,"unit":"percent","family":"gdpvalAaNormalized","index":45.044417047246995,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"omniscienceAccuracy","benchmarkName":"Artificial Analysis Omniscience Accuracy","category":"knowledge","score":14.5,"unit":"percent","family":"omniscienceAccuracy","index":28.96366223548786,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":0.9,"unit":"percent","family":"critpt","index":38.5914109978019,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":85,"benchmarkCount":17,"categoryCount":6,"confidence":"medium","state":"available"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"spark-x2-6-4b","providerId":"sparkllm","slug":"spark-x2-6-4b","name":"Spark-X2.6-4B","provider":"SparkLLM","description":"Spark-X2.6-4B is a reasoning model from SparkLLM in the Spark X2.6 family. 19 benchmarks count toward its index, in 6 categories.","logoUrl":null,"releaseDate":"2026-10-19","contextWindowTokens":1000000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":true,"overallScore":48.6794404062999,"overallScoreRange":{"low":43.46978798961292,"high":53.88909282298689},"badges":[],"capabilityScores":{"agentic":48.81391258050388,"coding":48.81466056576065,"reasoning":48.51043498664188,"multimodal":null,"knowledge":43.48343839690026,"multilingual":null,"instruction":55.77217415162387,"math":50.49595839700057},"capabilityCounts":{"instruction":2,"math":3,"agentic":7,"knowledge":2,"reasoning":2,"coding":3},"benchmarks":[{"benchmarkId":"ifeval","benchmarkName":"Instruction-Following Eval","category":"instruction","score":93.6,"unit":"percent","family":"ifeval","index":51.684814307106514,"aboveFit":false,"run":null,"source":"Jeffrey Zhou, Tianjian Lu, Swaroop Mishra, Siddhartha Brahma, Sujoy Basu, Yi Luan, Denny Zhou, Le Hou","sourceUrl":"https://arxiv.org/abs/2311.07911","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aime2026","benchmarkName":"AIME 2026","category":"math","score":91.2,"unit":"percent","family":"aime2026","index":50.949551918628686,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"tau2Bench","benchmarkName":"τ²-Bench Tool-Agent-User Evaluation","category":"agentic","score":86.9,"unit":"percent","family":"tau2-bench","index":60.45394357132643,"aboveFit":false,"run":null,"source":"Victor Barres, Honghua Dong, Soham Ray, Xujie Si, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2506.07982","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hmmtFeb2026","benchmarkName":"Harvard-MIT Mathematics Tournament February 2026","category":"math","score":81.3,"unit":"percent","family":"hmmtFeb2026","index":50.295970816078274,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gpqaDiamond","benchmarkName":"GPQA Diamond","category":"knowledge","score":77.1,"unit":"percent","family":"gpqa-diamond","index":47.92311397416495,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"ifBench","benchmarkName":"Instruction Following Benchmark","category":"instruction","score":75.8,"unit":"percent","family":"ifbench-family","index":59.85953399614123,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"imoAnswerBench","benchmarkName":"IMOAnswerBench","category":"math","score":75.2,"unit":"percent","family":"imoAnswerBench","index":50.24235245629476,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"bfclV4","benchmarkName":"Berkeley Function Calling Leaderboard v4","category":"agentic","score":71.8,"unit":"percent","family":"bfclV4","index":51.829794961482136,"aboveFit":false,"run":null,"source":"Shishir G. Patil et al.","sourceUrl":"https://gorilla.cs.berkeley.edu/leaderboard.html","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"clawEval","benchmarkName":"Claw-Eval","category":"agentic","score":69.9,"unit":"percent","family":"clawEval","index":66.20463259475865,"aboveFit":true,"run":null,"source":"Bowen Ye, Rang Li, Qibin Yang, Yuanxin Liu, Linli Yao, Hanglong Lv, Zhihui Xie, Chenxin An, Lei Li, Lingpeng Kong, Qi Liu, Zhifang Sui, Tong Yang","sourceUrl":"https://arxiv.org/abs/2604.06132","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gpqa","benchmarkName":"Graduate-Level Google-Proof Q&A","category":"knowledge","score":69.9,"unit":"percent","family":"gpqa-diamond","index":41.31947071474251,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":67.3,"unit":"percent","family":"lcr","index":53.47126883108563,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"sweVerified","benchmarkName":"Software Engineering Benchmark Verified","category":"coding","score":62.8,"unit":"percent","family":"swe-bench-verified","index":47.5465434142875,"aboveFit":false,"run":null,"source":"Carlos E. Jimenez, John Yang, Alexander Wettig, Shunyu Yao, Kexin Pei, Ofir Press, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2310.06770","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mcpAtlas","benchmarkName":"MCP Atlas","category":"agentic","score":62.4,"unit":"percent","family":"mcpAtlas","index":53.08216850648818,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"swePro","benchmarkName":"SWE-bench Pro","category":"coding","score":54.3,"unit":"percent","family":"swePro","index":55.081805652421,"aboveFit":false,"run":null,"source":"Xiang Deng, Jeff Da, Edwin Pan, Yannis Yiming He, Charles Ide, Kanak Garg, Niklas Lauffer, Andrew Park, Nitin Pasari, Chetan Rane, Karmini Sampath, Maya Krishnan, Srivatsa Kundurthy, Sean Hendryx, Zifan Wang, Chen Bo Calvin Zhang, Noah Jacobson, Bing Liu, Brad Kenstler","sourceUrl":"https://arxiv.org/abs/2509.16941","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"longBenchV2","benchmarkName":"LongBench v2","category":"reasoning","score":53.3,"unit":"percent","family":"longBenchV2","index":43.549601142198135,"aboveFit":false,"run":null,"source":"LongBench v2 authors","sourceUrl":"https://arxiv.org/abs/2412.15204","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"browseComp","benchmarkName":"BrowseComp","category":"agentic","score":53.1,"unit":"percent","family":"browseComp","index":41.85904574916142,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hleWithTools","benchmarkName":"Humanity's Last Exam with tools","category":"agentic","score":36.9,"unit":"percent","family":"hleWithTools","index":48.879123661592594,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"sciCode","benchmarkName":"Scientific Code Benchmark","category":"coding","score":36.8,"unit":"percent","family":"scicode-family","index":43.81563263057345,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"tau3Bench","benchmarkName":"τ³-Bench Tool-Agent-User Evaluation","category":"agentic","score":34.7,"unit":"percent","family":"tau3Bench","index":19.388679018717774,"aboveFit":false,"run":null,"source":"Sierra Research","sourceUrl":"https://github.com/sierra-research/tau2-bench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hleNoTools","benchmarkName":"Humanity's Last Exam without tools","category":"knowledge","score":17.9,"unit":"percent","family":"hle","index":42.345584449346795,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":85,"benchmarkCount":19,"categoryCount":6,"confidence":"medium","state":"available"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gpt-5-mini","providerId":"openai","slug":"gpt-5-mini","name":"GPT-5 mini","provider":"OpenAI","description":"GPT-5 mini is a reasoning model from OpenAI in the GPT-5 family. 23 benchmarks count toward its index, in 7 categories.","logoUrl":null,"releaseDate":"2025-08-07","contextWindowTokens":400000,"modalities":["text","image","file"],"openWeights":false,"details":{"apiId":"openai/gpt-5-mini","maxOutputTokens":128000,"knowledgeCutoff":"2024-05-31","retiresAt":"2026-12-11","tools":true,"structuredOutput":true,"cacheReadPerMillion":0.024999999999999998,"weightsUrl":null},"reasoning":true,"overallScore":48.40842666427813,"overallScoreRange":{"low":44.53674858897523,"high":52.280104739581034},"badges":[],"capabilityScores":{"agentic":41.35809418088489,"coding":49.44192356242606,"reasoning":46.98650197592593,"multimodal":57.85246251744948,"knowledge":46.13944668367782,"multilingual":51.79199079404251,"instruction":null,"math":52.237498835522565},"capabilityCounts":{"math":8,"coding":4,"knowledge":3,"multimodal":1,"multilingual":1,"reasoning":4,"agentic":2},"benchmarks":[{"benchmarkId":"epoch-ai:epoch-math-level-5:overall","benchmarkName":"MATH level 5","category":"math","domain":null,"eligible":true,"ranked":true,"score":97.8474320241692,"unit":"percent","family":"epoch-ai:epoch-math-level-5:overall","index":46.52782743755179,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0.3369422414145465,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:medqa:overall","benchmarkName":"MedQA","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":96.058,"unit":"percent","family":"vals-ai:medqa:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medqa","measuredAt":"2026-04-16","stderr":0.178,"costPerTest":0.002953,"latency":16.253},{"benchmarkId":"vals-ai:math500:overall","benchmarkName":"MATH 500","category":"math","domain":null,"eligible":true,"ranked":true,"score":94.8,"unit":"percent","family":"math-500","index":44.3074617947299,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/math500","measuredAt":"2026-01-09","stderr":0.993,"costPerTest":0.003686,"latency":12.057},{"benchmarkId":"vals-ai:mgsm:overall","benchmarkName":"MGSM","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":92.582,"unit":"percent","family":"vals-ai:mgsm:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mgsm","measuredAt":"2026-01-09","stderr":1.59,"costPerTest":0.003698,"latency":12.573},{"benchmarkId":"vals-ai:aime:overall","benchmarkName":"AIME","category":"math","domain":null,"eligible":true,"ranked":true,"score":91.458,"unit":"percent","family":"vals-ai:aime:overall","index":55.473897148036286,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/aime","measuredAt":"2026-04-16","stderr":0.836,"costPerTest":0.02276,"latency":105.53},{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":86.66666666666667,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":59.341241506765535,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":4.286165199944133,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:lcb:overall","benchmarkName":"LiveCodeBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":86.605,"unit":"percent","family":"livecodebench","index":61.203505641281005,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","measuredAt":"2026-09-01","stderr":0.979,"costPerTest":0.01153,"latency":33.673},{"benchmarkId":"vals-ai:mmlu_pro:overall","benchmarkName":"MMLU Pro","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":82.226,"unit":"percent","family":"mmlu-pro","index":48.075646227746766,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","measuredAt":"2026-09-01","stderr":0.373,"costPerTest":0.005196,"latency":22.196},{"benchmarkId":"vals-ai:legal_bench:overall","benchmarkName":"LegalBench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":81.77,"unit":"percent","family":"vals-ai:legal_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","measuredAt":"2026-10-13","stderr":0.452,"costPerTest":null,"latency":9.432},{"benchmarkId":"vals-ai:medscribe:overall","benchmarkName":"MedScribe","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":80.577,"unit":"percent","family":"vals-ai:medscribe:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medscribe","measuredAt":"2026-10-29","stderr":1.924,"costPerTest":0.033478,"latency":275.717},{"benchmarkId":"vals-ai:gpqa:overall","benchmarkName":"GPQA Diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":80.303,"unit":"percent","family":"gpqa-diamond","index":50.86081805193302,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","measuredAt":"2026-09-01","stderr":1.996,"costPerTest":0.013593,"latency":67.133},{"benchmarkId":"vals-ai:mmmu:overall","benchmarkName":"MMMU Pro","category":"multimodal","domain":null,"eligible":true,"ranked":true,"score":78.914,"unit":"percent","family":"mmmu-pro","index":57.85246251744948,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmmu","measuredAt":"2026-09-01","stderr":0.98,"costPerTest":0.007152,"latency":26.703},{"benchmarkId":"vals-ai:tax_eval_v2:overall","benchmarkName":"TaxEval v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":75.225,"unit":"percent","family":"vals-ai:tax_eval_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","measuredAt":"2026-09-01","stderr":0.847,"costPerTest":0.011528,"latency":37.434},{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":75,"unit":"percent","family":"gpqa-diamond","index":45.99705135683341,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.2767295427113803,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:case_law_v2:overall","benchmarkName":"CaseLaw v2","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":68.489,"unit":"percent","family":"vals-ai:case_law_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/case_law_v2","measuredAt":"2026-05-04","stderr":1.129,"costPerTest":0.011305,"latency":42.506},{"benchmarkId":"vals-ai:mortgage_tax:overall","benchmarkName":"MortgageTax","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":66.892,"unit":"percent","family":"vals-ai:mortgage_tax:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mortgage_tax","measuredAt":"2026-09-01","stderr":0.923,"costPerTest":0.00436,"latency":24.864},{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":66.01624999999999,"unit":"percent","family":"euroeval","index":61.78909680213952,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":0.67525342464885,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-swe-bench-verified:overall","benchmarkName":"SWE-Bench verified","category":"coding","domain":null,"eligible":true,"ranked":true,"score":64.6694214876033,"unit":"percent","family":"swe-bench-verified","index":48.92534939854098,"aboveFit":false,"run":"medium effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.1749601139671304,"costPerTest":null,"latency":null},{"benchmarkId":"arc-prize:v1-public-eval:overall","benchmarkName":"ARC-AGI-1 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":61.519999999999996,"unit":"percent","family":"arc-prize:v1-public-eval:overall","index":null,"aboveFit":false,"run":"high effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.0936,"latency":null},{"benchmarkId":"vals-ai:swebench:overall","benchmarkName":"SWE-bench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":60.8,"unit":"percent","family":"swe-bench-verified","index":46.07142825872012,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/swebench","measuredAt":"2026-09-01","stderr":2.185,"costPerTest":0.050393,"latency":187.129},{"benchmarkId":"vals-ai:corp_fin_v2:overall","benchmarkName":"CorpFin v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":60.179,"unit":"percent","family":"vals-ai:corp_fin_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","measuredAt":"2026-08-12","stderr":0.965,"costPerTest":0.020976,"latency":73.788},{"benchmarkId":"euroeval:euroeval-italian:overall","benchmarkName":"EuroEval Italian","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":60.14,"unit":"percent","family":"euroeval","index":53.81637763874404,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/italian","measuredAt":null,"stderr":0.3328827684035327,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-french:overall","benchmarkName":"EuroEval French","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":60.11249999999999,"unit":"percent","family":"euroeval","index":53.77906646631375,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/french","measuredAt":null,"stderr":0.382085150849912,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-dutch:overall","benchmarkName":"EuroEval Dutch","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":60.08444444444446,"unit":"percent","family":"euroeval","index":53.74100153282431,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/dutch","measuredAt":null,"stderr":0.42753095332312807,"costPerTest":null,"latency":null},{"benchmarkId":"swe-bench:swe-bench-verified:overall","benchmarkName":"SWE-bench Verified","category":"coding","domain":null,"eligible":true,"ranked":true,"score":59.8,"unit":"percent","family":"swe-bench-verified","index":45.33387068093642,"aboveFit":false,"run":"medium effort · mini-SWE-agent","source":"SWE-bench team","sourceUrl":"https://www.swebench.com/","measuredAt":"2026-02-26","stderr":null,"costPerTest":0.035477067300000005,"latency":null},{"benchmarkId":"swebench-experiments:swe-bench-verified:overall","benchmarkName":"SWE-bench Verified","category":"coding","domain":null,"eligible":true,"ranked":true,"score":59.8,"unit":"percent","family":"swe-bench-verified","index":45.33387068093642,"aboveFit":false,"run":"medium effort · mini-SWE-agent","source":"SWE-bench team","sourceUrl":"https://github.com/SWE-bench/experiments/tree/main/evaluation/verified","measuredAt":"2026-09-01","stderr":null,"costPerTest":0.035477067300000005,"latency":null},{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":57.21142857142858,"unit":"percent","family":"euroeval","index":49.8429800552607,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.34231504763409787,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-portuguese:overall","benchmarkName":"EuroEval Portuguese","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":55.86750000000001,"unit":"percent","family":"euroeval","index":48.01957821298659,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/portuguese","measuredAt":null,"stderr":0.4221170601859157,"costPerTest":null,"latency":null},{"benchmarkId":"arc-prize:v1-semi-private:overall","benchmarkName":"ARC-AGI-1 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":54.33,"unit":"percent","family":"arc-prize:v1-semi-private:overall","index":51.964821334789754,"aboveFit":false,"run":"high effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.116,"latency":null},{"benchmarkId":"euroeval:euroeval-spanish:overall","benchmarkName":"EuroEval Spanish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":54.235,"unit":"percent","family":"euroeval","index":45.80465134053507,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/spanish","measuredAt":null,"stderr":0.5601645963464666,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":52.51875,"unit":"percent","family":"euroeval","index":43.47609498840956,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.6440957809208192,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:finance_agent:overall","benchmarkName":"Finance Agent v1.1","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":51.928,"unit":"percent","family":"vals-ai:finance_agent:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/finance_agent","measuredAt":"2026-06-04","stderr":2.897,"costPerTest":0.142449,"latency":643.508},{"benchmarkId":"epoch-ai:epoch-frontiermath-tiers-1-3-v2-private:overall","benchmarkName":"FrontierMath-Tiers-1-3-v2-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":46.666666666666664,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tiers-1-3-v2-private:overall","index":55.457776165929666,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.9603535719125715,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:medcode:overall","benchmarkName":"MedCode","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":43.045,"unit":"percent","family":"vals-ai:medcode:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medcode","measuredAt":"2026-10-29","stderr":2.045,"costPerTest":0.00556,"latency":28.173},{"benchmarkId":"swe-bench:swe-bench-multilingual:overall","benchmarkName":"SWE-bench Multilingual","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":39.7,"unit":"percent","family":"swe-bench-multilingual","index":null,"aboveFit":false,"run":"mini-SWE-agent","source":"SWE-bench team","sourceUrl":"https://www.swebench.com/","measuredAt":"2026-02-20","stderr":null,"costPerTest":0.051632880333333325,"latency":null},{"benchmarkId":"swebench-experiments:swe-bench-multilingual:overall","benchmarkName":"SWE-bench Multilingual","category":"coding","domain":null,"eligible":true,"ranked":true,"score":39.7,"unit":"percent","family":"swe-bench-multilingual","index":null,"aboveFit":false,"run":"mini-SWE-agent","source":"SWE-bench team","sourceUrl":"https://github.com/SWE-bench/experiments/tree/main/evaluation/multilingual","measuredAt":"2026-09-02","stderr":null,"costPerTest":0.051632880333333325,"latency":null},{"benchmarkId":"vals-ai:terminal-bench:overall","benchmarkName":"Terminal-Bench 1.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":30,"unit":"percent","family":"vals-ai:terminal-bench:overall","index":39.92839172719647,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench","measuredAt":"2026-01-12","stderr":5.124,"costPerTest":0.366108,"latency":1485.244},{"benchmarkId":"epoch-ai:epoch-frontiermath-2025-02-28-public:overall","benchmarkName":"FrontierMath-2025-02-28-Public","category":"math","domain":null,"eligible":false,"ranked":true,"score":30,"unit":"percent","family":"epoch-ai:epoch-frontiermath-2025-02-28-public:overall","index":null,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":15.275252316519463,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-2025-02-28-private:overall","benchmarkName":"FrontierMath-2025-02-28-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":27.241379310344826,"unit":"percent","family":"epoch-ai:epoch-frontiermath-2025-02-28-private:overall","index":53.962253523814,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.6188332965202883,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:terminal-bench-2:overall","benchmarkName":"Terminal-Bench 2.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":26.966,"unit":"percent","family":"vals-ai:terminal-bench-2:overall","index":42.78779663457331,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2","measuredAt":"2026-06-04","stderr":4.731,"costPerTest":0.116731,"latency":1028.621},{"benchmarkId":"epoch-ai:epoch-chess-puzzles:overall","benchmarkName":"Chess Puzzles","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":24,"unit":"percent","family":"epoch-ai:epoch-chess-puzzles:overall","index":52.20104040874124,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":4.2923469599092785,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-simpleqa-verified:overall","benchmarkName":"SimpleQA Verified","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":21.6,"unit":"percent","family":"epoch-ai:epoch-simpleqa-verified:overall","index":41.91375911890346,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.3019735539307762,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:vibe-code:overall","benchmarkName":"Vibe Code Bench v1.1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":14.171,"unit":"percent","family":"vals-ai:vibe-code:overall","index":47.362642034069594,"aboveFit":false,"run":"OpenHands","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","measuredAt":"2026-11-09","stderr":3.539,"costPerTest":0.248587,"latency":698.246},{"benchmarkId":"epoch-ai:epoch-frontiermath-tier-4-v2-private:overall","benchmarkName":"FrontierMath-Tier-4-v2-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":12.195121951219512,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tier-4-v2-private:overall","index":53.27915862886624,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":5.173952057462543,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:ioi-v1:overall","benchmarkName":"IOI v1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":6.75,"unit":"percent","family":"vals-ai:ioi-v1:overall","index":43.49889710452537,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/ioi-v1","measuredAt":"2026-08-09","stderr":3.316,"costPerTest":0.039168,"latency":390.378},{"benchmarkId":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-private:overall","benchmarkName":"FrontierMath-Tier-4-2025-07-01-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":6.25,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-private:overall","index":49.55037447848712,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":3.530829267912701,"costPerTest":null,"latency":null},{"benchmarkId":"arc-prize:v2-public-eval:overall","benchmarkName":"ARC-AGI-2 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":5.83,"unit":"percent","family":"arc-prize:v2-public-eval:overall","index":null,"aboveFit":false,"run":"high effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.215,"latency":null},{"benchmarkId":"epoch-ai:epoch-mystery-game-puzzles:overall","benchmarkName":"Mystery Game Puzzles","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":5,"unit":"percent","family":"epoch-ai:epoch-mystery-game-puzzles:overall","index":42.55307268559606,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.1904291355759034,"costPerTest":null,"latency":null},{"benchmarkId":"arc-prize:v2-semi-private:overall","benchmarkName":"ARC-AGI-2 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":4.44,"unit":"percent","family":"arc-prize:v2-semi-private:overall","index":41.22707347457668,"aboveFit":false,"run":"high effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.1977,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-public:overall","benchmarkName":"FrontierMath-Tier-4-2025-07-01-Public","category":"math","domain":null,"eligible":false,"ranked":true,"score":0,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-public:overall","index":null,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":0.25,"outputPerMillion":2,"batch":{"inputPerMillion":0.125,"outputPerMillion":1},"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":78,"timeToFirstToken":2.581,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":{"elo":1373.3617965651488,"eloUpper":1377.9253661628813,"eloLower":1368.7982269674167,"votes":26718,"rank":185,"rankUpper":null,"rankLower":null,"arenaName":"GPT-5 mini-high","organization":"openai","updatedAt":"2026-11-09"},"quality":{"coveragePercent":95,"benchmarkCount":23,"categoryCount":7,"confidence":"medium","state":"available"},"hosts":[{"host":"databricks","name":"Databricks","kind":"cloud","url":"https://docs.databricks.com/aws/en/machine-learning/foundation-models/","inputPerMillion":0.25,"outputPerMillion":2,"cacheReadPerMillion":0.025,"contextTokens":400000,"maxOutputTokens":128000,"deprecated":false},{"host":"digitalocean","name":"DigitalOcean","kind":"cloud","url":"https://docs.digitalocean.com/products/gradient-ai-platform/details/models/","inputPerMillion":0.25,"outputPerMillion":2,"cacheReadPerMillion":0.025,"contextTokens":400000,"maxOutputTokens":128000,"deprecated":false},{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":0.25,"outputPerMillion":2,"cacheReadPerMillion":0.025,"contextTokens":400000,"maxOutputTokens":128000,"deprecated":false},{"host":"sap-ai-core","name":"SAP AI Core","kind":"cloud","url":"https://help.sap.com/docs/sap-ai-core","inputPerMillion":0.25,"outputPerMillion":2,"cacheReadPerMillion":0.025,"contextTokens":400000,"maxOutputTokens":128000,"deprecated":false},{"host":"snowflake","name":"Snowflake Cortex","kind":"cloud","url":"https://docs.snowflake.com/en/user-guide/snowflake-cortex/cortex-rest-api","inputPerMillion":0.25,"outputPerMillion":2,"cacheReadPerMillion":0.03,"contextTokens":272000,"maxOutputTokens":8192,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:swe-bench-team","name":"SWE-bench team","url":"https://www.swebench.com","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:arc-prize-foundation","name":"ARC Prize Foundation","url":"https://arcprize.org","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"kimik2thinking","providerId":"moonshot-ai","slug":"moonshot-ai-kimi-k2-thinking","name":"Kimi K2 Thinking","provider":"Moonshot AI","description":"Kimi K2 Thinking is a model from Moonshot AI. 10 benchmarks count toward its index, in 4 categories.","logoUrl":null,"releaseDate":"2025-11-06T14:50:22.000Z","contextWindowTokens":262144,"modalities":["text"],"openWeights":true,"details":{"apiId":"moonshotai/kimi-k2-thinking","maxOutputTokens":98304,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.15,"weightsUrl":"https://huggingface.co/moonshotai/Kimi-K2-Thinking"},"reasoning":null,"overallScore":48.182083525150524,"overallScoreRange":{"low":39.56868054865825,"high":56.7954865016428},"badges":[],"capabilityScores":{"agentic":48.021086046185395,"coding":44.86342718491723,"reasoning":null,"multimodal":null,"knowledge":49.23109325119601,"multilingual":null,"instruction":null,"math":50.521273006766314},"capabilityCounts":{"math":4,"knowledge":2,"coding":2,"agentic":2},"benchmarks":[{"benchmarkId":"vals-ai:medqa:overall","benchmarkName":"MedQA","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":92.592,"unit":"percent","family":"vals-ai:medqa:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medqa","measuredAt":"2026-04-16","stderr":0.239,"costPerTest":0.006153,"latency":225.305},{"benchmarkId":"vals-ai:mgsm:overall","benchmarkName":"MGSM","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":90.146,"unit":"percent","family":"vals-ai:mgsm:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mgsm","measuredAt":"2026-01-09","stderr":1.83,"costPerTest":0.000891,"latency":48.41},{"benchmarkId":"vals-ai:aime:overall","benchmarkName":"AIME","category":"math","domain":null,"eligible":true,"ranked":true,"score":85.417,"unit":"percent","family":"vals-ai:aime:overall","index":52.83370370925003,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/aime","measuredAt":"2026-04-16","stderr":1.327,"costPerTest":0.044508,"latency":924.913},{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":84.21717171717171,"unit":"percent","family":"gpqa-diamond","index":54.45078939586677,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.087984573198575,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":83.05555555555556,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":57.240318563840496,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":4.819264325804854,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:mmlu_pro:overall","benchmarkName":"MMLU Pro","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":81.068,"unit":"percent","family":"mmlu-pro","index":46.61670483678375,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","measuredAt":"2026-09-01","stderr":0.396,"costPerTest":0.002067,"latency":94.771},{"benchmarkId":"vals-ai:legal_bench:overall","benchmarkName":"LegalBench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":80.201,"unit":"percent","family":"vals-ai:legal_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","measuredAt":"2026-10-13","stderr":0.459,"costPerTest":0.002182,"latency":43.991},{"benchmarkId":"vals-ai:gpqa:overall","benchmarkName":"GPQA Diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":78.536,"unit":"percent","family":"gpqa-diamond","index":49.240173935349766,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","measuredAt":"2026-09-01","stderr":2.183,"costPerTest":0.023072,"latency":585.555},{"benchmarkId":"vals-ai:tax_eval_v2:overall","benchmarkName":"TaxEval v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":71.709,"unit":"percent","family":"vals-ai:tax_eval_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","measuredAt":"2026-09-01","stderr":0.878,"costPerTest":0.00913,"latency":61.408},{"benchmarkId":"vals-ai:case_law_v2:overall","benchmarkName":"CaseLaw v2","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":65.702,"unit":"percent","family":"vals-ai:case_law_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/case_law_v2","measuredAt":"2026-05-04","stderr":1.822,"costPerTest":0.015284,"latency":91.045},{"benchmarkId":"swe-bench:swe-bench-verified:overall","benchmarkName":"SWE-bench Verified","category":"coding","domain":null,"eligible":true,"ranked":true,"score":63.4,"unit":"percent","family":"swe-bench-verified","index":47.989077960957715,"aboveFit":false,"run":"mini-SWE-agent","source":"SWE-bench team","sourceUrl":"https://www.swebench.com/","measuredAt":"2026-02-26","stderr":null,"costPerTest":0.4383072354,"latency":null},{"benchmarkId":"swebench-experiments:swe-bench-verified:overall","benchmarkName":"SWE-bench Verified","category":"coding","domain":null,"eligible":true,"ranked":true,"score":63.4,"unit":"percent","family":"swe-bench-verified","index":47.989077960957715,"aboveFit":false,"run":"mini-SWE-agent","source":"SWE-bench team","sourceUrl":"https://github.com/SWE-bench/experiments/tree/main/evaluation/verified","measuredAt":"2026-09-01","stderr":null,"costPerTest":0.4383072354,"latency":null},{"benchmarkId":"vals-ai:lcb:overall","benchmarkName":"LiveCodeBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":63.145,"unit":"percent","family":"livecodebench","index":41.73777640887674,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","measuredAt":"2026-09-01","stderr":1.157,"costPerTest":0.033631,"latency":1150.624},{"benchmarkId":"vals-ai:corp_fin_v2:overall","benchmarkName":"CorpFin v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":60.567,"unit":"percent","family":"vals-ai:corp_fin_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","measuredAt":"2026-08-12","stderr":0.962,"costPerTest":0.038163,"latency":93.297},{"benchmarkId":"vals-ai:swebench:overall","benchmarkName":"SWE-bench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":60.2,"unit":"percent","family":"swe-bench-verified","index":45.6288937120499,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/swebench","measuredAt":"2026-09-01","stderr":2.191,"costPerTest":0.366026,"latency":1632.209},{"benchmarkId":"vals-ai:terminal-bench:overall","benchmarkName":"Terminal-Bench 1.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":40,"unit":"percent","family":"vals-ai:terminal-bench:overall","index":47.165224329822124,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench","measuredAt":"2026-01-12","stderr":5.477,"costPerTest":0.33518,"latency":133.903},{"benchmarkId":"vals-ai:terminal-bench-2:overall","benchmarkName":"Terminal-Bench 2.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":37.079,"unit":"percent","family":"vals-ai:terminal-bench-2:overall","index":48.87694776254867,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2","measuredAt":"2026-06-04","stderr":5.149,"costPerTest":0.115876,"latency":986.553},{"benchmarkId":"vals-ai:finance_agent:overall","benchmarkName":"Finance Agent v1.1","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":36.647,"unit":"percent","family":"vals-ai:finance_agent:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/finance_agent","measuredAt":"2026-06-04","stderr":2.609,"costPerTest":0.249634,"latency":666.531},{"benchmarkId":"epoch-ai:epoch-frontiermath-2025-02-28-public:overall","benchmarkName":"FrontierMath-2025-02-28-Public","category":"math","domain":null,"eligible":false,"ranked":true,"score":30,"unit":"percent","family":"epoch-ai:epoch-frontiermath-2025-02-28-public:overall","index":null,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":16.32993161855452,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-2025-02-28-private:overall","benchmarkName":"FrontierMath-2025-02-28-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":21.403508771929825,"unit":"percent","family":"epoch-ai:epoch-frontiermath-2025-02-28-private:overall","index":48.80871202680912,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.433800055326239,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-public:overall","benchmarkName":"FrontierMath-Tier-4-2025-07-01-Public","category":"math","domain":null,"eligible":false,"ranked":true,"score":0,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-public:overall","index":null,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-private:overall","benchmarkName":"FrontierMath-Tier-4-2025-07-01-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":0,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-private:overall","index":43.20235772716562,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":0.6,"outputPerMillion":2.5,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":48,"timeToFirstToken":1.764,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":65,"benchmarkCount":10,"categoryCount":4,"confidence":"low","state":"available"},"hosts":[{"host":"amazon-bedrock","name":"Amazon Bedrock","kind":"cloud","url":"https://docs.aws.amazon.com/bedrock/latest/userguide/models-supported.html","inputPerMillion":0.6,"outputPerMillion":2.5,"cacheReadPerMillion":null,"contextTokens":262144,"maxOutputTokens":16000,"deprecated":true},{"host":"io-net","name":"IO.NET","kind":"host","url":"https://io.net/docs/guides/intelligence/io-intelligence","inputPerMillion":0.6,"outputPerMillion":2.5,"cacheReadPerMillion":0.3,"contextTokens":32768,"maxOutputTokens":4096,"deprecated":false},{"host":"novita-ai","name":"Novita AI","kind":"host","url":"https://novita.ai/docs/guides/introduction","inputPerMillion":0.6,"outputPerMillion":2.5,"cacheReadPerMillion":0.15,"contextTokens":262144,"maxOutputTokens":98304,"deprecated":false},{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":0.6,"outputPerMillion":2.5,"cacheReadPerMillion":0.15,"contextTokens":262144,"maxOutputTokens":98304,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:swe-bench-team","name":"SWE-bench team","url":"https://www.swebench.com","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"qwen35plusthinking","providerId":"alibaba","slug":"alibaba-qwen3-5-plus-thinking","name":"Qwen3.5 Plus Thinking","provider":"Alibaba","description":"Qwen3.5 Plus Thinking is a model from Alibaba. 8 benchmarks count toward its index, in 5 categories.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":48.11925331153051,"overallScoreRange":{"low":39.90229498563727,"high":56.336211637423744},"badges":[],"capabilityScores":{"agentic":51.5828357452562,"coding":53.93744464351435,"reasoning":null,"multimodal":5,"knowledge":55.830359762437126,"multilingual":null,"instruction":null,"math":53.10685730952174},"capabilityCounts":{"knowledge":2,"math":1,"coding":3,"agentic":1,"multimodal":1},"benchmarks":[{"benchmarkId":"vals-ai:medqa:overall","benchmarkName":"MedQA","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":95.208,"unit":"percent","family":"vals-ai:medqa:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medqa","measuredAt":"2026-04-16","stderr":0.195,"costPerTest":0.00829,"latency":52.361},{"benchmarkId":"vals-ai:gpqa:overall","benchmarkName":"GPQA Diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":87.374,"unit":"percent","family":"gpqa-diamond","index":57.346146036290826,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","measuredAt":"2026-09-01","stderr":1.669,"costPerTest":0.018542,"latency":103.772},{"benchmarkId":"vals-ai:mmlu_pro:overall","benchmarkName":"MMLU Pro","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":87.178,"unit":"percent","family":"mmlu-pro","index":54.314573488583434,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","measuredAt":"2026-09-01","stderr":0.328,"costPerTest":0.008912,"latency":76.678},{"benchmarkId":"vals-ai:aime:overall","benchmarkName":"AIME","category":"math","domain":null,"eligible":true,"ranked":true,"score":86.042,"unit":"percent","family":"vals-ai:aime:overall","index":53.10685730952174,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/aime","measuredAt":"2026-04-16","stderr":1.111,"costPerTest":0.04516,"latency":213.536},{"benchmarkId":"vals-ai:lcb:overall","benchmarkName":"LiveCodeBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":85.326,"unit":"percent","family":"livecodebench","index":60.142266609386496,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","measuredAt":"2026-09-01","stderr":1.001,"costPerTest":0.04786,"latency":247.472},{"benchmarkId":"vals-ai:legal_bench:overall","benchmarkName":"LegalBench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":85.104,"unit":"percent","family":"vals-ai:legal_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","measuredAt":"2026-10-13","stderr":0.408,"costPerTest":0.003671,"latency":30.422},{"benchmarkId":"vals-ai:swebench:overall","benchmarkName":"SWE-bench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":71.2,"unit":"percent","family":"swe-bench-verified","index":53.74202706767051,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/swebench","measuredAt":"2026-09-01","stderr":2.027,"costPerTest":0.963553,"latency":496.245},{"benchmarkId":"vals-ai:corp_fin_v2:overall","benchmarkName":"CorpFin v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":65.307,"unit":"percent","family":"vals-ai:corp_fin_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","measuredAt":"2026-08-12","stderr":0.936,"costPerTest":0.058073,"latency":155.861},{"benchmarkId":"vals-ai:mortgage_tax:overall","benchmarkName":"MortgageTax","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":60.772,"unit":"percent","family":"vals-ai:mortgage_tax:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mortgage_tax","measuredAt":"2026-09-01","stderr":0.972,"costPerTest":0.010812,"latency":62.075},{"benchmarkId":"vals-ai:case_law_v2:overall","benchmarkName":"CaseLaw v2","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":59.7,"unit":"percent","family":"vals-ai:case_law_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/case_law_v2","measuredAt":"2026-05-04","stderr":1.134,"costPerTest":0.029604,"latency":94.691},{"benchmarkId":"vals-ai:finance_agent:overall","benchmarkName":"Finance Agent v1.1","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":54.475,"unit":"percent","family":"vals-ai:finance_agent:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/finance_agent","measuredAt":"2026-06-04","stderr":2.894,"costPerTest":0.236035,"latency":360.478},{"benchmarkId":"vals-ai:terminal-bench-2:overall","benchmarkName":"Terminal-Bench 2.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":41.573,"unit":"percent","family":"vals-ai:terminal-bench-2:overall","index":51.5828357452562,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2","measuredAt":"2026-06-04","stderr":5.254,"costPerTest":0.321994,"latency":1877.151},{"benchmarkId":"vals-ai:mmmu:overall","benchmarkName":"MMMU Pro","category":"multimodal","domain":null,"eligible":true,"ranked":true,"score":22.775,"unit":"percent","family":"mmmu-pro","index":5,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmmu","measuredAt":"2026-09-01","stderr":1.008,"costPerTest":0.01513,"latency":135.183},{"benchmarkId":"vals-ai:vibe-code:overall","benchmarkName":"Vibe Code Bench v1.1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":15.738,"unit":"percent","family":"vals-ai:vibe-code:overall","index":47.92804025348604,"aboveFit":false,"run":"OpenHands","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","measuredAt":"2026-11-09","stderr":3.179,"costPerTest":3.803761,"latency":3015.618}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":75,"benchmarkCount":8,"categoryCount":5,"confidence":"low","state":"available"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"glm-5v-turbo","providerId":"z-ai","slug":"glm-5v-turbo","name":"GLM-5V-Turbo","provider":"Z.AI","description":"GLM-5V-Turbo is a standard model from Z.AI in the GLM-5 family. 10 benchmarks count toward its index, in 5 categories.","logoUrl":null,"releaseDate":"2026-04-02","contextWindowTokens":202752,"modalities":["image","text","video"],"openWeights":false,"details":{"apiId":"z-ai/glm-5v-turbo","maxOutputTokens":131072,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":false,"cacheReadPerMillion":0.24,"weightsUrl":null},"reasoning":false,"overallScore":48.08637512156236,"overallScoreRange":{"low":39.47297214507009,"high":56.69977809805464},"badges":[],"capabilityScores":{"agentic":50.50369373992361,"coding":null,"reasoning":46.770078205317425,"multimodal":49.8956279285109,"knowledge":46.8555903972507,"multilingual":null,"instruction":46.12222795814753,"math":null},"capabilityCounts":{"agentic":3,"knowledge":3,"multimodal":1,"reasoning":2,"instruction":1},"benchmarks":[{"benchmarkId":"tau2Bench","benchmarkName":"τ²-Bench Tool-Agent-User Evaluation","category":"agentic","score":98.5,"unit":"percent","family":"tau2-bench","index":68.15139166961706,"aboveFit":true,"run":null,"source":"Victor Barres, Honghua Dong, Soham Ray, Xujie Si, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2506.07982","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":80.9,"unit":"percent","family":"gpqa-diamond","index":51.40837013886014,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaMmmuPro","benchmarkName":"Artificial Analysis MMMU-Pro","category":"multimodal","score":72.8,"unit":"percent","family":"mmmu-pro","index":49.8956279285109,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmmu-pro","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":70.3,"unit":"percent","family":"lcr","index":55.529510407842615,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaIfBench","benchmarkName":"Artificial Analysis IFBench","category":"instruction","score":61.1,"unit":"percent","family":"ifbench-family","index":46.12222795814753,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"clawEval","benchmarkName":"Claw-Eval","category":"agentic","score":53.8,"unit":"percent","family":"clawEval","index":42.90604430665405,"aboveFit":false,"run":null,"source":"Bowen Ye, Rang Li, Qibin Yang, Yuanxin Liu, Linli Yao, Hanglong Lv, Zhihui Xie, Chenxin An, Lei Li, Lingpeng Kong, Qi Liu, Zhifang Sui, Tong Yang","sourceUrl":"https://arxiv.org/abs/2604.06132","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gertLabs","benchmarkName":"Gert Labs Composite Game Benchmark","category":"agentic","score":30.76,"unit":"percent","family":"gertLabs","index":40.45364524349973,"aboveFit":false,"run":null,"source":"Gert Labs","sourceUrl":"https://gertlabs.com/rankings","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"omniscienceAccuracy","benchmarkName":"Artificial Analysis Omniscience Accuracy","category":"knowledge","score":29.3,"unit":"percent","family":"omniscienceAccuracy","index":47.615450799569,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":17.1,"unit":"percent","family":"hle","index":41.54295025332296,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":0.6,"unit":"percent","family":"critpt","index":38.010646002792235,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":1.2,"outputPerMillion":4,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":30,"timeToFirstToken":3.3205,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":{"elo":1437.863922977916,"eloUpper":1444.3092438275944,"eloLower":1431.4186021282371,"votes":10291,"rank":82,"rankUpper":null,"rankLower":null,"arenaName":"GLM-5V-Turbo","organization":"zai","updatedAt":"2026-11-09"},"quality":{"coveragePercent":65,"benchmarkCount":10,"categoryCount":5,"confidence":"low","state":"available"},"hosts":[{"host":"zhipu-ai","name":"Zhipu AI","kind":"maker","url":"https://docs.bigmodel.cn/cn/guide/start/pricing","inputPerMillion":0.74698,"outputPerMillion":3.28669,"cacheReadPerMillion":0.17927,"contextTokens":200000,"maxOutputTokens":131072,"deprecated":false},{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":1.2,"outputPerMillion":4,"cacheReadPerMillion":0.24,"contextTokens":202752,"maxOutputTokens":131072,"deprecated":false},{"host":"siliconflow","name":"SiliconFlow","kind":"host","url":"https://cloud.siliconflow.com/models","inputPerMillion":1.2,"outputPerMillion":4,"cacheReadPerMillion":0.24,"contextTokens":200000,"maxOutputTokens":131072,"deprecated":false},{"host":"z-ai","name":"Z.AI","kind":"maker","url":"https://docs.z.ai/guides/overview/pricing","inputPerMillion":1.2,"outputPerMillion":4,"cacheReadPerMillion":0.24,"contextTokens":200000,"maxOutputTokens":131072,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"claude-sonnet-4-5","providerId":"anthropic","slug":"claude-sonnet-4-5","name":"Claude Sonnet 4.5","provider":"Anthropic","description":"Claude Sonnet 4.5 is a standard model from Anthropic. 21 benchmarks count toward its index, in 5 categories.","logoUrl":null,"releaseDate":"2025-09-29","contextWindowTokens":1000000,"modalities":["text","image","file"],"openWeights":false,"details":{"apiId":"anthropic/claude-sonnet-4.5","maxOutputTokens":64000,"knowledgeCutoff":"2025-01-31","retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.3,"weightsUrl":null},"reasoning":false,"overallScore":47.877151762830344,"overallScoreRange":{"low":42.348568477714345,"high":53.405735047946344},"badges":[],"capabilityScores":{"agentic":49.2754843624847,"coding":53.88953858322726,"reasoning":45.31224839729006,"multimodal":null,"knowledge":49.41948252279101,"multilingual":null,"instruction":null,"math":46.574583171790366},"capabilityCounts":{"math":7,"agentic":6,"knowledge":2,"coding":1,"reasoning":5},"benchmarks":[{"benchmarkId":"epoch-ai:epoch-math-level-5:overall","benchmarkName":"MATH level 5","category":"math","domain":null,"eligible":true,"ranked":true,"score":97.73413897280967,"unit":"percent","family":"epoch-ai:epoch-math-level-5:overall","index":46.47279472854164,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0.4091286174812221,"costPerTest":null,"latency":null},{"benchmarkId":"tau2-bench:telecom:overall","benchmarkName":"τ²-bench Telecom","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":91.435,"unit":"percent","family":"tau2-bench","index":63.4632476338909,"aboveFit":false,"run":"enabled effort · Sierra","source":"Sierra Research","sourceUrl":"https://github.com/sierra-research/tau2-bench","measuredAt":"2026-03-02","stderr":null,"costPerTest":0.48769,"latency":null},{"benchmarkId":"aime2025","benchmarkName":"American Invitational Mathematics Examination 2025","category":"math","score":87,"unit":"percent","family":"aime2025","index":46.21566778096984,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:medscribe:overall","benchmarkName":"MedScribe","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":84.515,"unit":"percent","family":"vals-ai:medscribe:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medscribe","measuredAt":"2026-10-29","stderr":1.929,"costPerTest":0.054649,"latency":44.419},{"benchmarkId":"gpqa","benchmarkName":"Graduate-Level Google-Proof Q&A","category":"knowledge","score":83.4,"unit":"percent","family":"gpqa-diamond","index":53.70130182615961,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":80.25688355637594,"unit":"percent","family":"gpqa-diamond","index":50.81852130997656,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.912652283458678,"costPerTest":null,"latency":null},{"benchmarkId":"tau2-bench:retail:overall","benchmarkName":"τ²-bench Retail","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":79.285,"unit":"percent","family":"tau2-bench","index":55.400834324043394,"aboveFit":false,"run":"enabled effort · Sierra","source":"Sierra Research","sourceUrl":"https://github.com/sierra-research/tau2-bench","measuredAt":"2026-04-30","stderr":null,"costPerTest":0.25494,"latency":null},{"benchmarkId":"sweVerified","benchmarkName":"Software Engineering Benchmark Verified","category":"coding","score":77.2,"unit":"percent","family":"swe-bench-verified","index":58.16737253437267,"aboveFit":false,"run":null,"source":"Carlos E. Jimenez, John Yang, Alexander Wettig, Shunyu Yao, Kexin Pei, Ofir Press, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2310.06770","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":74.44444444444446,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":52.23042539225004,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":6.267511942419625,"costPerTest":null,"latency":null},{"benchmarkId":"swebench-experiments:swe-bench-verified:overall","benchmarkName":"SWE-bench Verified","category":"coding","domain":null,"eligible":true,"ranked":true,"score":71.4,"unit":"percent","family":"swe-bench-verified","index":53.88953858322726,"aboveFit":false,"run":"high effort · mini-SWE-agent","source":"SWE-bench team","sourceUrl":"https://github.com/SWE-bench/experiments/tree/main/evaluation/verified","measuredAt":"2026-09-01","stderr":null,"costPerTest":0.6578964984000001,"latency":null},{"benchmarkId":"epoch-ai:epoch-swe-bench-verified:overall","benchmarkName":"SWE-Bench verified","category":"coding","domain":null,"eligible":true,"ranked":true,"score":71.28099173553719,"unit":"percent","family":"swe-bench-verified","index":53.80176313595382,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.0587234937983245,"costPerTest":null,"latency":null},{"benchmarkId":"tau2-bench:airline:overall","benchmarkName":"τ²-bench Airline","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":71,"unit":"percent","family":"tau2-bench","index":49.90313109177289,"aboveFit":false,"run":"enabled effort · Sierra","source":"Sierra Research","sourceUrl":"https://github.com/sierra-research/tau2-bench","measuredAt":"2026-03-02","stderr":null,"costPerTest":0.29646,"latency":null},{"benchmarkId":"swebench-experiments:swe-bench-multilingual:overall","benchmarkName":"SWE-bench Multilingual","category":"coding","domain":null,"eligible":true,"ranked":true,"score":67,"unit":"percent","family":"swe-bench-multilingual","index":null,"aboveFit":false,"run":"mini-SWE-agent","source":"SWE-bench team","sourceUrl":"https://github.com/SWE-bench/experiments/tree/main/evaluation/multilingual","measuredAt":"2026-09-02","stderr":null,"costPerTest":0.669532236,"latency":null},{"benchmarkId":"osWorldVerified","benchmarkName":"OSWorld-Verified","category":"agentic","score":61.4,"unit":"percent","family":"osWorldVerified","index":50.04659417340847,"aboveFit":false,"run":null,"source":"Tianbao Xie, Danyang Zhang, Jixuan Chen, Xiaochuan Li, Siheng Zhao, Ruisheng Cao, Toh Jing Hua, Zhoujun Cheng, Dongchan Shin, Fangyu Lei, Yitao Liu, Yiheng Xu, Shuyan Zhou, Silvio Savarese, Caiming Xiong, Victor Zhong, Tao Yu","sourceUrl":"https://os-world.github.io/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:corp_fin_v2:overall","benchmarkName":"CorpFin v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":60.8,"unit":"percent","family":"vals-ai:corp_fin_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","measuredAt":"2026-08-12","stderr":0.962,"costPerTest":0.237245,"latency":13.383},{"benchmarkId":"terminalBench2","benchmarkName":"Terminal-Bench 2.0","category":"agentic","score":50,"unit":"percent","family":"terminalBench2","index":49.46491330193488,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gertLabs","benchmarkName":"Gert Labs Composite Game Benchmark","category":"agentic","score":48.51,"unit":"percent","family":"gertLabs","index":54.82611820302394,"aboveFit":false,"run":null,"source":"Gert Labs","sourceUrl":"https://gertlabs.com/rankings","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:medcode:overall","benchmarkName":"MedCode","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":40.569,"unit":"percent","family":"vals-ai:medcode:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medcode","measuredAt":"2026-10-29","stderr":1.995,"costPerTest":0.042403,"latency":12.006},{"benchmarkId":"arc-prize:v1-public-eval:overall","benchmarkName":"ARC-AGI-1 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":35.38,"unit":"percent","family":"arc-prize:v1-public-eval:overall","index":null,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.0691,"latency":null},{"benchmarkId":"jobBench","benchmarkName":"JobBench","category":"agentic","score":27.7,"unit":"percent","family":"jobBench","index":51.81443663099212,"aboveFit":false,"run":null,"source":"Yuetai Li, Yichen Feng, Zhangchen Xu, Zixian Ma, Kaiyuan Zheng, Fengqing Jiang, Xinghua Sun, Rulin Shao, Zichen Chen, Yue Huang, Xinyang Han, Brian Lee, Kayla Xu, Shenglai Zeng, Hang Hua, Xiangliang Zhang, Basel Alomair, Ranjay Krishna, Luke Zettlemoyer, Pang Wei Koh, Bhaskar Ramasubramanian, Luyao Niu, Xiang Yue, Radha Poovendran","sourceUrl":"https://arxiv.org/abs/2605.26329","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-simpleqa-verified:overall","benchmarkName":"SimpleQA Verified","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":27.2,"unit":"percent","family":"epoch-ai:epoch-simpleqa-verified:overall","index":46.579053477513945,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.459328489285274,"costPerTest":null,"latency":null},{"benchmarkId":"arc-prize:v1-semi-private:overall","benchmarkName":"ARC-AGI-1 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":25.5,"unit":"percent","family":"arc-prize:v1-semi-private:overall","index":38.71062948603792,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.0808,"latency":null},{"benchmarkId":"tau2-bench:banking-knowledge:overall","benchmarkName":"τ²-bench Banking","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":25.26,"unit":"percent","family":"tau2-bench","index":19.551296952478644,"aboveFit":false,"run":"enabled effort · Sierra","source":"Sierra Research","sourceUrl":"https://github.com/sierra-research/tau2-bench","measuredAt":"2026-10-20","stderr":null,"costPerTest":4.04814,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tiers-1-3-v2-private:overall","benchmarkName":"FrontierMath-Tiers-1-3-v2-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":23.859649122807017,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tiers-1-3-v2-private:overall","index":43.453263452099215,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.5291832284114157,"costPerTest":null,"latency":null},{"benchmarkId":"vitaBench","benchmarkName":"VITA-Bench","category":"agentic","score":17,"unit":"percent","family":"vitaBench","index":36.84886115764063,"aboveFit":false,"run":null,"source":"Meituan LongCat Team","sourceUrl":"https://vitabench.github.io/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-mystery-game-puzzles:overall","benchmarkName":"Mystery Game Puzzles","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":17,"unit":"percent","family":"epoch-ai:epoch-mystery-game-puzzles:overall","index":51.84605144605811,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":3.7752516806863685,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-2025-02-28-private:overall","benchmarkName":"FrontierMath-2025-02-28-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":13.494809688581316,"unit":"percent","family":"epoch-ai:epoch-frontiermath-2025-02-28-private:overall","index":41.82708942227173,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.0132987375676095,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-chess-puzzles:overall","benchmarkName":"Chess Puzzles","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":10,"unit":"percent","family":"epoch-ai:epoch-chess-puzzles:overall","index":41.574827676639885,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":3.015113445777636,"costPerTest":null,"latency":null},{"benchmarkId":"arc-prize:v2-public-eval:overall","benchmarkName":"ARC-AGI-2 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":3.75,"unit":"percent","family":"arc-prize:v2-public-eval:overall","index":null,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.1313,"latency":null},{"benchmarkId":"arc-prize:v2-semi-private:overall","benchmarkName":"ARC-AGI-2 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":3.75,"unit":"percent","family":"arc-prize:v2-semi-private:overall","index":40.910270530903134,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.1295,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-private:overall","benchmarkName":"FrontierMath-Tier-4-2025-07-01-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":3.1249999999999996,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-private:overall","index":46.37636610282637,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.9147663515556026,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tier-4-v2-private:overall","benchmarkName":"FrontierMath-Tier-4-v2-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":2.4390243902439024,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tier-4-v2-private:overall","index":49.446475323573715,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.4390243902439024,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-ebr-bench:overall","benchmarkName":"EBR-bench","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":2.380952380952381,"unit":"percent","family":"epoch-ai:epoch-ebr-bench:overall","index":53.51946284681125,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-2025-02-28-public:overall","benchmarkName":"FrontierMath-2025-02-28-Public","category":"math","domain":null,"eligible":false,"ranked":true,"score":0,"unit":"percent","family":"epoch-ai:epoch-frontiermath-2025-02-28-public:overall","index":null,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-public:overall","benchmarkName":"FrontierMath-Tier-4-2025-07-01-Public","category":"math","domain":null,"eligible":false,"ranked":true,"score":0,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-public:overall","index":null,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":3,"outputPerMillion":15,"batch":{"inputPerMillion":1.5,"outputPerMillion":7.5},"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":45,"timeToFirstToken":0.959,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":{"elo":1438.5944728654613,"eloUpper":1441.4498526303291,"eloLower":1435.739093100593,"votes":82119,"rank":80,"rankUpper":null,"rankLower":null,"arenaName":"Claude Sonnet 4.5-20250929","organization":"anthropic","updatedAt":"2026-11-09"},"quality":{"coveragePercent":80,"benchmarkCount":21,"categoryCount":5,"confidence":"medium","state":"available"},"hosts":[{"host":"anthropic","name":"Anthropic","kind":"maker","url":"https://docs.anthropic.com/en/docs/about-claude/models","inputPerMillion":3,"outputPerMillion":15,"cacheReadPerMillion":0.3,"contextTokens":200000,"maxOutputTokens":64000,"deprecated":true},{"host":"databricks","name":"Databricks","kind":"cloud","url":"https://docs.databricks.com/aws/en/machine-learning/foundation-models/","inputPerMillion":3,"outputPerMillion":15,"cacheReadPerMillion":0.3,"contextTokens":200000,"maxOutputTokens":64000,"deprecated":false},{"host":"digitalocean","name":"DigitalOcean","kind":"cloud","url":"https://docs.digitalocean.com/products/gradient-ai-platform/details/models/","inputPerMillion":3,"outputPerMillion":15,"cacheReadPerMillion":0.3,"contextTokens":200000,"maxOutputTokens":64000,"deprecated":false},{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":3,"outputPerMillion":15,"cacheReadPerMillion":0.3,"contextTokens":1000000,"maxOutputTokens":64000,"deprecated":false},{"host":"sap-ai-core","name":"SAP AI Core","kind":"cloud","url":"https://help.sap.com/docs/sap-ai-core","inputPerMillion":3,"outputPerMillion":15,"cacheReadPerMillion":0.3,"contextTokens":200000,"maxOutputTokens":64000,"deprecated":false},{"host":"snowflake","name":"Snowflake Cortex","kind":"cloud","url":"https://docs.snowflake.com/en/user-guide/snowflake-cortex/cortex-rest-api","inputPerMillion":3,"outputPerMillion":15,"cacheReadPerMillion":0.3,"contextTokens":200000,"maxOutputTokens":64000,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:sierra-research","name":"Sierra Research","url":"https://github.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:swe-bench-team","name":"SWE-bench team","url":"https://github.com","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:arc-prize-foundation","name":"ARC Prize Foundation","url":"https://arcprize.org","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gemma-4-31b","providerId":"google","slug":"gemma-4-31b","name":"Gemma 4 31B","provider":"Google","description":"Gemma 4 31B is a reasoning model from Google in the Gemma 4 family. 24 benchmarks count toward its index, in 8 categories.","logoUrl":null,"releaseDate":"2026-04-02","contextWindowTokens":256000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":true,"overallScore":47.8562192038575,"overallScoreRange":{"low":44.55553177545717,"high":51.15690663225783},"badges":[],"capabilityScores":{"agentic":44.87177002819141,"coding":50.128086787094794,"reasoning":46.58096140585252,"multimodal":56.868042667314306,"knowledge":44.553171105958654,"multilingual":47.49976574253759,"instruction":59.67263187317533,"math":51.583987563657715},"capabilityCounts":{"knowledge":5,"coding":4,"multimodal":2,"instruction":1,"math":1,"reasoning":4,"multilingual":1,"agentic":6},"benchmarks":[{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":85.7,"unit":"percent","family":"gpqa-diamond","index":55.81079897847511,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmluPro","benchmarkName":"Massive Multitask Language Understanding Professional","category":"knowledge","score":85.2,"unit":"percent","family":"mmlu-pro","index":51.82253024909739,"aboveFit":false,"run":null,"source":"Yubo Wang, Xueguang Ma, Ge Zhang, Yuansheng Ni, Abhranil Chandra, Shiguang Guo, Weiming Ren, Aaran Arulraj, Xuan He, Ziyan Jiang, Tianle Li, Max Ku, Kai Wang, Alex Zhuang, Rongqi Fan, Xiang Yue, Wenhu Chen","sourceUrl":"https://arxiv.org/abs/2406.01574","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gpqa","benchmarkName":"Graduate-Level Google-Proof Q&A","category":"knowledge","score":84.3,"unit":"percent","family":"gpqa-diamond","index":54.526757233587404,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"cyberHarmfulRefusal","benchmarkName":"Refusal of harmful cyber requests (JailbreakBench, StrongREJECT, AgentHarm)","category":"agentic","score":84,"unit":"percent","family":"cyberHarmfulRefusal","index":null,"aboveFit":false,"run":null,"source":"Mistral AI (composite of JailbreakBench, StrongREJECT and AgentHarm)","sourceUrl":"https://mistral.ai/news/mistral-large-4-oss/","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"b3AgentSecurity","benchmarkName":"b3: Backbone Breaker Benchmark","category":"agentic","score":83,"unit":"percent","family":"b3AgentSecurity","index":null,"aboveFit":false,"run":null,"source":"Lakera with the UK AI Security Institute","sourceUrl":"https://www.lakera.ai/blog/the-backbone-breaker-benchmark","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:sage:overall","benchmarkName":"SAGE","category":null,"domain":"Education","eligible":false,"ranked":true,"score":82.782,"unit":"percent","family":"vals-ai:sage:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/sage","measuredAt":"2026-11-05","stderr":2.085,"costPerTest":0,"latency":106.811},{"benchmarkId":"liveCodeBenchV6","benchmarkName":"LiveCodeBench v6","category":"coding","score":80,"unit":"percent","family":"liveCodeBenchV6","index":50.101642874078514,"aboveFit":false,"run":null,"source":"LiveCodeBench maintainers","sourceUrl":"https://github.com/LiveCodeBench/LiveCodeBench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmmuPro","benchmarkName":"Massive Multi-discipline Multimodal Understanding Pro","category":"multimodal","score":76.9,"unit":"percent","family":"mmmu-pro","index":55.23141821549949,"aboveFit":false,"run":null,"source":"MMMU-Pro authors","sourceUrl":"https://arxiv.org/abs/2409.02813","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":75.75757575757575,"unit":"percent","family":"gpqa-diamond","index":46.69187914086354,"aboveFit":false,"run":"minimal effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":3.053289223393202,"costPerTest":null,"latency":null},{"benchmarkId":"aaIfBench","benchmarkName":"Artificial Analysis IFBench","category":"instruction","score":75.6,"unit":"percent","family":"ifbench-family","index":59.67263187317533,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"reactNativeEvals","benchmarkName":"React Native Evals","category":"coding","score":75.2,"unit":"percent","family":"reactNativeEvals","index":51.58911106734454,"aboveFit":false,"run":null,"source":"Callstack","sourceUrl":"https://rn-evals.vercel.app/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaMmmuPro","benchmarkName":"Artificial Analysis MMMU-Pro","category":"multimodal","score":73.4,"unit":"percent","family":"mmmu-pro","index":50.676475287582406,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmmu-pro","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":73.33333333333333,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":51.583987563657715,"aboveFit":false,"run":"minimal effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":6.666666666666667,"costPerTest":null,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":69.7,"unit":"percent","family":"lcr","index":55.11786209249122,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-reasoning:overall","benchmarkName":"LiveBench Reasoning","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":68.3,"unit":"percent","family":"livebench","index":50.09786327222921,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Reasoning","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.00377000495049505,"latency":null},{"benchmarkId":"euroeval:euroeval-french:overall","benchmarkName":"EuroEval French","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":67.88625,"unit":"percent","family":"euroeval","index":64.32625652739793,"aboveFit":false,"run":"gemma-4-31B-it","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/french","measuredAt":null,"stderr":0.38623786388700937,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-portuguese:overall","benchmarkName":"EuroEval Portuguese","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":67.48375,"unit":"percent","family":"euroeval","index":63.78015664000942,"aboveFit":false,"run":"gemma-4-31B-it","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/portuguese","measuredAt":null,"stderr":0.5949133340243771,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-italian:overall","benchmarkName":"EuroEval Italian","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":66.84,"unit":"percent","family":"euroeval","index":62.90673601266446,"aboveFit":false,"run":"gemma-4-31B-it","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/italian","measuredAt":null,"stderr":0.4364809560565043,"costPerTest":null,"latency":null},{"benchmarkId":"mrcrv2","benchmarkName":"MRCRv2","category":"reasoning","score":66.4,"unit":"percent","family":"mrcrv2","index":51.264522573637485,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":63.11625,"unit":"percent","family":"euroeval","index":57.85446407312921,"aboveFit":false,"run":"gemma-4-31B-it","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":0.7642807075937479,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-dutch:overall","benchmarkName":"EuroEval Dutch","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":62.660000000000004,"unit":"percent","family":"euroeval","index":57.235437803263366,"aboveFit":false,"run":"gemma-4-31B-it","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/dutch","measuredAt":null,"stderr":0.44313696570064015,"costPerTest":null,"latency":null},{"benchmarkId":"parseBenchLlamaIndex","benchmarkName":"ParseBench: document parsing for AI agents (LlamaIndex leaderboard)","category":"multimodal","score":62.4,"unit":"percent","family":"parseBenchLlamaIndex","index":60.78213858308766,"aboveFit":false,"run":null,"source":"Boyang Zhang, Sebastián G. Acosta, Preston Carlson, Sacha Bron, Pierre-Loïc Doulcet, Daniel B. Ospina and Simon Suo (LlamaIndex)","sourceUrl":"https://arxiv.org/abs/2604.08538","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":61.642857142857146,"unit":"percent","family":"euroeval","index":55.85540898402427,"aboveFit":false,"run":"gemma-4-31B-it","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.43292078740026485,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:mortgage_tax:overall","benchmarkName":"MortgageTax","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":61.368,"unit":"percent","family":"vals-ai:mortgage_tax:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mortgage_tax","measuredAt":"2026-09-01","stderr":0.967,"costPerTest":0,"latency":41.928},{"benchmarkId":"tau2Bench","benchmarkName":"τ²-Bench Tool-Agent-User Evaluation","category":"agentic","score":59.9,"unit":"percent","family":"tau2-bench","index":42.53746954944306,"aboveFit":false,"run":null,"source":"Victor Barres, Honghua Dong, Soham Ray, Xujie Si, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2506.07982","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-design:overall","benchmarkName":"LiveBench Design","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":58.4,"unit":"percent","family":"livebench-2-agentic","index":61.29920752572489,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Design","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.014792000000000001,"latency":null},{"benchmarkId":"euroeval:euroeval-spanish:overall","benchmarkName":"EuroEval Spanish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":55.76625,"unit":"percent","family":"euroeval","index":47.88220525994785,"aboveFit":false,"run":"gemma-4-31B-it","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/spanish","measuredAt":null,"stderr":0.6564797216974794,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":55.66250000000001,"unit":"percent","family":"euroeval","index":47.741440382142756,"aboveFit":false,"run":"gemma-4-31B-it","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.6648331557616541,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-portuguese:overall","benchmarkName":"EuroEval Portuguese","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":55.306250000000006,"unit":"percent","family":"euroeval","index":47.258091102932426,"aboveFit":false,"run":"gemma-4-31B","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/portuguese","measuredAt":null,"stderr":0.6512068123875855,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-data-analysis:overall","benchmarkName":"LiveBench Data Analysis","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":53.3,"unit":"percent","family":"livebench","index":30.66635333003086,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Data%20Analysis","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.00456496,"latency":null},{"benchmarkId":"vals-ai:case_law_v2:overall","benchmarkName":"CaseLaw v2","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":52.626,"unit":"percent","family":"vals-ai:case_law_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/case_law_v2","measuredAt":"2026-05-04","stderr":1.343,"costPerTest":0,"latency":28.716},{"benchmarkId":"euroeval:euroeval-dutch:overall","benchmarkName":"EuroEval Dutch","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":51.83777777777778,"unit":"percent","family":"euroeval","index":42.552172370098376,"aboveFit":false,"run":"gemma-4-31B","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/dutch","measuredAt":null,"stderr":0.7455845333000787,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:finance_agent:overall","benchmarkName":"Finance Agent v1.1","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":50.788,"unit":"percent","family":"vals-ai:finance_agent:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/finance_agent","measuredAt":"2026-06-04","stderr":2.856,"costPerTest":0,"latency":5360.333},{"benchmarkId":"euroeval:euroeval-italian:overall","benchmarkName":"EuroEval Italian","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":50.321250000000006,"unit":"percent","family":"euroeval","index":40.494593118754324,"aboveFit":false,"run":"gemma-4-31B","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/italian","measuredAt":null,"stderr":0.9067912177011861,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":50.30375,"unit":"percent","family":"euroeval","index":40.4708496453896,"aboveFit":false,"run":"gemma-4-31B","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":1.1995155272025453,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-french:overall","benchmarkName":"EuroEval French","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":50.04125,"unit":"percent","family":"euroeval","index":40.11469754491883,"aboveFit":false,"run":"gemma-4-31B","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/french","measuredAt":null,"stderr":0.7908559445309873,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-coding-2:overall","benchmarkName":"LiveBench Coding 2.0","category":"coding","domain":null,"eligible":true,"ranked":true,"score":49,"unit":"percent","family":"livebench:livebench-coding-2:overall","index":43.66384760257451,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Coding","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.23261034482758622,"latency":null},{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":46.690000000000005,"unit":"percent","family":"euroeval","index":35.56782239557544,"aboveFit":false,"run":"gemma-4-31B","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.8879740757385168,"costPerTest":null,"latency":null},{"benchmarkId":"aaSciCode","benchmarkName":"Artificial Analysis SciCode","category":"coding","score":45.5,"unit":"percent","family":"scicode-family","index":55.157745604381624,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/scicode","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-research:overall","benchmarkName":"LiveBench Research","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":44.4,"unit":"percent","family":"livebench-2-agentic","index":51.06057219683849,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Research","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.038431818181818185,"latency":null},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":42.64125,"unit":"percent","family":"euroeval","index":30.074600236409715,"aboveFit":false,"run":"gemma-4-31B","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.793350293061016,"costPerTest":null,"latency":null},{"benchmarkId":"sweRebench","benchmarkName":"SWE-Rebench","category":"coding","score":41.6,"unit":"percent","family":"sweRebench","index":null,"aboveFit":false,"run":null,"source":"Nebius","sourceUrl":"https://swe-rebench.com","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-spanish:overall","benchmarkName":"EuroEval Spanish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":40.77875000000001,"unit":"percent","family":"euroeval","index":27.547616285450506,"aboveFit":false,"run":"gemma-4-31B","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/spanish","measuredAt":null,"stderr":0.8453660449769673,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:terminal-bench-2:overall","benchmarkName":"Terminal-Bench 2.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":39.326,"unit":"percent","family":"vals-ai:terminal-bench-2:overall","index":50.229891753902436,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2","measuredAt":"2026-06-04","stderr":5.207,"costPerTest":null,"latency":855.298},{"benchmarkId":"aaItbench","benchmarkName":"Artificial Analysis ITBench-AA","category":"agentic","score":37.3,"unit":"percent","family":"aaItbench","index":54.42537660126898,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/itbench-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gertLabs","benchmarkName":"Gert Labs Composite Game Benchmark","category":"agentic","score":35.26,"unit":"percent","family":"gertLabs","index":44.097370782534036,"aboveFit":false,"run":null,"source":"Gert Labs","sourceUrl":"https://gertlabs.com/rankings","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hle","benchmarkName":"Humanity's Last Exam","category":"knowledge","score":26.5,"unit":"percent","family":"hle","index":50.97390205660302,"aboveFit":false,"run":null,"source":"Center for AI Safety, Scale AI, and thousands of expert contributors","sourceUrl":"https://lastexam.ai/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":23.6,"unit":"percent","family":"hle","index":48.064353096016625,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"omniscienceAccuracy","benchmarkName":"Artificial Analysis Omniscience Accuracy","category":"knowledge","score":19.9,"unit":"percent","family":"omniscienceAccuracy","index":35.76904454940935,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hleNoTools","benchmarkName":"Humanity's Last Exam without tools","category":"knowledge","score":19.5,"unit":"percent","family":"hle","index":43.950852841394465,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-long-horizon:overall","benchmarkName":"LiveBench Long Agentic","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":13.3,"unit":"percent","family":"livebench-2-agentic","index":28.316175144812277,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Long%20Agentic","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.8136827586206897,"latency":null},{"benchmarkId":"epoch-ai:epoch-simpleqa-verified:overall","benchmarkName":"SimpleQA Verified","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":10.4,"unit":"percent","family":"epoch-ai:epoch-simpleqa-verified:overall","index":32.583170401682494,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0.9658016218524242,"costPerTest":null,"latency":null},{"benchmarkId":"gdpvalAaNormalized","benchmarkName":"GDPval-AA normalized","category":"agentic","score":5.9,"unit":"percent","family":"gdpvalAaNormalized","index":38.25213781117459,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-browser:overall","benchmarkName":"LiveBench Browser Use","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":4,"unit":"percent","family":"livebench-2-agentic","index":21.514795962052027,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Browser","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.25412,"latency":null},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":1.4,"unit":"percent","family":"critpt","index":39.55935265615135,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":0,"outputPerMillion":0,"batch":null,"freeTier":true,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":35,"timeToFirstToken":50.03,"source":"BenchLM","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":{"elo":1442.53571459654,"eloUpper":1449.9389309360909,"eloLower":1435.132498256989,"votes":6130,"rank":73,"rankUpper":null,"rankLower":null,"arenaName":"Gemma 4 31B","organization":"google","updatedAt":"2026-11-09"},"quality":{"coveragePercent":100,"benchmarkCount":24,"categoryCount":8,"confidence":"high","state":"available"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:livebench","name":"LiveBench","url":"https://livebench.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"claudeopus41","providerId":"anthropic","slug":"anthropic-claude-opus-4-1","name":"Claude Opus 4.1","provider":"Anthropic","description":"Claude Opus 4.1 is a model from Anthropic. 16 benchmarks count toward its index, in 6 categories.","logoUrl":null,"releaseDate":"2025-08-05T16:33:11.000Z","contextWindowTokens":200000,"modalities":["image","text","file"],"openWeights":null,"details":{"apiId":"anthropic/claude-opus-4.1","maxOutputTokens":32000,"knowledgeCutoff":"2025-01-31","retiresAt":null,"tools":true,"structuredOutput":false,"cacheReadPerMillion":1.5,"weightsUrl":null},"reasoning":null,"overallScore":47.47452793139145,"overallScoreRange":{"low":42.432027931391445,"high":52.51702793139145},"badges":[],"capabilityScores":{"agentic":48.708699490314004,"coding":48.201494787874196,"reasoning":55.50638341115814,"multimodal":51.08642015109494,"knowledge":49.4258759734285,"multilingual":null,"instruction":null,"math":41.9679409650218},"capabilityCounts":{"math":7,"knowledge":2,"agentic":1,"multimodal":1,"coding":3,"reasoning":2},"benchmarks":[{"benchmarkId":"vals-ai:mgsm:overall","benchmarkName":"MGSM","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":94.218,"unit":"percent","family":"vals-ai:mgsm:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mgsm","measuredAt":"2026-01-09","stderr":1.456,"costPerTest":0.027189,"latency":7.897},{"benchmarkId":"vals-ai:math500:overall","benchmarkName":"MATH 500","category":"math","domain":null,"eligible":true,"ranked":true,"score":93,"unit":"percent","family":"math-500","index":42.40529701235612,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/math500","measuredAt":"2026-01-09","stderr":1.141,"costPerTest":0.045983,"latency":30.932},{"benchmarkId":"vals-ai:medqa:overall","benchmarkName":"MedQA","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":92.533,"unit":"percent","family":"vals-ai:medqa:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medqa","measuredAt":"2026-04-16","stderr":0.24,"costPerTest":0.041528,"latency":13.971},{"benchmarkId":"vals-ai:mmlu_pro:overall","benchmarkName":"MMLU Pro","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":87.214,"unit":"percent","family":"mmlu-pro","index":54.3599291794942,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","measuredAt":"2026-09-01","stderr":0.328,"costPerTest":0.007078,"latency":17.757},{"benchmarkId":"vals-ai:legal_bench:overall","benchmarkName":"LegalBench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":83.458,"unit":"percent","family":"vals-ai:legal_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","measuredAt":"2026-10-13","stderr":0.449,"costPerTest":null,"latency":2.79},{"benchmarkId":"tau2-bench:retail:overall","benchmarkName":"τ²-bench Retail","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":82.4,"unit":"percent","family":"tau2-bench","index":57.4678645676792,"aboveFit":false,"run":"Anthropic","source":"Sierra Research","sourceUrl":"https://github.com/sierra-research/tau2-bench","measuredAt":"2026-04-30","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":76.76767676767676,"unit":"percent","family":"gpqa-diamond","index":47.61831618623706,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":3.0088629490217444,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:mmmu:overall","benchmarkName":"MMMU Pro","category":"multimodal","domain":null,"eligible":true,"ranked":true,"score":73.715,"unit":"percent","family":"mmmu-pro","index":51.08642015109494,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmmu","measuredAt":"2026-09-01","stderr":1.058,"costPerTest":0.044475,"latency":46.277},{"benchmarkId":"epoch-ai:epoch-swe-bench-verified:overall","benchmarkName":"SWE-Bench verified","category":"coding","domain":null,"eligible":true,"ranked":true,"score":73.34710743801654,"unit":"percent","family":"swe-bench-verified","index":55.325642428895335,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.0118245554677086,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:medscribe:overall","benchmarkName":"MedScribe","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":71.753,"unit":"percent","family":"vals-ai:medscribe:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medscribe","measuredAt":"2026-10-29","stderr":2.021,"costPerTest":0.187162,"latency":38.04},{"benchmarkId":"vals-ai:tax_eval_v2:overall","benchmarkName":"TaxEval v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":71.464,"unit":"percent","family":"vals-ai:tax_eval_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","measuredAt":"2026-09-01","stderr":0.882,"costPerTest":0.039842,"latency":32.602},{"benchmarkId":"vals-ai:gpqa:overall","benchmarkName":"GPQA Diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":69.95,"unit":"percent","family":"gpqa-diamond","index":41.365329348488494,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","measuredAt":"2026-09-01","stderr":2.303,"costPerTest":0.067421,"latency":35.934},{"benchmarkId":"vals-ai:lcb:overall","benchmarkName":"LiveCodeBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":64.559,"unit":"percent","family":"livecodebench","index":42.91103050668661,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","measuredAt":"2026-09-01","stderr":1.112,"costPerTest":0.055988,"latency":32.509},{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":64.44444444444444,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":46.41248493491917,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":7.21639236343101,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:mortgage_tax:overall","benchmarkName":"MortgageTax","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":61.089,"unit":"percent","family":"vals-ai:mortgage_tax:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mortgage_tax","measuredAt":"2026-09-01","stderr":0.969,"costPerTest":0.088232,"latency":10.065},{"benchmarkId":"tau2-bench:airline:overall","benchmarkName":"τ²-bench Airline","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":56,"unit":"percent","family":"tau2-bench","index":39.9495344129488,"aboveFit":false,"run":"Anthropic","source":"Sierra Research","sourceUrl":"https://github.com/sierra-research/tau2-bench","measuredAt":"2026-03-02","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:aime:overall","benchmarkName":"AIME","category":"math","domain":null,"eligible":true,"ranked":true,"score":44.236,"unit":"percent","family":"vals-ai:aime:overall","index":34.83572224878704,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/aime","measuredAt":"2026-04-16","stderr":1.677,"costPerTest":0,"latency":29.675},{"benchmarkId":"vals-ai:medcode:overall","benchmarkName":"MedCode","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":41.372,"unit":"percent","family":"vals-ai:medcode:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medcode","measuredAt":"2026-10-29","stderr":1.958,"costPerTest":0.20627,"latency":13.084},{"benchmarkId":"epoch-ai:epoch-mystery-game-puzzles:overall","benchmarkName":"Mystery Game Puzzles","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":21,"unit":"percent","family":"epoch-ai:epoch-mystery-game-puzzles:overall","index":54.9437110328788,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":4.093601807403323,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tiers-1-3-v2-private:overall","benchmarkName":"FrontierMath-Tiers-1-3-v2-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":12.631578947368421,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tiers-1-3-v2-private:overall","index":37.54334950067499,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.9712735463357798,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:ioi-v1:overall","benchmarkName":"IOI v1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":12.516,"unit":"percent","family":"vals-ai:ioi-v1:overall","index":46.36781142804064,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/ioi-v1","measuredAt":"2026-08-09","stderr":4.913,"costPerTest":17.585952,"latency":6124.896},{"benchmarkId":"epoch-ai:epoch-ebr-bench:overall","benchmarkName":"EBR-bench","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":7.9365000000000006,"unit":"percent","family":"epoch-ai:epoch-ebr-bench:overall","index":56.06905578943748,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-2025-02-28-private:overall","benchmarkName":"FrontierMath-2025-02-28-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":6.551724137931035,"unit":"percent","family":"epoch-ai:epoch-frontiermath-2025-02-28-private:overall","index":35.6978888401283,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.5245401561146894,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-private:overall","benchmarkName":"FrontierMath-Tier-4-2025-07-01-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":4.166666666666666,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-private:overall","index":47.43436889471329,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.9147663515556026,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tier-4-v2-private:overall","benchmarkName":"FrontierMath-Tier-4-v2-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":2.4390243902439024,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tier-4-v2-private:overall","index":49.446475323573715,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.4390243902439024,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-2025-02-28-public:overall","benchmarkName":"FrontierMath-2025-02-28-Public","category":"math","domain":null,"eligible":false,"ranked":true,"score":0,"unit":"percent","family":"epoch-ai:epoch-frontiermath-2025-02-28-public:overall","index":null,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":15,"outputPerMillion":75,"batch":{"inputPerMillion":7.5,"outputPerMillion":37.5},"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":12,"timeToFirstToken":4.67,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":{"elo":1418.6329605249603,"eloUpper":1421.6145452142046,"eloLower":1415.651375835716,"votes":76931,"rank":124,"rankUpper":null,"rankLower":null,"arenaName":"Claude Opus 4.1-20250805","organization":"anthropic","updatedAt":"2026-11-09"},"quality":{"coveragePercent":90,"benchmarkCount":16,"categoryCount":6,"confidence":"medium","state":"available"},"hosts":[{"host":"databricks","name":"Databricks","kind":"cloud","url":"https://docs.databricks.com/aws/en/machine-learning/foundation-models/","inputPerMillion":15,"outputPerMillion":75,"cacheReadPerMillion":1.5,"contextTokens":200000,"maxOutputTokens":32000,"deprecated":false},{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":15,"outputPerMillion":75,"cacheReadPerMillion":1.5,"contextTokens":200000,"maxOutputTokens":32000,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:sierra-research","name":"Sierra Research","url":"https://github.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"qwen3-235b-2507-reasoning","providerId":"alibaba","slug":"qwen3-235b-2507-reasoning","name":"Qwen3 235B 2507 (Reasoning)","provider":"Alibaba","description":"Qwen3 235B 2507 (Reasoning) is a reasoning model from Alibaba in the Qwen3 235B 2507 family. 8 benchmarks count toward its index, in 5 categories.","logoUrl":null,"releaseDate":"2025-07-25","contextWindowTokens":128000,"modalities":["text"],"openWeights":true,"details":{"apiId":"qwen/qwen3-235b-a22b-thinking-2507","maxOutputTokens":16384,"knowledgeCutoff":"2025-06-30","retiresAt":null,"tools":true,"structuredOutput":false,"cacheReadPerMillion":null,"weightsUrl":"https://huggingface.co/Qwen/Qwen3-235B-A22B-Thinking-2507"},"reasoning":true,"overallScore":47.42408525071423,"overallScoreRange":{"low":37.20712692482099,"high":57.641043576607466},"badges":[],"capabilityScores":{"agentic":null,"coding":45.25740570646145,"reasoning":null,"multimodal":null,"knowledge":48.86663029051431,"multilingual":50.40787459580192,"instruction":40.00420079122482,"math":49.091584607568905},"capabilityCounts":{"math":1,"instruction":1,"knowledge":4,"multilingual":1,"coding":1},"benchmarks":[{"benchmarkId":"aime2025","benchmarkName":"American Invitational Mathematics Examination 2025","category":"math","score":92.3,"unit":"percent","family":"aime2025","index":49.091584607568905,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"ifeval","benchmarkName":"Instruction-Following Eval","category":"instruction","score":87.8,"unit":"percent","family":"ifeval","index":40.00420079122482,"aboveFit":false,"run":null,"source":"Jeffrey Zhou, Tianjian Lu, Swaroop Mishra, Siddhartha Brahma, Sujoy Basu, Yi Luan, Denny Zhou, Le Hou","sourceUrl":"https://arxiv.org/abs/2311.07911","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmluPro","benchmarkName":"Massive Multitask Language Understanding Professional","category":"knowledge","score":84.4,"unit":"percent","family":"mmlu-pro","index":50.81462600663589,"aboveFit":false,"run":null,"source":"Yubo Wang, Xueguang Ma, Ge Zhang, Yuansheng Ni, Abhranil Chandra, Shiguang Guo, Weiming Ren, Aaran Arulraj, Xuan He, Ziyan Jiang, Tianle Li, Max Ku, Kai Wang, Alex Zhuang, Rongqi Fan, Xiang Yue, Wenhu Chen","sourceUrl":"https://arxiv.org/abs/2406.01574","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gpqa","benchmarkName":"Graduate-Level Google-Proof Q&A","category":"knowledge","score":81.1,"unit":"percent","family":"gpqa-diamond","index":51.59180467384409,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmluProX","benchmarkName":"MMLU-ProX","category":"multilingual","score":81,"unit":"percent","family":"mmluProX","index":50.40787459580192,"aboveFit":false,"run":null,"source":"MMLU-ProX authors","sourceUrl":"https://arxiv.org/abs/2503.10497","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"liveCodeBenchV6","benchmarkName":"LiveCodeBench v6","category":"coding","score":74.1,"unit":"percent","family":"liveCodeBenchV6","index":45.25740570646145,"aboveFit":false,"run":null,"source":"LiveCodeBench maintainers","sourceUrl":"https://github.com/LiveCodeBench/LiveCodeBench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"superGpqa","benchmarkName":"SuperGPQA: Scaling LLM Evaluation Across 285 Graduate Disciplines","category":"knowledge","score":64.9,"unit":"percent","family":"superGpqa","index":50.41351820872151,"aboveFit":false,"run":null,"source":"Xiaoxuan Du, Yao Yao, Kexin Ma, Bowen Wang, Tianyu Zheng, Kaiyan Zhu, Yiming Zhang, Yutao Zhu, Jiawei Zhou, Jingren Zhou","sourceUrl":"https://arxiv.org/abs/2502.14739","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hle","benchmarkName":"Humanity's Last Exam","category":"knowledge","score":18.2,"unit":"percent","family":"hle","index":42.646572272855735,"aboveFit":false,"run":null,"source":"Center for AI Safety, Scale AI, and thousands of expert contributors","sourceUrl":"https://lastexam.ai/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":0.44999999999999996,"outputPerMillion":3.5,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":108,"timeToFirstToken":1.0665,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":{"elo":1415.5429590912127,"eloUpper":1422.0806693876561,"eloLower":1409.0052487947692,"votes":8812,"rank":133,"rankUpper":null,"rankLower":null,"arenaName":"Qwen3-235B-A22B-Thinking-2507","organization":"alibaba","updatedAt":"2026-11-09"},"quality":{"coveragePercent":55.00000000000001,"benchmarkCount":8,"categoryCount":5,"confidence":"low","state":"available"},"hosts":[{"host":"io-net","name":"IO.NET","kind":"host","url":"https://io.net/docs/guides/intelligence/io-intelligence","inputPerMillion":0.11,"outputPerMillion":0.6,"cacheReadPerMillion":0.055,"contextTokens":262144,"maxOutputTokens":4096,"deprecated":true},{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":0.45,"outputPerMillion":3.5,"cacheReadPerMillion":null,"contextTokens":128000,"maxOutputTokens":16384,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"grok-4-1-fast-reasoning","providerId":"xai","slug":"grok-4-1-fast-reasoning","name":"Grok 4.1 Fast (Reasoning)","provider":"xAI","description":"Grok 4.1 Fast (Reasoning) is a reasoning model from xAI in the Grok 4.1 Fast family. 17 benchmarks count toward its index, in 8 categories.","logoUrl":null,"releaseDate":"2025-11-19","contextWindowTokens":2000000,"modalities":["text"],"openWeights":false,"details":null,"reasoning":true,"overallScore":47.32467608696423,"overallScoreRange":{"low":43.40287503012673,"high":51.246477143801734},"badges":[],"capabilityScores":{"agentic":48.386483927570836,"coding":43.25546941388621,"reasoning":50.26559299185462,"multimodal":43.62217010450397,"knowledge":48.09590301682535,"multilingual":59.46101155067602,"instruction":38.272338793579706,"math":55.656145230137575},"capabilityCounts":{"agentic":3,"math":1,"knowledge":4,"coding":4,"reasoning":2,"multimodal":1,"multilingual":1,"instruction":1},"benchmarks":[{"benchmarkId":"tau2Bench","benchmarkName":"τ²-Bench Tool-Agent-User Evaluation","category":"agentic","score":93.3,"unit":"percent","family":"tau2-bench","index":64.70081148762469,"aboveFit":false,"run":null,"source":"Victor Barres, Honghua Dong, Soham Ray, Xujie Si, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2506.07982","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:medqa:overall","benchmarkName":"MedQA","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":92.083,"unit":"percent","family":"vals-ai:medqa:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medqa","measuredAt":"2026-04-16","stderr":0.246,"costPerTest":0.000571,"latency":17.636},{"benchmarkId":"vals-ai:aime:overall","benchmarkName":"AIME","category":"math","domain":null,"eligible":true,"ranked":true,"score":91.875,"unit":"percent","family":"vals-ai:aime:overall","index":55.656145230137575,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/aime","measuredAt":"2026-04-16","stderr":0.584,"costPerTest":0.004094,"latency":91.997},{"benchmarkId":"vals-ai:mgsm:overall","benchmarkName":"MGSM","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":89.527,"unit":"percent","family":"vals-ai:mgsm:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mgsm","measuredAt":"2026-01-09","stderr":1.892,"costPerTest":0.000518,"latency":13.501},{"benchmarkId":"aaMmluPro","benchmarkName":"Artificial Analysis MMLU-Pro","category":"knowledge","score":85.4,"unit":"percent","family":"mmlu-pro","index":52.07450630971277,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmlu-pro","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":85.3,"unit":"percent","family":"gpqa-diamond","index":55.443929908507194,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:gpqa:overall","benchmarkName":"GPQA Diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":84.343,"unit":"percent","family":"gpqa-diamond","index":54.56619565860896,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","measuredAt":"2026-09-01","stderr":1.85,"costPerTest":0.002916,"latency":68.481},{"benchmarkId":"vals-ai:mmlu_pro:overall","benchmarkName":"MMLU Pro","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":84.18,"unit":"percent","family":"mmlu-pro","index":50.53745233995898,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","measuredAt":"2026-09-01","stderr":0.357,"costPerTest":0.001078,"latency":25.469},{"benchmarkId":"vals-ai:legal_bench:overall","benchmarkName":"LegalBench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":82.451,"unit":"percent","family":"vals-ai:legal_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","measuredAt":"2026-10-13","stderr":0.457,"costPerTest":0.000384,"latency":8.579},{"benchmarkId":"aaLiveCodeBench","benchmarkName":"Artificial Analysis LiveCodeBench","category":"coding","score":82.2,"unit":"percent","family":"livecodebench","index":57.54849552752394,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/livecodebench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:lcb:overall","benchmarkName":"LiveCodeBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":80.641,"unit":"percent","family":"livecodebench","index":56.254928951508674,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","measuredAt":"2026-09-01","stderr":1.091,"costPerTest":0.003892,"latency":103.449},{"benchmarkId":"vals-ai:medscribe:overall","benchmarkName":"MedScribe","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":78.732,"unit":"percent","family":"vals-ai:medscribe:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medscribe","measuredAt":"2026-10-29","stderr":1.866,"costPerTest":0.002387,"latency":39.292},{"benchmarkId":"vals-ai:sage:overall","benchmarkName":"SAGE","category":null,"domain":"Education","eligible":false,"ranked":true,"score":74.475,"unit":"percent","family":"vals-ai:sage:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/sage","measuredAt":"2026-11-05","stderr":2.249,"costPerTest":0.004658,"latency":13.672},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":74,"unit":"percent","family":"lcr","index":58.06800835250956,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:tax_eval_v2:overall","benchmarkName":"TaxEval v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":73.14,"unit":"percent","family":"vals-ai:tax_eval_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","measuredAt":"2026-09-01","stderr":0.865,"costPerTest":0.001333,"latency":30.758},{"benchmarkId":"vals-ai:mmmu:overall","benchmarkName":"MMMU Pro","category":"multimodal","domain":null,"eligible":true,"ranked":true,"score":72.659,"unit":"percent","family":"mmmu-pro","index":49.71212879912911,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmmu","measuredAt":"2026-09-01","stderr":1.072,"costPerTest":0.00056,"latency":96.354},{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":68.27,"unit":"percent","family":"euroeval","index":64.84691697903851,"aboveFit":true,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":0.6017266821406543,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-italian:overall","benchmarkName":"EuroEval Italian","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":66.13375,"unit":"percent","family":"euroeval","index":61.9485172661598,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/italian","measuredAt":null,"stderr":0.37940331442938136,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:corp_fin_v2:overall","benchmarkName":"CorpFin v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":65.967,"unit":"percent","family":"vals-ai:corp_fin_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","measuredAt":"2026-08-12","stderr":0.934,"costPerTest":0.015643,"latency":28.411},{"benchmarkId":"euroeval:euroeval-french:overall","benchmarkName":"EuroEval French","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":65.37625,"unit":"percent","family":"euroeval","index":60.920764061944155,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/french","measuredAt":null,"stderr":0.3985697084576298,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-portuguese:overall","benchmarkName":"EuroEval Portuguese","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":64.78625,"unit":"percent","family":"euroeval","index":60.12026981707653,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/portuguese","measuredAt":null,"stderr":0.45721473346776564,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-dutch:overall","benchmarkName":"EuroEval Dutch","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":63.81444444444443,"unit":"percent","family":"euroeval","index":58.8017532842755,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/dutch","measuredAt":null,"stderr":0.4334316127867959,"costPerTest":null,"latency":null},{"benchmarkId":"aaMmmuPro","benchmarkName":"Artificial Analysis MMMU-Pro","category":"multimodal","score":63.3,"unit":"percent","family":"mmmu-pro","index":37.532211409878826,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmmu-pro","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:case_law_v2:overall","benchmarkName":"CaseLaw v2","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":60.45,"unit":"percent","family":"vals-ai:case_law_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/case_law_v2","measuredAt":"2026-05-04","stderr":0.809,"costPerTest":0.011453,"latency":19.094},{"benchmarkId":"euroeval:euroeval-spanish:overall","benchmarkName":"EuroEval Spanish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":59.80375,"unit":"percent","family":"euroeval","index":53.36016375766482,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/spanish","measuredAt":null,"stderr":0.4659784732152334,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":57.64571428571429,"unit":"percent","family":"euroeval","index":50.43220584325043,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.42994778990107196,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":54.77499999999999,"unit":"percent","family":"euroeval","index":46.53730709007491,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.6122627091371807,"costPerTest":null,"latency":null},{"benchmarkId":"aaIfBench","benchmarkName":"Artificial Analysis IFBench","category":"instruction","score":52.7,"unit":"percent","family":"ifbench-family","index":38.272338793579706,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:finance_agent:overall","benchmarkName":"Finance Agent v1.1","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":52.448,"unit":"percent","family":"vals-ai:finance_agent:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/finance_agent","measuredAt":"2026-06-04","stderr":2.805,"costPerTest":0.058346,"latency":90.582},{"benchmarkId":"vals-ai:public-benefits-bench:overall","benchmarkName":"Public Benefits Bench v1.1","category":null,"domain":"Public benefits","eligible":false,"ranked":true,"score":44.79,"unit":"percent","family":"vals-ai:public-benefits-bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/public-benefits-bench","measuredAt":"2026-10-12","stderr":1.293,"costPerTest":0.031221,"latency":116.823},{"benchmarkId":"vals-ai:public-benefits-bench-v1:overall","benchmarkName":"Public Benefits Bench v1","category":null,"domain":"Public benefits","eligible":false,"ranked":true,"score":44.317,"unit":"percent","family":"vals-ai:public-benefits-bench-v1:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/public-benefits-bench-v1","measuredAt":"2026-06-09","stderr":1.292,"costPerTest":0.064873,"latency":185.115},{"benchmarkId":"vals-ai:mortgage_tax:overall","benchmarkName":"MortgageTax","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":42.607,"unit":"percent","family":"vals-ai:mortgage_tax:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mortgage_tax","measuredAt":"2026-09-01","stderr":0.952,"costPerTest":0.002438,"latency":47.45},{"benchmarkId":"vals-ai:swebench:overall","benchmarkName":"SWE-bench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":41.4,"unit":"percent","family":"swe-bench-verified","index":31.762811249716492,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/swebench","measuredAt":"2026-09-01","stderr":2.205,"costPerTest":0.027516,"latency":138.424},{"benchmarkId":"vals-ai:terminal-bench:overall","benchmarkName":"Terminal-Bench 1.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":28.75,"unit":"percent","family":"vals-ai:terminal-bench:overall","index":39.02378765186826,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench","measuredAt":"2026-01-12","stderr":5.06,"costPerTest":0.009636,"latency":238.437},{"benchmarkId":"vals-ai:medcode:overall","benchmarkName":"MedCode","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":28.08,"unit":"percent","family":"vals-ai:medcode:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medcode","measuredAt":"2026-10-29","stderr":1.992,"costPerTest":0.002108,"latency":46.5},{"benchmarkId":"omniscienceAccuracy","benchmarkName":"Artificial Analysis Omniscience Accuracy","category":"knowledge","score":25.1,"unit":"percent","family":"omniscienceAccuracy","index":42.322375666518944,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:terminal-bench-2:overall","benchmarkName":"Terminal-Bench 2.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":24.719,"unit":"percent","family":"vals-ai:terminal-bench-2:overall","index":41.43485264321955,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2","measuredAt":"2026-06-04","stderr":4.599,"costPerTest":0.055526,"latency":479.157},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":19.3,"unit":"percent","family":"hle","index":43.75019429238851,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:ioi-v1:overall","benchmarkName":"IOI v1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":3.084,"unit":"percent","family":"vals-ai:ioi-v1:overall","index":41.674852721936574,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/ioi-v1","measuredAt":"2026-08-09","stderr":1.749,"costPerTest":0.02997,"latency":1533.288},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":2.9,"unit":"percent","family":"critpt","index":42.46317763119968,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:vibe-code:overall","benchmarkName":"Vibe Code Bench v1.1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":1.2,"unit":"percent","family":"vals-ai:vibe-code:overall","index":42.682501444375475,"aboveFit":false,"run":"OpenHands","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","measuredAt":"2026-11-09","stderr":1.2,"costPerTest":0.209444,"latency":527.562}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":{"elo":1408.1902236101869,"eloUpper":1411.377934650227,"eloLower":1405.0025125701468,"votes":57133,"rank":145,"rankUpper":null,"rankLower":null,"arenaName":"Grok 4.1 Fast-reasoning","organization":"xai","updatedAt":"2026-11-09"},"quality":{"coveragePercent":100,"benchmarkCount":17,"categoryCount":8,"confidence":"medium","state":"available"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"mimo-v2-omni","providerId":"xiaomi","slug":"mimo-v2-omni","name":"MiMo-V2-Omni","provider":"Xiaomi","description":"MiMo-V2-Omni is a reasoning model from Xiaomi. 10 benchmarks count toward its index, in 6 categories.","logoUrl":null,"releaseDate":"2026-03-18","contextWindowTokens":262000,"modalities":["text"],"openWeights":false,"details":null,"reasoning":true,"overallScore":47.18641278221573,"overallScoreRange":{"low":40.573009805723466,"high":53.799815758708},"badges":[],"capabilityScores":{"agentic":46.884071804227304,"coding":56.3972343476918,"reasoning":48.866338269618446,"multimodal":46.12153235966533,"knowledge":44.907767719551096,"multilingual":null,"instruction":39.01994728544331,"math":null},"capabilityCounts":{"agentic":2,"knowledge":3,"reasoning":2,"coding":1,"multimodal":1,"instruction":1},"benchmarks":[{"benchmarkId":"tau2Bench","benchmarkName":"τ²-Bench Tool-Agent-User Evaluation","category":"agentic","score":91.2,"unit":"percent","family":"tau2-bench","index":63.307307952589326,"aboveFit":false,"run":null,"source":"Victor Barres, Honghua Dong, Soham Ray, Xujie Si, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2506.07982","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":82.8,"unit":"percent","family":"gpqa-diamond","index":53.15099822120773,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":75,"unit":"percent","family":"lcr","index":58.75408887809522,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"sweVerified","benchmarkName":"Software Engineering Benchmark Verified","category":"coding","score":74.8,"unit":"percent","family":"swe-bench-verified","index":56.3972343476918,"aboveFit":false,"run":null,"source":"Carlos E. Jimenez, John Yang, Alexander Wettig, Shunyu Yao, Kexin Pei, Ofir Press, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2310.06770","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaMmmuPro","benchmarkName":"Artificial Analysis MMMU-Pro","category":"multimodal","score":69.9,"unit":"percent","family":"mmmu-pro","index":46.12153235966533,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmmu-pro","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaIfBench","benchmarkName":"Artificial Analysis IFBench","category":"instruction","score":53.5,"unit":"percent","family":"ifbench-family","index":39.01994728544331,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"clawEval","benchmarkName":"Claw-Eval","category":"agentic","score":45.2,"unit":"percent","family":"clawEval","index":30.460835655865274,"aboveFit":false,"run":null,"source":"Bowen Ye, Rang Li, Qibin Yang, Yuanxin Liu, Linli Yao, Hanglong Lv, Zhihui Xie, Chenxin An, Lei Li, Lingpeng Kong, Qi Liu, Zhifang Sui, Tong Yang","sourceUrl":"https://arxiv.org/abs/2604.06132","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":22.1,"unit":"percent","family":"hle","index":46.55941397847193,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"omniscienceAccuracy","benchmarkName":"Artificial Analysis Omniscience Accuracy","category":"knowledge","score":19.3,"unit":"percent","family":"omniscienceAccuracy","index":35.012890958973635,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":1.1,"unit":"percent","family":"critpt","index":38.97858766114168,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":{"elo":1422.645807775506,"eloUpper":1428.1993132731545,"eloLower":1417.0923022778575,"votes":20411,"rank":110,"rankUpper":null,"rankLower":null,"arenaName":"MiMo-V2-Omni","organization":"xiaomi","updatedAt":"2026-11-09"},"quality":{"coveragePercent":85,"benchmarkCount":10,"categoryCount":6,"confidence":"low","state":"available"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"claude-4-opus-thinking","providerId":"anthropic","slug":"claude-4-opus-thinking","name":"Claude 4 Opus Thinking","provider":"Anthropic","description":"Claude 4 Opus Thinking is a reasoning model from Anthropic in the Claude 4 Opus family. 10 benchmarks count toward its index, in 6 categories.","logoUrl":null,"releaseDate":"2025-05-22","contextWindowTokens":200000,"modalities":["text"],"openWeights":false,"details":null,"reasoning":true,"overallScore":47.03644048041184,"overallScoreRange":{"low":39.42303750391957,"high":54.64984345690411},"badges":[],"capabilityScores":{"agentic":51.49570656038475,"coding":null,"reasoning":54.843429882256956,"multimodal":44.8560716423758,"knowledge":47.09421619097415,"multilingual":null,"instruction":39.20684940840921,"math":43.93795530502886},"capabilityCounts":{"math":2,"knowledge":4,"multimodal":1,"agentic":1,"reasoning":1,"instruction":1},"benchmarks":[{"benchmarkId":"aaMath500","benchmarkName":"Artificial Analysis MATH-500","category":"math","score":98.2,"unit":"percent","family":"math-500","index":47.90043971699151,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/math-500","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmmlu","benchmarkName":"MMMLU","category":"knowledge","score":88.8,"unit":"percent","family":"mmmlu","index":46.96539513969343,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaMmluPro","benchmarkName":"Artificial Analysis MMLU-Pro","category":"knowledge","score":87.3,"unit":"percent","family":"mmlu-pro","index":54.46827888555881,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmlu-pro","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gpqa","benchmarkName":"Graduate-Level Google-Proof Q&A","category":"knowledge","score":79.6,"unit":"percent","family":"gpqa-diamond","index":50.21604566146441,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":79.6,"unit":"percent","family":"gpqa-diamond","index":50.21604566146441,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmmu","benchmarkName":"Massive Multi-discipline Multimodal Understanding","category":"multimodal","score":76.5,"unit":"percent","family":"mmmu","index":44.8560716423758,"aboveFit":false,"run":null,"source":"MMMU authors","sourceUrl":"https://arxiv.org/abs/2401.05508","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aime2025","benchmarkName":"American Invitational Mathematics Examination 2025","category":"math","score":75.5,"unit":"percent","family":"aime2025","index":39.97547089306621,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"tau2Bench","benchmarkName":"τ²-Bench Tool-Agent-User Evaluation","category":"agentic","score":73.4,"unit":"percent","family":"tau2-bench","index":51.49570656038475,"aboveFit":false,"run":null,"source":"Victor Barres, Honghua Dong, Soham Ray, Xujie Si, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2506.07982","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":69.3,"unit":"percent","family":"lcr","index":54.843429882256956,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaIfBench","benchmarkName":"Artificial Analysis IFBench","category":"instruction","score":53.7,"unit":"percent","family":"ifbench-family","index":39.20684940840921,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":12.3,"unit":"percent","family":"hle","index":36.72714507717995,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":75,"benchmarkCount":10,"categoryCount":6,"confidence":"low","state":"available"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"claude-4-1-opus-thinking","providerId":"anthropic","slug":"claude-4-1-opus-thinking","name":"Claude 4.1 Opus Thinking","provider":"Anthropic","description":"Claude 4.1 Opus Thinking is a reasoning model from Anthropic in the Claude 4.1 Opus family. 8 benchmarks count toward its index, in 5 categories.","logoUrl":null,"releaseDate":"2025-08-05","contextWindowTokens":200000,"modalities":["text"],"openWeights":false,"details":null,"reasoning":true,"overallScore":47.029733796151426,"overallScoreRange":{"low":37.81277547025819,"high":56.246692122044664},"badges":[],"capabilityScores":{"agentic":50.16856033654154,"coding":null,"reasoning":48.14464270822689,"multimodal":43.518707829427,"knowledge":47.89545628758623,"multilingual":null,"instruction":40.79551745361937,"math":null},"capabilityCounts":{"knowledge":3,"reasoning":2,"agentic":1,"multimodal":1,"instruction":1},"benchmarks":[{"benchmarkId":"aaMmluPro","benchmarkName":"Artificial Analysis MMLU-Pro","category":"knowledge","score":88,"unit":"percent","family":"mmlu-pro","index":55.35019509771263,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmlu-pro","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":80.9,"unit":"percent","family":"gpqa-diamond","index":51.40837013886014,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":76,"unit":"percent","family":"lcr","index":59.44016940368088,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"tau2Bench","benchmarkName":"τ²-Bench Tool-Agent-User Evaluation","category":"agentic","score":71.4,"unit":"percent","family":"tau2-bench","index":50.16856033654154,"aboveFit":false,"run":null,"source":"Victor Barres, Honghua Dong, Soham Ray, Xujie Si, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2506.07982","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaMmmuPro","benchmarkName":"Artificial Analysis MMMU-Pro","category":"multimodal","score":67.9,"unit":"percent","family":"mmmu-pro","index":43.518707829427,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmmu-pro","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaIfBench","benchmarkName":"Artificial Analysis IFBench","category":"instruction","score":55.4,"unit":"percent","family":"ifbench-family","index":40.79551745361937,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":12.5,"unit":"percent","family":"hle","index":36.927803626185906,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":0,"unit":"percent","family":"critpt","index":36.8491160127729,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":29,"timeToFirstToken":15,"source":"BenchLM","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":65,"benchmarkCount":8,"categoryCount":5,"confidence":"low","state":"available"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"claudesonnet420250514thinking","providerId":"anthropic","slug":"anthropic-claude-sonnet-4-thinking","name":"Claude Sonnet 4 Thinking","provider":"Anthropic","description":"Claude Sonnet 4 Thinking is a model from Anthropic. 8 benchmarks count toward its index, in 5 categories.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":46.88345995104146,"overallScoreRange":{"low":38.66650162514822,"high":55.1004182769347},"badges":[],"capabilityScores":{"agentic":50.78364063113496,"coding":41.767084828732294,"reasoning":null,"multimodal":52.66503322868448,"knowledge":48.06882070066158,"multilingual":null,"instruction":null,"math":46.0390044028224},"capabilityCounts":{"math":2,"knowledge":2,"multimodal":1,"coding":2,"agentic":1},"benchmarks":[{"benchmarkId":"vals-ai:math500:overall","benchmarkName":"MATH 500","category":"math","domain":null,"eligible":true,"ranked":true,"score":93.8,"unit":"percent","family":"math-500","index":43.25070358230003,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/math500","measuredAt":"2026-01-09","stderr":1.078,"costPerTest":0.068378,"latency":63.474},{"benchmarkId":"vals-ai:medqa:overall","benchmarkName":"MedQA","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":92.708,"unit":"percent","family":"vals-ai:medqa:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medqa","measuredAt":"2026-04-16","stderr":0.237,"costPerTest":0.025234,"latency":26.989},{"benchmarkId":"vals-ai:mgsm:overall","benchmarkName":"MGSM","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":90.873,"unit":"percent","family":"vals-ai:mgsm:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mgsm","measuredAt":"2026-01-09","stderr":1.728,"costPerTest":0.014593,"latency":14.087},{"benchmarkId":"vals-ai:mmlu_pro:overall","benchmarkName":"MMLU Pro","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":83.865,"unit":"percent","family":"mmlu-pro","index":50.140590044489755,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","measuredAt":"2026-09-01","stderr":0.361,"costPerTest":0.045537,"latency":47.087},{"benchmarkId":"vals-ai:legal_bench:overall","benchmarkName":"LegalBench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":82.065,"unit":"percent","family":"vals-ai:legal_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","measuredAt":"2026-10-13","stderr":0.455,"costPerTest":null,"latency":10.605},{"benchmarkId":"vals-ai:aime:overall","benchmarkName":"AIME","category":"math","domain":null,"eligible":true,"ranked":true,"score":76.25,"unit":"percent","family":"vals-ai:aime:overall","index":48.82730522334478,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/aime","measuredAt":"2026-04-16","stderr":1.435,"costPerTest":0.295026,"latency":271.788},{"benchmarkId":"vals-ai:gpqa:overall","benchmarkName":"GPQA Diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":75,"unit":"percent","family":"gpqa-diamond","index":45.99705135683341,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","measuredAt":"2026-09-01","stderr":2.191,"costPerTest":0.116706,"latency":118.458},{"benchmarkId":"vals-ai:mmmu:overall","benchmarkName":"MMMU Pro","category":"multimodal","domain":null,"eligible":true,"ranked":true,"score":74.928,"unit":"percent","family":"mmmu-pro","index":52.66503322868448,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmmu","measuredAt":"2026-09-01","stderr":1.042,"costPerTest":0.050728,"latency":48.854},{"benchmarkId":"vals-ai:tax_eval_v2:overall","benchmarkName":"TaxEval v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":71.995,"unit":"percent","family":"vals-ai:tax_eval_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","measuredAt":"2026-09-01","stderr":0.878,"costPerTest":0.029421,"latency":32.385},{"benchmarkId":"vals-ai:medscribe:overall","benchmarkName":"MedScribe","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":69.353,"unit":"percent","family":"vals-ai:medscribe:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medscribe","measuredAt":"2026-10-29","stderr":2.212,"costPerTest":0.053443,"latency":39.572},{"benchmarkId":"vals-ai:lcb:overall","benchmarkName":"LiveCodeBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":62.392,"unit":"percent","family":"livecodebench","index":41.112981263437675,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","measuredAt":"2026-09-01","stderr":1.125,"costPerTest":0.042318,"latency":42.611},{"benchmarkId":"vals-ai:corp_fin_v2:overall","benchmarkName":"CorpFin v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":61.228,"unit":"percent","family":"vals-ai:corp_fin_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","measuredAt":"2026-08-12","stderr":0.959,"costPerTest":0.236746,"latency":168.523},{"benchmarkId":"vals-ai:mortgage_tax:overall","benchmarkName":"MortgageTax","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":49.88,"unit":"percent","family":"vals-ai:mortgage_tax:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mortgage_tax","measuredAt":"2026-09-01","stderr":0.985,"costPerTest":0.052253,"latency":46.316},{"benchmarkId":"vals-ai:terminal-bench:overall","benchmarkName":"Terminal-Bench 1.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":45,"unit":"percent","family":"vals-ai:terminal-bench:overall","index":50.78364063113496,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench","measuredAt":"2026-01-12","stderr":5.562,"costPerTest":2.017755,"latency":717.326},{"benchmarkId":"vals-ai:medcode:overall","benchmarkName":"MedCode","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":34.959,"unit":"percent","family":"vals-ai:medcode:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medcode","measuredAt":"2026-10-29","stderr":1.939,"costPerTest":0.069896,"latency":39.799},{"benchmarkId":"vals-ai:ioi-v1:overall","benchmarkName":"IOI v1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":4.584,"unit":"percent","family":"vals-ai:ioi-v1:overall","index":42.42118839402691,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/ioi-v1","measuredAt":"2026-08-09","stderr":1.139,"costPerTest":1.329832,"latency":2216.287}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":75,"benchmarkCount":8,"categoryCount":5,"confidence":"low","state":"available"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"agents-a1-4b","providerId":"internscience","slug":"agents-a1-4b","name":"Agents-A1-4B","provider":"InternScience","description":"Agents-A1-4B is a reasoning model from InternScience in the Agents-A1 family. 8 benchmarks count toward its index, in 4 categories.","logoUrl":null,"releaseDate":"2026-07-13","contextWindowTokens":262000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":true,"overallScore":46.755441247847436,"overallScoreRange":{"low":37.0384829219542,"high":56.472399573740674},"badges":[],"capabilityScores":{"agentic":54.08608644073267,"coding":33.890566725284806,"reasoning":42.12796153342534,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":53.84990292117418,"math":null},"capabilityCounts":{"instruction":2,"agentic":3,"coding":2,"reasoning":1},"benchmarks":[{"benchmarkId":"gaia","benchmarkName":"General AI Assistants","category":"agentic","score":95.1,"unit":"percent","family":"gaia","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"ifeval","benchmarkName":"Instruction-Following Eval","category":"instruction","score":94.8,"unit":"percent","family":"ifeval","index":54.1014929655648,"aboveFit":false,"run":null,"source":"Jeffrey Zhou, Tianjian Lu, Swaroop Mishra, Siddhartha Brahma, Sujoy Basu, Yi Luan, Denny Zhou, Le Hou","sourceUrl":"https://arxiv.org/abs/2311.07911","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"tau2Bench","benchmarkName":"τ²-Bench Tool-Agent-User Evaluation","category":"agentic","score":78.2,"unit":"percent","family":"tau2-bench","index":54.68085749760845,"aboveFit":false,"run":null,"source":"Victor Barres, Honghua Dong, Soham Ray, Xujie Si, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2506.07982","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"ifBench","benchmarkName":"Instruction Following Benchmark","category":"instruction","score":69.1,"unit":"percent","family":"ifbench-family","index":53.59831287678356,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"browseComp","benchmarkName":"BrowseComp","category":"agentic","score":66.8,"unit":"percent","family":"browseComp","index":52.68231709135213,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"liveCodeBenchV6","benchmarkName":"LiveCodeBench v6","category":"coding","score":59.6,"unit":"percent","family":"liveCodeBenchV6","index":33.35207707418223,"aboveFit":false,"run":null,"source":"LiveCodeBench maintainers","sourceUrl":"https://github.com/LiveCodeBench/LiveCodeBench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"longBenchV2","benchmarkName":"LongBench v2","category":"reasoning","score":52.1,"unit":"percent","family":"longBenchV2","index":42.12796153342534,"aboveFit":false,"run":null,"source":"LongBench v2 authors","sourceUrl":"https://arxiv.org/abs/2412.15204","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vitaBench","benchmarkName":"VITA-Bench","category":"agentic","score":40.3,"unit":"percent","family":"vitaBench","index":54.89508473323744,"aboveFit":false,"run":null,"source":"Meituan LongCat Team","sourceUrl":"https://vitabench.github.io/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"frontierScienceResearch","benchmarkName":"FrontierScience Research","category":"knowledge","score":33.3,"unit":"percent","family":"frontierScienceResearch","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"sciCode","benchmarkName":"Scientific Code Benchmark","category":"coding","score":29.6,"unit":"percent","family":"scicode-family","index":34.42905637638738,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mleBenchLite","benchmarkName":"MLE-Bench Lite","category":"agentic","score":22.7,"unit":"percent","family":"mleBenchLite","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":60,"benchmarkCount":8,"categoryCount":4,"confidence":"low","state":"available"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gemini-2-5-pro","providerId":"google","slug":"gemini-2-5-pro","name":"Gemini 2.5 Pro","provider":"Google","description":"Gemini 2.5 Pro is a standard model from Google. 20 benchmarks count toward its index, in 7 categories.","logoUrl":null,"releaseDate":"2025-03-01","contextWindowTokens":1048576,"modalities":["text","image","file","audio","video"],"openWeights":false,"details":{"apiId":"google/gemini-2.5-pro","maxOutputTokens":65536,"knowledgeCutoff":"2025-01-31","retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.125,"weightsUrl":null},"reasoning":false,"overallScore":46.58298120334674,"overallScoreRange":{"low":42.46725928372958,"high":50.6987031229639},"badges":[],"capabilityScores":{"agentic":40.35107465358065,"coding":48.8194550802937,"reasoning":48.26000918038564,"multimodal":52.62859368526116,"knowledge":53.032365780585245,"multilingual":null,"instruction":34.5342963342617,"math":45.963142560394154},"capabilityCounts":{"math":2,"knowledge":4,"coding":5,"multimodal":1,"reasoning":2,"agentic":5,"instruction":1},"benchmarks":[{"benchmarkId":"aaMath500","benchmarkName":"Artificial Analysis MATH-500","category":"math","score":96.7,"unit":"percent","family":"math-500","index":46.31530239834669,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/math-500","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aime2024","benchmarkName":"American Invitational Mathematics Examination 2024","category":"math","score":92,"unit":"percent","family":"aime2024","index":45.61098272244162,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaGlobalMmluLite","benchmarkName":"Artificial Analysis Global-MMLU-Lite","category":"multilingual","score":90.3,"unit":"percent","family":"aaGlobalMmluLite","index":null,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/global-mmlu-lite","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaMmluPro","benchmarkName":"Artificial Analysis MMLU-Pro","category":"knowledge","score":86.2,"unit":"percent","family":"mmlu-pro","index":53.08241055217426,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmlu-pro","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":84.4,"unit":"percent","family":"gpqa-diamond","index":54.61847450107939,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gpqa","benchmarkName":"Graduate-Level Google-Proof Q&A","category":"knowledge","score":83,"unit":"percent","family":"gpqa-diamond","index":53.33443275619169,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaLiveCodeBench","benchmarkName":"Artificial Analysis LiveCodeBench","category":"coding","score":80.1,"unit":"percent","family":"livecodebench","index":55.80603894661818,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/livecodebench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaMmmuPro","benchmarkName":"Artificial Analysis MMMU-Pro","category":"multimodal","score":74.9,"unit":"percent","family":"mmmu-pro","index":52.62859368526116,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmmu-pro","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:medscribe:overall","benchmarkName":"MedScribe","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":73.552,"unit":"percent","family":"vals-ai:medscribe:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medscribe","measuredAt":"2026-10-29","stderr":1.91,"costPerTest":0.046379,"latency":35.909},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":69,"unit":"percent","family":"lcr","index":54.637605724581256,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:mortgage_tax:overall","benchmarkName":"MortgageTax","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":68.918,"unit":"percent","family":"vals-ai:mortgage_tax:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mortgage_tax","measuredAt":"2026-09-01","stderr":0.907,"costPerTest":0.004794,"latency":24.008},{"benchmarkId":"vals-ai:case_law_v2:overall","benchmarkName":"CaseLaw v2","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":63.88,"unit":"percent","family":"vals-ai:case_law_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/case_law_v2","measuredAt":"2026-05-04","stderr":0.597,"costPerTest":0.257186,"latency":44.02},{"benchmarkId":"sweVerified","benchmarkName":"Software Engineering Benchmark Verified","category":"coding","score":63.8,"unit":"percent","family":"swe-bench-verified","index":48.28410099207119,"aboveFit":false,"run":null,"source":"Carlos E. Jimenez, John Yang, Alexander Wettig, Shunyu Yao, Kexin Pei, Ofir Press, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2310.06770","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:corp_fin_v2:overall","benchmarkName":"CorpFin v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":60.8,"unit":"percent","family":"vals-ai:corp_fin_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","measuredAt":"2026-08-12","stderr":0.961,"costPerTest":0.09535,"latency":27.425},{"benchmarkId":"vals-ai:swebench:overall","benchmarkName":"SWE-bench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":54.4,"unit":"percent","family":"swe-bench-verified","index":41.35105976090449,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/swebench","measuredAt":"2026-09-01","stderr":2.23,"costPerTest":0.351578,"latency":207.791},{"benchmarkId":"tau2Bench","benchmarkName":"τ²-Bench Tool-Agent-User Evaluation","category":"agentic","score":54.1,"unit":"percent","family":"tau2-bench","index":38.68874550029775,"aboveFit":false,"run":null,"source":"Victor Barres, Honghua Dong, Soham Ray, Xujie Si, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2506.07982","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"swe-bench:swe-bench-verified:overall","benchmarkName":"SWE-bench Verified","category":"coding","domain":null,"eligible":true,"ranked":true,"score":53.6,"unit":"percent","family":"swe-bench-verified","index":40.76101369867754,"aboveFit":false,"run":"mini-SWE-agent","source":"SWE-bench team","sourceUrl":"https://www.swebench.com/","measuredAt":"2026-02-26","stderr":null,"costPerTest":0.28837209562500005,"latency":null},{"benchmarkId":"swebench-experiments:swe-bench-verified:overall","benchmarkName":"SWE-bench Verified","category":"coding","domain":null,"eligible":true,"ranked":true,"score":53.6,"unit":"percent","family":"swe-bench-verified","index":40.76101369867754,"aboveFit":false,"run":"mini-SWE-agent","source":"SWE-bench team","sourceUrl":"https://github.com/SWE-bench/experiments/tree/main/evaluation/verified","measuredAt":"2026-09-01","stderr":null,"costPerTest":0.28837209562500005,"latency":null},{"benchmarkId":"vals-ai:medcode:overall","benchmarkName":"MedCode","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":50.59,"unit":"percent","family":"vals-ai:medcode:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medcode","measuredAt":"2026-10-29","stderr":2.113,"costPerTest":0.015389,"latency":27.415},{"benchmarkId":"aaIfBench","benchmarkName":"Artificial Analysis IFBench","category":"instruction","score":48.7,"unit":"percent","family":"ifbench-family","index":34.5342963342617,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaSciCode","benchmarkName":"Artificial Analysis SciCode","category":"coding","score":46.3,"unit":"percent","family":"scicode-family","index":56.20069852151341,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/scicode","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gertLabs","benchmarkName":"Gert Labs Composite Game Benchmark","category":"agentic","score":42.01,"unit":"percent","family":"gertLabs","index":49.562959091085496,"aboveFit":false,"run":null,"source":"Gert Labs","sourceUrl":"https://gertlabs.com/rankings","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:finance_agent:overall","benchmarkName":"Finance Agent v1.1","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":41.589,"unit":"percent","family":"vals-ai:finance_agent:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/finance_agent","measuredAt":"2026-06-04","stderr":2.783,"costPerTest":0.34118,"latency":1839.558},{"benchmarkId":"vals-ai:terminal-bench:overall","benchmarkName":"Terminal-Bench 1.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":41.25,"unit":"percent","family":"vals-ai:terminal-bench:overall","index":48.06982840515033,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench","measuredAt":"2026-01-12","stderr":5.504,"costPerTest":0.664031,"latency":352.476},{"benchmarkId":"omniscienceAccuracy","benchmarkName":"Artificial Analysis Omniscience Accuracy","category":"knowledge","score":39.1,"unit":"percent","family":"omniscienceAccuracy","index":59.96595944335246,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:terminal-bench-2:overall","benchmarkName":"Terminal-Bench 2.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":30.337,"unit":"percent","family":"vals-ai:terminal-bench-2:overall","index":44.817513677231766,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2","measuredAt":"2026-06-04","stderr":4.901,"costPerTest":0.472003,"latency":796.526},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":22.5,"unit":"percent","family":"hle","index":46.96073107648385,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hle","benchmarkName":"Humanity's Last Exam","category":"knowledge","score":18.8,"unit":"percent","family":"hle","index":43.24854791987361,"aboveFit":false,"run":null,"source":"Center for AI Safety, Scale AI, and thousands of expert contributors","sourceUrl":"https://lastexam.ai/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:ioi-v1:overall","benchmarkName":"IOI v1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":17.084,"unit":"percent","family":"vals-ai:ioi-v1:overall","index":48.64065232811309,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/ioi-v1","measuredAt":"2026-08-09","stderr":6.17,"costPerTest":0.615461,"latency":2964.799},{"benchmarkId":"tau2-bench:banking-knowledge:overall","benchmarkName":"τ²-bench Banking","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":13.66,"unit":"percent","family":"tau2-bench","index":11.853848854188001,"aboveFit":false,"run":"high effort · Sierra","source":"Sierra Research","sourceUrl":"https://github.com/sierra-research/tau2-bench","measuredAt":"2026-10-20","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":2.6,"unit":"percent","family":"critpt","index":41.88241263619001,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:vibe-code:overall","benchmarkName":"Vibe Code Bench v1.1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":0.4,"unit":"percent","family":"vals-ai:vibe-code:overall","index":42.39384887543281,"aboveFit":false,"run":"OpenHands","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","measuredAt":"2026-11-09","stderr":0.4,"costPerTest":1.223831,"latency":2097.144},{"benchmarkId":"gdpvalAaNormalized","benchmarkName":"GDPval-AA normalized","category":"agentic","score":0,"unit":"percent","family":"gdpvalAaNormalized","index":34.03377491719279,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":1.25,"outputPerMillion":10,"batch":{"inputPerMillion":0.625,"outputPerMillion":5},"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":91,"timeToFirstToken":2.917,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":{"elo":1457.8799629992895,"eloUpper":1460.31794880279,"eloLower":1455.4419771957891,"votes":124789,"rank":42,"rankUpper":null,"rankLower":null,"arenaName":"Gemini 2.5 Pro","organization":"google","updatedAt":"2026-11-09"},"quality":{"coveragePercent":95,"benchmarkCount":20,"categoryCount":7,"confidence":"medium","state":"available"},"hosts":[{"host":"databricks","name":"Databricks","kind":"cloud","url":"https://docs.databricks.com/aws/en/machine-learning/foundation-models/","inputPerMillion":1.25,"outputPerMillion":10,"cacheReadPerMillion":0.125,"contextTokens":1048576,"maxOutputTokens":65536,"deprecated":true},{"host":"google-ai-studio","name":"Google AI Studio","kind":"maker","url":"https://ai.google.dev/gemini-api/docs/models","inputPerMillion":1.25,"outputPerMillion":10,"cacheReadPerMillion":0.125,"contextTokens":1048576,"maxOutputTokens":65536,"deprecated":false},{"host":"google-vertex","name":"Google Vertex AI","kind":"maker","url":"https://cloud.google.com/vertex-ai/generative-ai/docs/models","inputPerMillion":1.25,"outputPerMillion":10,"cacheReadPerMillion":0.125,"contextTokens":1048576,"maxOutputTokens":65536,"deprecated":true},{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":1.25,"outputPerMillion":10,"cacheReadPerMillion":0.125,"contextTokens":1048576,"maxOutputTokens":65536,"deprecated":true},{"host":"sap-ai-core","name":"SAP AI Core","kind":"cloud","url":"https://help.sap.com/docs/sap-ai-core","inputPerMillion":1.25,"outputPerMillion":10,"cacheReadPerMillion":0.125,"contextTokens":1048576,"maxOutputTokens":65536,"deprecated":true}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:swe-bench-team","name":"SWE-bench team","url":"https://www.swebench.com","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:sierra-research","name":"Sierra Research","url":"https://github.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gemini25flashpreview","providerId":"google","slug":"google-gemini-2-5-flash-preview","name":"Gemini 2.5 Flash (Preview)","provider":"Google","description":"Gemini 2.5 Flash (Preview) is a model from Google. 8 benchmarks count toward its index, in 6 categories.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":46.56380138996486,"overallScoreRange":{"low":39.84684306407162,"high":53.280759715858096},"badges":[],"capabilityScores":{"agentic":40.83299580252468,"coding":51.62663237613139,"reasoning":41.137408109609844,"multimodal":58.58906185950693,"knowledge":50.96713930529786,"multilingual":null,"instruction":null,"math":37.263948493762456},"capabilityCounts":{"knowledge":2,"multimodal":1,"coding":1,"math":1,"reasoning":2,"agentic":1},"benchmarks":[{"benchmarkId":"vals-ai:medqa:overall","benchmarkName":"MedQA","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":91.433,"unit":"percent","family":"vals-ai:medqa:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medqa","measuredAt":"2026-04-16","stderr":0.255,"costPerTest":0.001272,"latency":8.534},{"benchmarkId":"vals-ai:mgsm:overall","benchmarkName":"MGSM","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":89.854,"unit":"percent","family":"vals-ai:mgsm:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mgsm","measuredAt":"2026-01-09","stderr":1.8,"costPerTest":0.001015,"latency":3.413},{"benchmarkId":"vals-ai:mmlu_pro:overall","benchmarkName":"MMLU Pro","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":83.686,"unit":"percent","family":"mmlu-pro","index":49.915071470239006,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","measuredAt":"2026-09-01","stderr":0.362,"costPerTest":0.001454,"latency":8.525},{"benchmarkId":"vals-ai:legal_bench:overall","benchmarkName":"LegalBench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":82.486,"unit":"percent","family":"vals-ai:legal_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","measuredAt":"2026-10-13","stderr":0.431,"costPerTest":null,"latency":1.154},{"benchmarkId":"vals-ai:gpqa:overall","benchmarkName":"GPQA Diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":81.566,"unit":"percent","family":"gpqa-diamond","index":52.01920714035672,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","measuredAt":"2026-09-01","stderr":1.996,"costPerTest":0.002894,"latency":19.483},{"benchmarkId":"vals-ai:mmmu:overall","benchmarkName":"MMMU Pro","category":"multimodal","domain":null,"eligible":true,"ranked":true,"score":79.48,"unit":"percent","family":"mmmu-pro","index":58.58906185950693,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmmu","measuredAt":"2026-09-01","stderr":0.971,"costPerTest":0.00187,"latency":25.643},{"benchmarkId":"vals-ai:medscribe:overall","benchmarkName":"MedScribe","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":77.946,"unit":"percent","family":"vals-ai:medscribe:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medscribe","measuredAt":"2026-10-29","stderr":1.923,"costPerTest":0.014385,"latency":22.893},{"benchmarkId":"vals-ai:lcb:overall","benchmarkName":"LiveCodeBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":75.063,"unit":"percent","family":"livecodebench","index":51.62663237613139,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","measuredAt":"2026-09-01","stderr":1.13,"costPerTest":0.002367,"latency":46.785},{"benchmarkId":"vals-ai:tax_eval_v2:overall","benchmarkName":"TaxEval v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":72.731,"unit":"percent","family":"vals-ai:tax_eval_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","measuredAt":"2026-09-01","stderr":0.869,"costPerTest":0.003098,"latency":6.267},{"benchmarkId":"vals-ai:mortgage_tax:overall","benchmarkName":"MortgageTax","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":62.599,"unit":"percent","family":"vals-ai:mortgage_tax:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mortgage_tax","measuredAt":"2026-09-01","stderr":0.956,"costPerTest":0.001152,"latency":7.274},{"benchmarkId":"vals-ai:corp_fin_v2:overall","benchmarkName":"CorpFin v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":58.974,"unit":"percent","family":"vals-ai:corp_fin_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","measuredAt":"2026-08-12","stderr":0.969,"costPerTest":0.026123,"latency":51.895},{"benchmarkId":"vals-ai:aime:overall","benchmarkName":"AIME","category":"math","domain":null,"eligible":true,"ranked":true,"score":49.792,"unit":"percent","family":"vals-ai:aime:overall","index":37.263948493762456,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/aime","measuredAt":"2026-04-16","stderr":2.215,"costPerTest":0.004524,"latency":58.378},{"benchmarkId":"vals-ai:medcode:overall","benchmarkName":"MedCode","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":40.538,"unit":"percent","family":"vals-ai:medcode:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medcode","measuredAt":"2026-10-29","stderr":1.932,"costPerTest":0.003692,"latency":12.702},{"benchmarkId":"arc-prize:v1-public-eval:overall","benchmarkName":"ARC-AGI-1 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":36.3,"unit":"percent","family":"arc-prize:v1-public-eval:overall","index":null,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.033,"latency":null},{"benchmarkId":"arc-prize:v1-semi-private:overall","benchmarkName":"ARC-AGI-1 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":33.33,"unit":"percent","family":"arc-prize:v1-semi-private:overall","index":42.310363172327435,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.0371,"latency":null},{"benchmarkId":"vals-ai:terminal-bench:overall","benchmarkName":"Terminal-Bench 1.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":31.25,"unit":"percent","family":"vals-ai:terminal-bench:overall","index":40.83299580252468,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench","measuredAt":"2026-01-12","stderr":5.182,"costPerTest":0.134487,"latency":79.464},{"benchmarkId":"arc-prize:v2-public-eval:overall","benchmarkName":"ARC-AGI-2 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":1.94,"unit":"percent","family":"arc-prize:v2-public-eval:overall","index":null,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.0586,"latency":null},{"benchmarkId":"arc-prize:v2-semi-private:overall","benchmarkName":"ARC-AGI-2 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":1.69,"unit":"percent","family":"arc-prize:v2-semi-private:overall","index":39.964453046892245,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.057,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":90,"benchmarkCount":8,"categoryCount":6,"confidence":"low","state":"available"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:arc-prize-foundation","name":"ARC Prize Foundation","url":"https://arcprize.org","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"nemotron-3-5-bolt-74b-a9b","providerId":"nvidia","slug":"nemotron-3-5-bolt-74b-a9b","name":"Nemotron 3.5 Bolt 74B-A9B","provider":"NVIDIA","description":"Nemotron 3.5 Bolt 74B-A9B is a reasoning model from NVIDIA. 11 benchmarks count toward its index, in 4 categories.","logoUrl":null,"releaseDate":"2026-10-15","contextWindowTokens":256000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":true,"overallScore":46.36896361964841,"overallScoreRange":{"low":38.49352517782597,"high":54.24440206147085},"badges":[],"capabilityScores":{"agentic":42.264517238263565,"coding":47.59633695517618,"reasoning":50.14563818353467,"multimodal":null,"knowledge":52.974648985112445,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"agentic":6,"knowledge":2,"reasoning":2,"coding":1},"benchmarks":[{"benchmarkId":"pinchBench","benchmarkName":"PinchBench","category":"agentic","score":87.82,"unit":"percent","family":"pinchBench","index":47.58048901598903,"aboveFit":false,"run":null,"source":"Kilo Code","sourceUrl":"https://pinchbench.com/about","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gpqaDiamond","benchmarkName":"GPQA Diamond","category":"knowledge","score":86.43,"unit":"percent","family":"gpqa-diamond","index":56.480335031166554,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":69.7,"unit":"percent","family":"lcr","index":55.11786209249122,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"browseComp","benchmarkName":"BrowseComp","category":"agentic","score":66.75,"unit":"percent","family":"browseComp","index":52.64281610105217,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mcpAtlas","benchmarkName":"MCP Atlas","category":"agentic","score":52.2,"unit":"percent","family":"mcpAtlas","index":46.213533426556005,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"terminalBench21","benchmarkName":"Terminal-Bench 2.1","category":"agentic","score":50.98,"unit":"percent","family":"terminal-bench-2-1","index":50.957047688187416,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"webArena","benchmarkName":"WebArena Web Agent Benchmark","category":"agentic","score":50.86,"unit":"percent","family":"webArena","index":null,"aboveFit":false,"run":null,"source":"Shuyan Zhou, Frank F. Xu, Hao Zhu, Xuhui Zhou, Robert Lo, Abishek Sridhar, Xianyi Cheng, Tianyue Ou, Yonatan Bisk, Daniel Fried, Uri Alon, Graham Neubig","sourceUrl":"https://arxiv.org/abs/2307.13854","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaSciCode","benchmarkName":"Artificial Analysis SciCode","category":"coding","score":39.7,"unit":"percent","family":"scicode-family","index":47.59633695517618,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/scicode","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hleNoTools","benchmarkName":"Humanity's Last Exam without tools","category":"knowledge","score":25.3,"unit":"percent","family":"hle","index":49.76995076256728,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":24.7,"unit":"percent","family":"hle","index":49.167975115549396,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gdpvalAaNormalized","benchmarkName":"GDPval-AA normalized","category":"agentic","score":24,"unit":"percent","family":"gdpvalAaNormalized","index":51.193217197796756,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"tau3Bench","benchmarkName":"τ³-Bench Tool-Agent-User Evaluation","category":"agentic","score":17.01,"unit":"percent","family":"tau3Bench","index":5,"aboveFit":false,"run":null,"source":"Sierra Research","sourceUrl":"https://github.com/sierra-research/tau2-bench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":4.3,"unit":"percent","family":"critpt","index":45.17341427457812,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":70,"benchmarkCount":11,"categoryCount":4,"confidence":"low","state":"available"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"mercury-2-5","providerId":"inception","slug":"mercury-2-5","name":"Mercury 2.5","provider":"Inception","description":"Mercury 2.5 is a reasoning model from Inception. 17 benchmarks count toward its index, in 6 categories.","logoUrl":null,"releaseDate":"2026-09-08","contextWindowTokens":260000,"modalities":["text"],"openWeights":false,"details":{"apiId":"inception/mercury-2.5","maxOutputTokens":65536,"knowledgeCutoff":null,"retiresAt":"2026-11-08","tools":true,"structuredOutput":true,"cacheReadPerMillion":0.004,"weightsUrl":null},"reasoning":true,"overallScore":46.24094505813381,"overallScoreRange":{"low":40.81914400129631,"high":51.66274611497131},"badges":[],"capabilityScores":{"agentic":49.27177608534114,"coding":44.498940797032425,"reasoning":45.40032060588425,"multimodal":null,"knowledge":41.435607625703994,"multilingual":null,"instruction":60.980946733936634,"math":48.19235469365171},"capabilityCounts":{"agentic":5,"knowledge":3,"instruction":1,"reasoning":2,"coding":5,"math":1},"benchmarks":[{"benchmarkId":"tau3Bench","benchmarkName":"τ³-Bench Tool-Agent-User Evaluation","category":"agentic","score":96,"unit":"percent","family":"tau3Bench","index":89.90094134594221,"aboveFit":true,"run":null,"source":"Sierra Research","sourceUrl":"https://github.com/sierra-research/tau2-bench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:legal_bench:overall","benchmarkName":"LegalBench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":83.06,"unit":"percent","family":"vals-ai:legal_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","measuredAt":"2026-10-13","stderr":0.482,"costPerTest":0.000787,"latency":3.908},{"benchmarkId":"gpqaDiamond","benchmarkName":"GPQA Diamond","category":"knowledge","score":79,"unit":"percent","family":"gpqa-diamond","index":49.66574205651255,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"ifBench","benchmarkName":"Instruction Following Benchmark","category":"instruction","score":77,"unit":"percent","family":"ifbench-family","index":60.980946733936634,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":68,"unit":"percent","family":"lcr","index":53.9515251989956,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:medscribe:overall","benchmarkName":"MedScribe","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":55.093,"unit":"percent","family":"vals-ai:medscribe:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medscribe","measuredAt":"2026-10-29","stderr":2.095,"costPerTest":0.004476,"latency":9.992},{"benchmarkId":"vals-ai:cyber:overall","benchmarkName":"CyberBench v1.1","category":null,"domain":"Other","eligible":false,"ranked":true,"score":45.476,"unit":"percent","family":"vals-ai:cyber:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/cyber","measuredAt":"2026-11-05","stderr":6.056,"costPerTest":0.12462,"latency":1139.168},{"benchmarkId":"vals-ai:public-benefits-bench:overall","benchmarkName":"Public Benefits Bench v1.1","category":null,"domain":"Public benefits","eligible":false,"ranked":true,"score":45.264,"unit":"percent","family":"vals-ai:public-benefits-bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/public-benefits-bench","measuredAt":"2026-10-12","stderr":1.295,"costPerTest":0.067576,"latency":68.325},{"benchmarkId":"aaSciCode","benchmarkName":"Artificial Analysis SciCode","category":"coding","score":38.5,"unit":"percent","family":"scicode-family","index":46.0319075794785,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/scicode","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"sciCode","benchmarkName":"Scientific Code Benchmark","category":"coding","score":38,"unit":"percent","family":"scicode-family","index":45.38006200627113,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:terminal-bench-2-1:overall","benchmarkName":"Terminal-Bench 2.1","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":34.457,"unit":"percent","family":"terminal-bench-2-1","index":41.29348250100041,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2-1","measuredAt":"2026-10-01","stderr":0.749,"costPerTest":0.214987,"latency":688.923},{"benchmarkId":"deepSearchQa","benchmarkName":"DeepSearchQA","category":"agentic","score":34,"unit":"percent","family":"deepSearchQa","index":26.61466819671455,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:medcode:overall","benchmarkName":"MedCode","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":31.326,"unit":"percent","family":"vals-ai:medcode:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medcode","measuredAt":"2026-10-29","stderr":1.953,"costPerTest":0.004535,"latency":7.93},{"benchmarkId":"omniscienceAccuracy","benchmarkName":"Artificial Analysis Omniscience Accuracy","category":"knowledge","score":22,"unit":"percent","family":"omniscienceAccuracy","index":38.415582115934384,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:fabv2:overall","benchmarkName":"Finance Agent (v2)","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":18.894,"unit":"percent","family":"vals-ai:fabv2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/fabv2","measuredAt":"2026-10-29","stderr":0.216,"costPerTest":0.106441,"latency":92.482},{"benchmarkId":"vals-ai:skillsbench:overall","benchmarkName":"SkillsBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":18.061,"unit":"percent","family":"vals-ai:skillsbench:overall","index":38.58739686834353,"aboveFit":false,"run":"OpenHands","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/skillsbench","measuredAt":"2026-10-29","stderr":3.079,"costPerTest":0.101286,"latency":49.364},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":11.8,"unit":"percent","family":"hle","index":36.22549870466505,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:emb:overall","benchmarkName":"EMB","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":8.789,"unit":"percent","family":"vals-ai:emb:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/emb","measuredAt":"2026-10-20","stderr":1.275,"costPerTest":0.417608,"latency":300.757},{"benchmarkId":"vals-ai:code-migration:overall","benchmarkName":"Code Migration","category":"coding","domain":null,"eligible":true,"ranked":true,"score":4.517,"unit":"percent","family":"vals-ai:code-migration:overall","index":47.58874894788414,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/code-migration","measuredAt":"2026-10-29","stderr":1.048,"costPerTest":0.627223,"latency":974.083},{"benchmarkId":"vals-ai:legal_research:overall","benchmarkName":"Legal Research Bench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":4.327,"unit":"percent","family":"vals-ai:legal_research:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_research","measuredAt":"2026-11-09","stderr":1.414,"costPerTest":0.058112,"latency":47.609},{"benchmarkId":"vals-ai:vibe-code:overall","benchmarkName":"Vibe Code Bench v1.1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":3.683,"unit":"percent","family":"vals-ai:vibe-code:overall","index":43.57840685523127,"aboveFit":false,"run":"OpenHands","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","measuredAt":"2026-11-09","stderr":1.407,"costPerTest":0.378016,"latency":193.041},{"benchmarkId":"vals-ai:proof_bench:overall","benchmarkName":"ProofBench v1.1","category":"math","domain":null,"eligible":true,"ranked":true,"score":3,"unit":"percent","family":"vals-ai:proof_bench:overall","index":48.19235469365171,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/proof_bench","measuredAt":"2026-11-09","stderr":1.714,"costPerTest":0.044105,"latency":201.725},{"benchmarkId":"vals-ai:tax_agent_bench:overall","benchmarkName":"Tax Agent Bench","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":2.487,"unit":"percent","family":"vals-ai:tax_agent_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench","measuredAt":"2026-11-09","stderr":1.018,"costPerTest":0.023633,"latency":29.67},{"benchmarkId":"vals-ai:ioi:overall","benchmarkName":"IOI","category":"coding","domain":null,"eligible":true,"ranked":true,"score":2.444,"unit":"percent","family":"vals-ai:ioi:overall","index":47.03416652082835,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/ioi","measuredAt":"2026-10-29","stderr":0.683,"costPerTest":0.160874,"latency":242.94},{"benchmarkId":"gdpvalAaNormalized","benchmarkName":"GDPval-AA normalized","category":"agentic","score":0,"unit":"percent","family":"gdpvalAaNormalized","index":34.03377491719279,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":0,"unit":"percent","family":"critpt","index":36.8491160127729,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:terminal-bench-4:overall","benchmarkName":"Terminal-Bench 4.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":0,"unit":"percent","family":"terminal-bench-4","index":54.516013465855735,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-4","measuredAt":"2026-11-09","stderr":0,"costPerTest":0.368017,"latency":913.643},{"benchmarkId":"vals-ai:terminal-bench-science:overall","benchmarkName":"Terminal-Bench-Science","category":null,"domain":"Science","eligible":false,"ranked":true,"score":0,"unit":"percent","family":"vals-ai:terminal-bench-science:overall","index":null,"aboveFit":false,"run":"Mini-SWE-agent","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-science","measuredAt":"2026-11-09","stderr":0,"costPerTest":0.17804,"latency":1693.946}],"price":{"inputPerMillion":0.04,"outputPerMillion":0.15,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":125,"timeToFirstToken":0.726,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":85,"benchmarkCount":17,"categoryCount":6,"confidence":"medium","state":"available"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"glm-4-5","providerId":"z-ai","slug":"glm-4-5","name":"GLM-4.5","provider":"Z.AI","description":"GLM-4.5 is a standard model from Z.AI. 8 benchmarks count toward its index, in 4 categories.","logoUrl":null,"releaseDate":"2025-07-28","contextWindowTokens":131072,"modalities":["text"],"openWeights":true,"details":{"apiId":"z-ai/glm-4.5","maxOutputTokens":98304,"knowledgeCutoff":"2024-12-31","retiresAt":"2026-12-31","tools":true,"structuredOutput":false,"cacheReadPerMillion":0.11,"weightsUrl":"https://huggingface.co/zai-org/GLM-4.5"},"reasoning":false,"overallScore":45.737868088295,"overallScoreRange":{"low":36.520909762401764,"high":54.95482641418824},"badges":[],"capabilityScores":{"agentic":48.06982840515033,"coding":43.315232102800564,"reasoning":null,"multimodal":null,"knowledge":45.12993603468191,"multilingual":null,"instruction":null,"math":48.42103306728973},"capabilityCounts":{"math":2,"knowledge":2,"coding":3,"agentic":1},"benchmarks":[{"benchmarkId":"vals-ai:math500:overall","benchmarkName":"MATH 500","category":"math","domain":null,"eligible":true,"ranked":true,"score":94,"unit":"percent","family":"math-500","index":43.462055224786006,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/math500","measuredAt":"2026-01-09","stderr":1.062,"costPerTest":0.012744,"latency":84.928},{"benchmarkId":"vals-ai:mgsm:overall","benchmarkName":"MGSM","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":90.836,"unit":"percent","family":"vals-ai:mgsm:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mgsm","measuredAt":"2026-01-09","stderr":1.791,"costPerTest":0.006319,"latency":89.549},{"benchmarkId":"vals-ai:medqa:overall","benchmarkName":"MedQA","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":89.975,"unit":"percent","family":"vals-ai:medqa:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medqa","measuredAt":"2026-04-16","stderr":0.274,"costPerTest":0.003386,"latency":59.609},{"benchmarkId":"vals-ai:aime:overall","benchmarkName":"AIME","category":"math","domain":null,"eligible":true,"ranked":true,"score":86.667,"unit":"percent","family":"vals-ai:aime:overall","index":53.380010909793455,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/aime","measuredAt":"2026-04-16","stderr":1.332,"costPerTest":0.04689,"latency":333.117},{"benchmarkId":"vals-ai:mmlu_pro:overall","benchmarkName":"MMLU Pro","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":81.222,"unit":"percent","family":"mmlu-pro","index":46.81072640345758,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","measuredAt":"2026-09-01","stderr":0.378,"costPerTest":0.009757,"latency":136.72},{"benchmarkId":"vals-ai:legal_bench:overall","benchmarkName":"LegalBench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":75.627,"unit":"percent","family":"vals-ai:legal_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","measuredAt":"2026-10-13","stderr":0.511,"costPerTest":null,"latency":16.402},{"benchmarkId":"vals-ai:tax_eval_v2:overall","benchmarkName":"TaxEval v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":72.404,"unit":"percent","family":"vals-ai:tax_eval_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","measuredAt":"2026-09-01","stderr":0.872,"costPerTest":0.009559,"latency":129.388},{"benchmarkId":"vals-ai:gpqa:overall","benchmarkName":"GPQA Diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":72.222,"unit":"percent","family":"gpqa-diamond","index":43.44914566590624,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","measuredAt":"2026-09-01","stderr":2.262,"costPerTest":0.019454,"latency":704.224},{"benchmarkId":"vals-ai:lcb:overall","benchmarkName":"LiveCodeBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":67.446,"unit":"percent","family":"livecodebench","index":45.30649343481752,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","measuredAt":"2026-09-01","stderr":1.131,"costPerTest":0.018278,"latency":391.268},{"benchmarkId":"vals-ai:corp_fin_v2:overall","benchmarkName":"CorpFin v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":60.956,"unit":"percent","family":"vals-ai:corp_fin_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","measuredAt":"2026-08-12","stderr":0.959,"costPerTest":0.03783,"latency":95.244},{"benchmarkId":"swebench-experiments:swe-bench-verified:overall","benchmarkName":"SWE-bench Verified","category":"coding","domain":null,"eligible":true,"ranked":true,"score":59.2,"unit":"percent","family":"swe-bench-verified","index":44.89133613426621,"aboveFit":false,"run":"Undisclosed","source":"SWE-bench team","sourceUrl":"https://github.com/SWE-bench/experiments/tree/main/evaluation/verified","measuredAt":"2026-09-01","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"swe-bench:swe-bench-verified:overall","benchmarkName":"SWE-bench Verified","category":"coding","domain":null,"eligible":true,"ranked":true,"score":54.2,"unit":"percent","family":"swe-bench-verified","index":41.203548245347754,"aboveFit":false,"run":"mini-SWE-agent","source":"SWE-bench team","sourceUrl":"https://www.swebench.com/","measuredAt":"2026-02-26","stderr":null,"costPerTest":0.2971903488,"latency":null},{"benchmarkId":"vals-ai:terminal-bench:overall","benchmarkName":"Terminal-Bench 1.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":41.25,"unit":"percent","family":"vals-ai:terminal-bench:overall","index":48.06982840515033,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench","measuredAt":"2026-01-12","stderr":5.504,"costPerTest":0.197232,"latency":169.409},{"benchmarkId":"vals-ai:ioi-v1:overall","benchmarkName":"IOI v1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":2.917,"unit":"percent","family":"vals-ai:ioi-v1:overall","index":41.59176068377718,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/ioi-v1","measuredAt":"2026-08-09","stderr":0.987,"costPerTest":0.942275,"latency":2336.43},{"benchmarkId":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-public:overall","benchmarkName":"FrontierMath-Tier-4-2025-07-01-Public","category":"math","domain":null,"eligible":false,"ranked":true,"score":0,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-public:overall","index":null,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-2025-02-28-public:overall","benchmarkName":"FrontierMath-2025-02-28-Public","category":"math","domain":null,"eligible":false,"ranked":true,"score":0,"unit":"percent","family":"epoch-ai:epoch-frontiermath-2025-02-28-public:overall","index":null,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":0.6,"outputPerMillion":2.2,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":25,"timeToFirstToken":4.131,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":{"elo":1430.1898500000298,"eloUpper":1435.0892450345013,"eloLower":1425.2904549655582,"votes":23759,"rank":95,"rankUpper":null,"rankLower":null,"arenaName":"GLM-4.5","organization":"zai","updatedAt":"2026-11-09"},"quality":{"coveragePercent":65,"benchmarkCount":8,"categoryCount":4,"confidence":"low","state":"available"},"hosts":[{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":0.6,"outputPerMillion":2.2,"cacheReadPerMillion":0.11,"contextTokens":131072,"maxOutputTokens":98304,"deprecated":true},{"host":"z-ai","name":"Z.AI","kind":"maker","url":"https://docs.z.ai/guides/overview/pricing","inputPerMillion":0.6,"outputPerMillion":2.2,"cacheReadPerMillion":0.11,"contextTokens":131072,"maxOutputTokens":98304,"deprecated":false},{"host":"zhipu-ai","name":"Zhipu AI","kind":"maker","url":"https://docs.bigmodel.cn/cn/guide/start/pricing","inputPerMillion":0.6,"outputPerMillion":2.2,"cacheReadPerMillion":0.11,"contextTokens":131072,"maxOutputTokens":98304,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:swe-bench-team","name":"SWE-bench team","url":"https://github.com","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"grok-4-fast-reasoning","providerId":"xai","slug":"grok-4-fast-reasoning","name":"Grok 4 Fast (Reasoning)","provider":"xAI","description":"Grok 4 Fast (Reasoning) is a reasoning model from xAI in the Grok 4 Fast family. 18 benchmarks count toward its index, in 7 categories.","logoUrl":null,"releaseDate":"2025-09-19","contextWindowTokens":2000000,"modalities":["text"],"openWeights":false,"details":null,"reasoning":true,"overallScore":45.33880371133998,"overallScoreRange":{"low":41.027498160744486,"high":49.65010926193547},"badges":[],"capabilityScores":{"agentic":42.90415837074922,"coding":44.86235905508006,"reasoning":47.80566707596981,"multimodal":42.7215928170415,"knowledge":46.600313890188275,"multilingual":null,"instruction":36.2164154409548,"math":55.38299162986586},"capabilityCounts":{"math":1,"knowledge":4,"coding":4,"reasoning":4,"multimodal":1,"agentic":3,"instruction":1},"benchmarks":[{"benchmarkId":"vals-ai:medqa:overall","benchmarkName":"MedQA","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":92.067,"unit":"percent","family":"vals-ai:medqa:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medqa","measuredAt":"2026-04-16","stderr":0.247,"costPerTest":0.000744,"latency":5.343},{"benchmarkId":"vals-ai:aime:overall","benchmarkName":"AIME","category":"math","domain":null,"eligible":true,"ranked":true,"score":91.25,"unit":"percent","family":"vals-ai:aime:overall","index":55.38299162986586,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/aime","measuredAt":"2026-04-16","stderr":1.033,"costPerTest":0.004317,"latency":34.559},{"benchmarkId":"vals-ai:mgsm:overall","benchmarkName":"MGSM","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":90.873,"unit":"percent","family":"vals-ai:mgsm:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mgsm","measuredAt":"2026-01-09","stderr":1.781,"costPerTest":0.000639,"latency":3.853},{"benchmarkId":"aaAime2025","benchmarkName":"Artificial Analysis AIME 2025","category":"math","score":89.7,"unit":"percent","family":"aaAime2025","index":null,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/aime-2025","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:gpqa:overall","benchmarkName":"GPQA Diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":85.354,"unit":"percent","family":"gpqa-diamond","index":55.49345723295286,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","measuredAt":"2026-09-01","stderr":1.789,"costPerTest":0.003049,"latency":21.408},{"benchmarkId":"aaMmluPro","benchmarkName":"Artificial Analysis MMLU-Pro","category":"knowledge","score":85,"unit":"percent","family":"mmlu-pro","index":51.57055418848201,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmlu-pro","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":84.7,"unit":"percent","family":"gpqa-diamond","index":54.89362630355532,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaLiveCodeBench","benchmarkName":"Artificial Analysis LiveCodeBench","category":"coding","score":83.2,"unit":"percent","family":"livecodebench","index":58.378236756526675,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/livecodebench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:medscribe:overall","benchmarkName":"MedScribe","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":81.632,"unit":"percent","family":"vals-ai:medscribe:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medscribe","measuredAt":"2026-10-29","stderr":2.137,"costPerTest":0.002535,"latency":12.423},{"benchmarkId":"vals-ai:legal_bench:overall","benchmarkName":"LegalBench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":80.601,"unit":"percent","family":"vals-ai:legal_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","measuredAt":"2026-10-13","stderr":0.453,"costPerTest":null,"latency":3.803},{"benchmarkId":"vals-ai:mmlu_pro:overall","benchmarkName":"MMLU Pro","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":79.704,"unit":"percent","family":"mmlu-pro","index":44.898228103386884,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","measuredAt":"2026-09-01","stderr":0.393,"costPerTest":0.001742,"latency":7.343},{"benchmarkId":"vals-ai:lcb:overall","benchmarkName":"LiveCodeBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":78.973,"unit":"percent","family":"livecodebench","index":54.8709205815321,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","measuredAt":"2026-09-01","stderr":1.1,"costPerTest":0.004507,"latency":51.14},{"benchmarkId":"vals-ai:tax_eval_v2:overall","benchmarkName":"TaxEval v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":75.697,"unit":"percent","family":"vals-ai:tax_eval_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","measuredAt":"2026-09-01","stderr":0.76,"costPerTest":0.001268,"latency":10.184},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":73.7,"unit":"percent","family":"lcr","index":57.86218419483386,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:mmmu:overall","benchmarkName":"MMMU Pro","category":"multimodal","domain":null,"eligible":true,"ranked":true,"score":72.775,"unit":"percent","family":"mmmu-pro","index":49.86309262188293,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmmu","measuredAt":"2026-09-01","stderr":1.07,"costPerTest":0.00288,"latency":19.482},{"benchmarkId":"vals-ai:corp_fin_v2:overall","benchmarkName":"CorpFin v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":66.9,"unit":"percent","family":"vals-ai:corp_fin_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","measuredAt":"2026-08-12","stderr":0.925,"costPerTest":0.040289,"latency":11.839},{"benchmarkId":"tau2Bench","benchmarkName":"τ²-Bench Tool-Agent-User Evaluation","category":"agentic","score":65.8,"unit":"percent","family":"tau2-bench","index":46.452550909780534,"aboveFit":false,"run":null,"source":"Victor Barres, Honghua Dong, Soham Ray, Xujie Si, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2506.07982","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:case_law_v2:overall","benchmarkName":"CaseLaw v2","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":65.702,"unit":"percent","family":"vals-ai:case_law_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/case_law_v2","measuredAt":"2026-05-04","stderr":0.429,"costPerTest":0.007292,"latency":23.709},{"benchmarkId":"aaMmmuPro","benchmarkName":"Artificial Analysis MMMU-Pro","category":"multimodal","score":61.8,"unit":"percent","family":"mmmu-pro","index":35.58009301220008,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmmu-pro","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"arc-prize:v1-public-eval:overall","benchmarkName":"ARC-AGI-1 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":58.13,"unit":"percent","family":"arc-prize:v1-public-eval:overall","index":null,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.0278,"latency":null},{"benchmarkId":"aaIfBench","benchmarkName":"Artificial Analysis IFBench","category":"instruction","score":50.5,"unit":"percent","family":"ifbench-family","index":36.2164154409548,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"arc-prize:v1-semi-private:overall","benchmarkName":"ARC-AGI-1 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":48.5,"unit":"percent","family":"arc-prize:v1-semi-private:overall","index":49.28455985444903,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.0307,"latency":null},{"benchmarkId":"vals-ai:finance_agent:overall","benchmarkName":"Finance Agent v1.1","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":46.084,"unit":"percent","family":"vals-ai:finance_agent:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/finance_agent","measuredAt":"2026-06-04","stderr":2.841,"costPerTest":0.046595,"latency":58.799},{"benchmarkId":"vals-ai:swebench:overall","benchmarkName":"SWE-bench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":45.4,"unit":"percent","family":"swe-bench-verified","index":34.713041560851266,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/swebench","measuredAt":"2026-09-01","stderr":2.229,"costPerTest":0.050771,"latency":125.811},{"benchmarkId":"vals-ai:mortgage_tax:overall","benchmarkName":"MortgageTax","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":42.09,"unit":"percent","family":"vals-ai:mortgage_tax:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mortgage_tax","measuredAt":"2026-09-01","stderr":0.95,"costPerTest":0.004554,"latency":14.475},{"benchmarkId":"vals-ai:medcode:overall","benchmarkName":"MedCode","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":37.385,"unit":"percent","family":"vals-ai:medcode:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medcode","measuredAt":"2026-10-29","stderr":1.941,"costPerTest":0.002143,"latency":17.515},{"benchmarkId":"vals-ai:terminal-bench-2:overall","benchmarkName":"Terminal-Bench 2.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":29.213,"unit":"percent","family":"vals-ai:terminal-bench-2:overall","index":44.14074062592708,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2","measuredAt":"2026-06-04","stderr":4.848,"costPerTest":0.03431,"latency":431.268},{"benchmarkId":"vals-ai:terminal-bench:overall","benchmarkName":"Terminal-Bench 1.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":27.5,"unit":"percent","family":"vals-ai:terminal-bench:overall","index":38.11918357654005,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench","measuredAt":"2026-01-12","stderr":4.992,"costPerTest":0.188369,"latency":134.967},{"benchmarkId":"omniscienceAccuracy","benchmarkName":"Artificial Analysis Omniscience Accuracy","category":"knowledge","score":22.8,"unit":"percent","family":"omniscienceAccuracy","index":39.42378690318201,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":19.1,"unit":"percent","family":"hle","index":43.54953574338255,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:ioi-v1:overall","benchmarkName":"IOI v1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":11.5,"unit":"percent","family":"vals-ai:ioi-v1:overall","index":45.86229339947811,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/ioi-v1","measuredAt":"2026-08-09","stderr":4.388,"costPerTest":0.97412,"latency":1408.454},{"benchmarkId":"arc-prize:v2-semi-private:overall","benchmarkName":"ARC-AGI-2 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":5.28,"unit":"percent","family":"arc-prize:v2-semi-private:overall","index":41.61274662339665,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.061,"latency":null},{"benchmarkId":"arc-prize:v2-public-eval:overall","benchmarkName":"ARC-AGI-2 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":4.58,"unit":"percent","family":"arc-prize:v2-public-eval:overall","index":null,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.0614,"latency":null},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":2.9,"unit":"percent","family":"critpt","index":42.46317763119968,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:vibe-code:overall","benchmarkName":"Vibe Code Bench v1.1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":0,"unit":"percent","family":"vals-ai:vibe-code:overall","index":42.24952259096148,"aboveFit":false,"run":"OpenHands","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","measuredAt":"2026-11-09","stderr":0,"costPerTest":0.133206,"latency":1448.855}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":{"elo":1399.581907659835,"eloUpper":1404.6347067100842,"eloLower":1394.5291086095858,"votes":18206,"rank":158,"rankUpper":null,"rankLower":null,"arenaName":"Grok 4 Fast-reasoning","organization":"xai","updatedAt":"2026-11-09"},"quality":{"coveragePercent":95,"benchmarkCount":18,"categoryCount":7,"confidence":"medium","state":"available"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:arc-prize-foundation","name":"ARC Prize Foundation","url":"https://arcprize.org","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"qwen3-max","providerId":"alibaba","slug":"qwen3-max","name":"Qwen3 Max","provider":"Alibaba","description":"Qwen3 Max is a reasoning model from Alibaba. 20 benchmarks count toward its index, in 6 categories.","logoUrl":null,"releaseDate":"2025-09-24","contextWindowTokens":262144,"modalities":["text"],"openWeights":false,"details":{"apiId":"qwen/qwen3-max","maxOutputTokens":65536,"knowledgeCutoff":"2025-06-30","retiresAt":"2026-10-09","tools":true,"structuredOutput":true,"cacheReadPerMillion":0.156,"weightsUrl":null},"reasoning":true,"overallScore":45.029166107989624,"overallScoreRange":{"low":39.913444188372466,"high":50.14488802760678},"badges":[],"capabilityScores":{"agentic":47.061551118075286,"coding":46.344064062234686,"reasoning":40.33475481227423,"multimodal":null,"knowledge":48.091180610802645,"multilingual":null,"instruction":30.235547506045982,"math":46.276518887462004},"capabilityCounts":{"math":4,"knowledge":5,"agentic":4,"coding":3,"reasoning":3,"instruction":1},"benchmarks":[{"benchmarkId":"epoch-ai:epoch-math-level-5:overall","benchmarkName":"MATH level 5","category":"math","domain":null,"eligible":true,"ranked":true,"score":97.1299093655589,"unit":"percent","family":"epoch-ai:epoch-math-level-5:overall","index":46.179286947154175,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0.42006948157449053,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:mgsm:overall","benchmarkName":"MGSM","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":92.146,"unit":"percent","family":"vals-ai:mgsm:overall","index":null,"aboveFit":false,"run":"qwen3-max-preview","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mgsm","measuredAt":"2026-01-09","stderr":1.644,"costPerTest":0.002344,"latency":21.159},{"benchmarkId":"vals-ai:mgsm:overall","benchmarkName":"MGSM","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":91.818,"unit":"percent","family":"vals-ai:mgsm:overall","index":null,"aboveFit":false,"run":"qwen3-max","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mgsm","measuredAt":"2026-01-09","stderr":1.677,"costPerTest":0.002856,"latency":11.451},{"benchmarkId":"vals-ai:medqa:overall","benchmarkName":"MedQA","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":87.375,"unit":"percent","family":"vals-ai:medqa:overall","index":null,"aboveFit":false,"run":"qwen3-max-preview","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medqa","measuredAt":"2026-04-16","stderr":0.303,"costPerTest":0.00318,"latency":19.552},{"benchmarkId":"vals-ai:medqa:overall","benchmarkName":"MedQA","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":87.367,"unit":"percent","family":"vals-ai:medqa:overall","index":null,"aboveFit":false,"run":"qwen3-max","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medqa","measuredAt":"2026-04-16","stderr":0.303,"costPerTest":0.003172,"latency":0},{"benchmarkId":"vals-ai:mmlu_pro:overall","benchmarkName":"MMLU Pro","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":84.36,"unit":"percent","family":"mmlu-pro","index":50.76423079451281,"aboveFit":false,"run":"qwen3-max","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","measuredAt":"2026-09-01","stderr":0.356,"costPerTest":0.006426,"latency":0},{"benchmarkId":"tau2-bench:telecom:overall","benchmarkName":"τ²-bench Telecom","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":84.2,"unit":"percent","family":"tau2-bench","index":58.66229616913809,"aboveFit":false,"run":"Alibaba","source":"Sierra Research","sourceUrl":"https://github.com/sierra-research/tau2-bench","measuredAt":"2026-03-02","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:mmlu_pro:overall","benchmarkName":"MMLU Pro","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":83.537,"unit":"percent","family":"mmlu-pro","index":49.72734930508055,"aboveFit":false,"run":"qwen3-max-preview","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","measuredAt":"2026-09-01","stderr":0.364,"costPerTest":0.006194,"latency":43.428},{"benchmarkId":"vals-ai:legal_bench:overall","benchmarkName":"LegalBench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":81.861,"unit":"percent","family":"vals-ai:legal_bench:overall","index":null,"aboveFit":false,"run":"qwen3-max","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","measuredAt":"2026-10-13","stderr":0.399,"costPerTest":null,"latency":0},{"benchmarkId":"vals-ai:aime:overall","benchmarkName":"AIME","category":"math","domain":null,"eligible":true,"ranked":true,"score":81.042,"unit":"percent","family":"vals-ai:aime:overall","index":50.921628507348046,"aboveFit":false,"run":"qwen3-max","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/aime","measuredAt":"2026-04-16","stderr":1.1,"costPerTest":0.033364,"latency":16.76},{"benchmarkId":"vals-ai:legal_bench:overall","benchmarkName":"LegalBench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":80.333,"unit":"percent","family":"vals-ai:legal_bench:overall","index":null,"aboveFit":false,"run":"qwen3-max-preview","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","measuredAt":"2026-10-13","stderr":0.397,"costPerTest":null,"latency":1.886},{"benchmarkId":"vals-ai:gpqa:overall","benchmarkName":"GPQA Diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":79.546,"unit":"percent","family":"gpqa-diamond","index":50.16651833701876,"aboveFit":false,"run":"qwen3-max","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","measuredAt":"2026-09-01","stderr":2.026,"costPerTest":0.016466,"latency":70.152},{"benchmarkId":"vals-ai:lcb:overall","benchmarkName":"LiveCodeBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":78.215,"unit":"percent","family":"livecodebench","index":54.24197672994802,"aboveFit":false,"run":"qwen3-max","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","measuredAt":"2026-09-01","stderr":1.108,"costPerTest":0.021213,"latency":107.845},{"benchmarkId":"vals-ai:gpqa:overall","benchmarkName":"GPQA Diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":77.778,"unit":"percent","family":"gpqa-diamond","index":48.544957047760576,"aboveFit":false,"run":"qwen3-max-preview","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","measuredAt":"2026-09-01","stderr":2.09,"costPerTest":0.01338,"latency":106.784},{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":76.4,"unit":"percent","family":"gpqa-diamond","index":47.28109310172111,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"tau2Bench","benchmarkName":"τ²-Bench Tool-Agent-User Evaluation","category":"agentic","score":74.3,"unit":"percent","family":"tau2-bench","index":52.092922361114184,"aboveFit":false,"run":null,"source":"Victor Barres, Honghua Dong, Soham Ray, Xujie Si, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2506.07982","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:tax_eval_v2:overall","benchmarkName":"TaxEval v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":73.958,"unit":"percent","family":"vals-ai:tax_eval_v2:overall","index":null,"aboveFit":false,"run":"qwen3-max-preview","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","measuredAt":"2026-09-01","stderr":0.856,"costPerTest":0.00671,"latency":21.405},{"benchmarkId":"vals-ai:tax_eval_v2:overall","benchmarkName":"TaxEval v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":73.508,"unit":"percent","family":"vals-ai:tax_eval_v2:overall","index":null,"aboveFit":false,"run":"qwen3-max","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","measuredAt":"2026-09-01","stderr":0.86,"costPerTest":0.00654,"latency":29.296},{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":73.33333333333333,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":51.583987563657715,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":5.913945170082951,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:medscribe:overall","benchmarkName":"MedScribe","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":72.709,"unit":"percent","family":"vals-ai:medscribe:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medscribe","measuredAt":"2026-10-29","stderr":1.905,"costPerTest":0.085327,"latency":362.378},{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":72.6010101010101,"unit":"percent","family":"gpqa-diamond","index":43.7967633740713,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.734528103171235,"costPerTest":null,"latency":null},{"benchmarkId":"tau2-bench:retail:overall","benchmarkName":"τ²-bench Retail","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":72.2,"unit":"percent","family":"tau2-bench","index":50.69941882607882,"aboveFit":false,"run":"Alibaba","source":"Sierra Research","sourceUrl":"https://github.com/sierra-research/tau2-bench","measuredAt":"2026-04-30","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:lcb:overall","benchmarkName":"LiveCodeBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":66.91,"unit":"percent","family":"livecodebench","index":44.86175213607205,"aboveFit":false,"run":"qwen3-max-preview","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","measuredAt":"2026-09-01","stderr":1.167,"costPerTest":0.01353,"latency":96.247},{"benchmarkId":"vals-ai:aime:overall","benchmarkName":"AIME","category":"math","domain":null,"eligible":true,"ranked":true,"score":60.694,"unit":"percent","family":"vals-ai:aime:overall","index":42.02862137402198,"aboveFit":false,"run":"qwen3-max-preview","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/aime","measuredAt":"2026-04-16","stderr":2.562,"costPerTest":0.023054,"latency":206.753},{"benchmarkId":"tau2-bench:airline:overall","benchmarkName":"τ²-bench Airline","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":59.5,"unit":"percent","family":"tau2-bench","index":42.27204030467442,"aboveFit":false,"run":"Alibaba","source":"Sierra Research","sourceUrl":"https://github.com/sierra-research/tau2-bench","measuredAt":"2026-03-02","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:corp_fin_v2:overall","benchmarkName":"CorpFin v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":55.944,"unit":"percent","family":"vals-ai:corp_fin_v2:overall","index":null,"aboveFit":false,"run":"qwen3-max","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","measuredAt":"2026-08-12","stderr":0.977,"costPerTest":0.086893,"latency":55.675},{"benchmarkId":"vals-ai:corp_fin_v2:overall","benchmarkName":"CorpFin v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":52.953,"unit":"percent","family":"vals-ai:corp_fin_v2:overall","index":null,"aboveFit":false,"run":"qwen3-max-preview","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","measuredAt":"2026-08-12","stderr":0.975,"costPerTest":0.09654,"latency":227.451},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":50,"unit":"percent","family":"lcr","index":41.60207573845371,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-simpleqa-verified:overall","benchmarkName":"SimpleQA Verified","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":48.74874874874875,"unit":"percent","family":"epoch-ai:epoch-simpleqa-verified:overall","index":64.53106347263042,"aboveFit":true,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.582226675546796,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:case_law_v2:overall","benchmarkName":"CaseLaw v2","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":47.481,"unit":"percent","family":"vals-ai:case_law_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/case_law_v2","measuredAt":"2026-05-04","stderr":0.467,"costPerTest":0.050593,"latency":84.725},{"benchmarkId":"vals-ai:finance_agent:overall","benchmarkName":"Finance Agent v1.1","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":44.295,"unit":"percent","family":"vals-ai:finance_agent:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/finance_agent","measuredAt":"2026-06-04","stderr":2.806,"costPerTest":0.801603,"latency":315.362},{"benchmarkId":"aaIfBench","benchmarkName":"Artificial Analysis IFBench","category":"instruction","score":44.1,"unit":"percent","family":"ifbench-family","index":30.235547506045982,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gertLabs","benchmarkName":"Gert Labs Composite Game Benchmark","category":"agentic","score":43.74,"unit":"percent","family":"gertLabs","index":50.963769131647574,"aboveFit":false,"run":null,"source":"Gert Labs","sourceUrl":"https://gertlabs.com/rankings","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:terminal-bench:overall","benchmarkName":"Terminal-Bench 1.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":36.25,"unit":"percent","family":"vals-ai:terminal-bench:overall","index":44.451412103837505,"aboveFit":false,"run":"qwen3-max","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench","measuredAt":"2026-01-12","stderr":5.375,"costPerTest":0.480579,"latency":297.376},{"benchmarkId":"vals-ai:terminal-bench:overall","benchmarkName":"Terminal-Bench 1.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":36.25,"unit":"percent","family":"vals-ai:terminal-bench:overall","index":44.451412103837505,"aboveFit":false,"run":"qwen3-max-preview","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench","measuredAt":"2026-01-12","stderr":5.375,"costPerTest":0.715376,"latency":172.116},{"benchmarkId":"vals-ai:medcode:overall","benchmarkName":"MedCode","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":31.373,"unit":"percent","family":"vals-ai:medcode:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medcode","measuredAt":"2026-10-29","stderr":1.888,"costPerTest":0.014776,"latency":182.194},{"benchmarkId":"vals-ai:terminal-bench-2:overall","benchmarkName":"Terminal-Bench 2.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":24.719,"unit":"percent","family":"vals-ai:terminal-bench-2:overall","index":41.43485264321955,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2","measuredAt":"2026-06-04","stderr":4.599,"costPerTest":0.423036,"latency":912.424},{"benchmarkId":"omniscienceAccuracy","benchmarkName":"Artificial Analysis Omniscience Accuracy","category":"knowledge","score":24.4,"unit":"percent","family":"omniscienceAccuracy","index":41.44019647767727,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tiers-1-3-v2-private:overall","benchmarkName":"FrontierMath-Tiers-1-3-v2-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":18.947368421052634,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tiers-1-3-v2-private:overall","index":40.86767609835112,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.3254062737737247,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:ioi-v1:overall","benchmarkName":"IOI v1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":15.666,"unit":"percent","family":"vals-ai:ioi-v1:overall","index":47.93511633943035,"aboveFit":false,"run":"qwen3-max","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/ioi-v1","measuredAt":"2026-08-09","stderr":6.352,"costPerTest":0.824523,"latency":2045.247},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":11.9,"unit":"percent","family":"hle","index":36.32582797916803,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:ioi-v1:overall","benchmarkName":"IOI v1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":7.75,"unit":"percent","family":"vals-ai:ioi-v1:overall","index":43.996454219252264,"aboveFit":false,"run":"qwen3-max-preview","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/ioi-v1","measuredAt":"2026-08-09","stderr":3.165,"costPerTest":1.399136,"latency":3372.048},{"benchmarkId":"epoch-ai:epoch-mystery-game-puzzles:overall","benchmarkName":"Mystery Game Puzzles","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":5,"unit":"percent","family":"epoch-ai:epoch-mystery-game-puzzles:overall","index":42.55307268559606,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.1904291355759034,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:vibe-code:overall","benchmarkName":"Vibe Code Bench v1.1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":3.506,"unit":"percent","family":"vals-ai:vibe-code:overall","index":43.51454247435271,"aboveFit":false,"run":"OpenHands","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","measuredAt":"2026-11-09","stderr":1.292,"costPerTest":6.023105,"latency":3465.17},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":0,"unit":"percent","family":"critpt","index":36.8491160127729,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":0.78,"outputPerMillion":3.9,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":62,"timeToFirstToken":0.734,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":{"elo":1412.7862799030922,"eloUpper":1419.244346414342,"eloLower":1406.3282133918424,"votes":9004,"rank":138,"rankUpper":null,"rankLower":null,"arenaName":"Qwen3-Max-2025-09-23","organization":"alibaba","updatedAt":"2026-11-09"},"quality":{"coveragePercent":85,"benchmarkCount":20,"categoryCount":6,"confidence":"medium","state":"available"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:sierra-research","name":"Sierra Research","url":"https://github.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"o4-mini","providerId":"openai","slug":"o4-mini","name":"o4-mini","provider":"OpenAI","description":"o4-mini is a reasoning model from OpenAI. 29 benchmarks count toward its index, in 8 categories.","logoUrl":null,"releaseDate":"2025-04-16","contextWindowTokens":200000,"modalities":["image","text","file"],"openWeights":false,"details":{"apiId":"openai/o4-mini","maxOutputTokens":100000,"knowledgeCutoff":"2024-06-30","retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.275,"weightsUrl":null},"reasoning":true,"overallScore":44.65185605891473,"overallScoreRange":{"low":41.6491624405918,"high":47.65454967723766},"badges":[],"capabilityScores":{"agentic":38.522852222317354,"coding":45.17014259901387,"reasoning":44.91974298621332,"multimodal":52.02018345131795,"knowledge":40.83162552581885,"multilingual":35.51907770541009,"instruction":53.224508630851766,"math":50.4855215795626},"capabilityCounts":{"math":8,"coding":4,"multilingual":1,"knowledge":7,"multimodal":1,"instruction":1,"agentic":1,"reasoning":6},"benchmarks":[{"benchmarkId":"aaMath500","benchmarkName":"Artificial Analysis MATH-500","category":"math","score":98.9,"unit":"percent","family":"math-500","index":48.64017046569242,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/math-500","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-math-level-5:overall","benchmarkName":"MATH level 5","category":"math","domain":null,"eligible":true,"ranked":true,"score":97.8285498489426,"unit":"percent","family":"epoch-ai:epoch-math-level-5:overall","index":46.51865531938343,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0.33204985518619606,"costPerTest":null,"latency":null},{"benchmarkId":"math500","benchmarkName":"MATH-500 Problem Set","category":"math","score":97.5,"unit":"percent","family":"math-500","index":47.160708968290585,"aboveFit":false,"run":null,"source":"Dan Hendrycks, Collin Burns, Saurav Kadavath, Akul Arora, Steven Basart, Eric Tang, Dawn Song, Jacob Steinhardt","sourceUrl":"https://arxiv.org/abs/2103.03874","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"humaneval","benchmarkName":"Evaluating Large Language Models Trained on Code","category":"coding","score":97.3,"unit":"percent","family":"humaneval","index":37.402074165233856,"aboveFit":false,"run":null,"source":"Mark Chen, Jerry Tworek, Heewoo Jun, Qiming Yuan, Henrique Ponde de Oliveira Pinto, Jared Kaplan, Harri Edwards, Yuri Burda, Nicholas Joseph, Greg Brockman, Alex Ray, Raul Puri, Gretchen Krueger, Michael Petrov, Heidy Khlaaf, Girish Sastry, Pamela Mishkin, Brooke Chan, Scott Gray, Nick Ryder, Mikhail Pavlov, Alethea Power, Lukasz Kaiser, Mohammad Bavarian, Clemens Winter, Philippe Tillet, Felipe Petroski Such, Dave Cummings, Matthias Plappert, Fotios Chantzis, Elizabeth Barnes, Ariel Herbert-Voss, William Hebgen Guss, Alex Nichol, Alex Paino, Nikolas Tezak, Jie Tang, Igor Babuschkin, Suchir Balaji, Shantanu Jain, William Saunders, Christopher Hesse, Andrew N. Carr, Jan Leike, Josh Achiam, Vedant Misra, Evan Morikawa, Alec Radford, Matthew Knight, Miles Brundage, Mira Murati, Katie Mayer, Peter Welinder, Bob McGrew, Dario Amodei, Sam McCandlish, Ilya Sutskever, Wojciech Zaremba","sourceUrl":"https://arxiv.org/abs/2107.03374","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:medqa:overall","benchmarkName":"MedQA","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":96.017,"unit":"percent","family":"vals-ai:medqa:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medqa","measuredAt":"2026-04-16","stderr":0.178,"costPerTest":0.002784,"latency":6.604},{"benchmarkId":"vals-ai:math500:overall","benchmarkName":"MATH 500","category":"math","domain":null,"eligible":true,"ranked":true,"score":94.2,"unit":"percent","family":"math-500","index":43.673406867271986,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/math500","measuredAt":"2026-01-09","stderr":1.045,"costPerTest":0.004784,"latency":12.536},{"benchmarkId":"mgsm","benchmarkName":"Multilingual Grade School Math","category":"multilingual","score":93.7,"unit":"percent","family":"mgsm","index":35.51907770541009,"aboveFit":false,"run":null,"source":"Freda Shi, Mirac Suzgun, Markus Freitag, Xuezhi Wang, Suraj Srivats, Soroush Vosoughi, Hyung Won Chung, Yi Tay, Sebastian Ruder, Denny Zhou, Dipanjan Das, Jason Wei","sourceUrl":"https://arxiv.org/abs/2210.03057","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:mgsm:overall","benchmarkName":"MGSM","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":93.418,"unit":"percent","family":"vals-ai:mgsm:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mgsm","measuredAt":"2026-01-09","stderr":1.507,"costPerTest":0.002905,"latency":8.067},{"benchmarkId":"aaAime2025","benchmarkName":"Artificial Analysis AIME 2025","category":"math","score":90.7,"unit":"percent","family":"aaAime2025","index":null,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/aime-2025","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmlu","benchmarkName":"Massive Multitask Language Understanding","category":"knowledge","score":90,"unit":"percent","family":"mmlu","index":38.991587985272176,"aboveFit":false,"run":null,"source":"Dan Hendrycks, Collin Burns, Steven Basart, Andy Zou, Mantas Mazeika, Dawn Song, Jacob Steinhardt","sourceUrl":"https://arxiv.org/abs/2009.03300","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaLiveCodeBench","benchmarkName":"Artificial Analysis LiveCodeBench","category":"coding","score":85.9,"unit":"percent","family":"livecodebench","index":60.61853807483408,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/livecodebench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:aime:overall","benchmarkName":"AIME","category":"math","domain":null,"eligible":true,"ranked":true,"score":83.667,"unit":"percent","family":"vals-ai:aime:overall","index":52.068873628489236,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/aime","measuredAt":"2026-04-16","stderr":1.178,"costPerTest":0.025707,"latency":55.109},{"benchmarkId":"vals-ai:lcb:overall","benchmarkName":"LiveCodeBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":82.208,"unit":"percent","family":"livecodebench","index":57.55513345735596,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","measuredAt":"2026-09-01","stderr":1.044,"costPerTest":0.013647,"latency":32.84},{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":81.66666666666667,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":56.4322712781001,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":4.693913962018081,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:mmlu_pro:overall","benchmarkName":"MMLU Pro","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":80.561,"unit":"percent","family":"mmlu-pro","index":45.977945523123786,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","measuredAt":"2026-09-01","stderr":0.386,"costPerTest":0.005524,"latency":10.443},{"benchmarkId":"vals-ai:mmmu:overall","benchmarkName":"MMMU Pro","category":"multimodal","domain":null,"eligible":true,"ranked":true,"score":79.665,"unit":"percent","family":"mmmu-pro","index":58.829823128553976,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmmu","measuredAt":"2026-09-01","stderr":0.967,"costPerTest":0.010516,"latency":19.718},{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":79.60858585858585,"unit":"percent","family":"gpqa-diamond","index":50.22392037635009,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.4034764096121957,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:legal_bench:overall","benchmarkName":"LegalBench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":79.185,"unit":"percent","family":"vals-ai:legal_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","measuredAt":"2026-10-13","stderr":0.498,"costPerTest":null,"latency":4.03},{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":78.4,"unit":"percent","family":"gpqa-diamond","index":49.115438451560685,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"drop","benchmarkName":"Discrete Reasoning Over Paragraphs","category":"reasoning","score":77.7,"unit":"percent","family":"drop","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gpqa","benchmarkName":"Graduate-Level Google-Proof Q&A","category":"knowledge","score":77.6,"unit":"percent","family":"gpqa-diamond","index":48.38170031162484,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:tax_eval_v2:overall","benchmarkName":"TaxEval v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":74.776,"unit":"percent","family":"vals-ai:tax_eval_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","measuredAt":"2026-09-01","stderr":0.853,"costPerTest":0.007057,"latency":15.447},{"benchmarkId":"vals-ai:gpqa:overall","benchmarkName":"GPQA Diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":74.495,"unit":"percent","family":"gpqa-diamond","index":45.53387915599892,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","measuredAt":"2026-09-01","stderr":2.19,"costPerTest":0.012961,"latency":30.004},{"benchmarkId":"aaMmmuPro","benchmarkName":"Artificial Analysis MMMU-Pro","category":"multimodal","score":69.2,"unit":"percent","family":"mmmu-pro","index":45.21054377408191,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmmu-pro","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:medscribe:overall","benchmarkName":"MedScribe","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":69.139,"unit":"percent","family":"vals-ai:medscribe:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medscribe","measuredAt":"2026-10-29","stderr":1.957,"costPerTest":0.040605,"latency":81.959},{"benchmarkId":"aaIfBench","benchmarkName":"Artificial Analysis IFBench","category":"instruction","score":68.7,"unit":"percent","family":"ifbench-family","index":53.224508630851766,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"tau2-bench:retail:overall","benchmarkName":"τ²-bench Retail","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":68.3,"unit":"percent","family":"tau2-bench","index":48.11148368958455,"aboveFit":false,"run":"Sierra","source":"Sierra Research","sourceUrl":"https://github.com/sierra-research/tau2-bench","measuredAt":"2026-04-30","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"arc-prize:v1-public-eval:overall","benchmarkName":"ARC-AGI-1 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":68.03,"unit":"percent","family":"arc-prize:v1-public-eval:overall","index":null,"aboveFit":false,"run":"high effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.3246,"latency":null},{"benchmarkId":"vals-ai:mortgage_tax:overall","benchmarkName":"MortgageTax","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":64.826,"unit":"percent","family":"vals-ai:mortgage_tax:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mortgage_tax","measuredAt":"2026-09-01","stderr":0.944,"costPerTest":0.011654,"latency":12.996},{"benchmarkId":"swebench-experiments:swe-bench-verified:overall","benchmarkName":"SWE-bench Verified","category":"coding","domain":null,"eligible":true,"ranked":true,"score":64.60000000000001,"unit":"percent","family":"swe-bench-verified","index":48.87414705429815,"aboveFit":false,"run":"PatchPilot-v1.1","source":"SWE-bench team","sourceUrl":"https://github.com/SWE-bench/experiments/tree/main/evaluation/verified","measuredAt":"2026-09-01","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":61,"unit":"percent","family":"lcr","index":49.14896151989598,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:corp_fin_v2:overall","benchmarkName":"CorpFin v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":58.974,"unit":"percent","family":"vals-ai:corp_fin_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","measuredAt":"2026-08-12","stderr":0.967,"costPerTest":0.074104,"latency":17.528},{"benchmarkId":"arc-prize:v1-semi-private:overall","benchmarkName":"ARC-AGI-1 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":58.67,"unit":"percent","family":"arc-prize:v1-semi-private:overall","index":53.96007602169864,"aboveFit":false,"run":"high effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.4058,"latency":null},{"benchmarkId":"tau2Bench","benchmarkName":"τ²-Bench Tool-Agent-User Evaluation","category":"agentic","score":55.6,"unit":"percent","family":"tau2-bench","index":39.68410516818016,"aboveFit":false,"run":null,"source":"Victor Barres, Honghua Dong, Soham Ray, Xujie Si, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2506.07982","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"tau2-bench:airline:overall","benchmarkName":"τ²-bench Airline","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":52.1,"unit":"percent","family":"tau2-bench","index":37.36159927645454,"aboveFit":false,"run":"Sierra","source":"Sierra Research","sourceUrl":"https://github.com/sierra-research/tau2-bench","measuredAt":"2026-03-02","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"tau2-bench:telecom:overall","benchmarkName":"τ²-bench Telecom","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":50.2,"unit":"percent","family":"tau2-bench","index":36.10081036380349,"aboveFit":false,"run":"Sierra","source":"Sierra Research","sourceUrl":"https://github.com/sierra-research/tau2-bench","measuredAt":"2026-03-02","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"swe-bench:swe-bench-verified:overall","benchmarkName":"SWE-bench Verified","category":"coding","domain":null,"eligible":true,"ranked":true,"score":45,"unit":"percent","family":"swe-bench-verified","index":34.418018529737786,"aboveFit":false,"run":"mini-SWE-agent","source":"SWE-bench team","sourceUrl":"https://www.swebench.com/","measuredAt":"2026-02-26","stderr":null,"costPerTest":0.2099733328,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tiers-1-3-v2-private:overall","benchmarkName":"FrontierMath-Tiers-1-3-v2-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":36.140350877192986,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tiers-1-3-v2-private:overall","index":49.91723183646946,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.8506918644975,"costPerTest":null,"latency":null},{"benchmarkId":"swe-bench:swe-bench-multimodal:overall","benchmarkName":"SWE-bench Multimodal","category":"multimodal","domain":null,"eligible":true,"ranked":true,"score":33.85,"unit":"percent","family":"swe-bench-multimodal","index":null,"aboveFit":false,"run":"GUIRepair","source":"SWE-bench team","sourceUrl":"https://www.swebench.com/","measuredAt":"2025-11-17","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"swebench-experiments:swe-bench-multimodal:overall","benchmarkName":"SWE-bench Multimodal","category":"multimodal","domain":null,"eligible":true,"ranked":true,"score":33.849129593810446,"unit":"percent","family":"swe-bench-multimodal","index":null,"aboveFit":false,"run":"GUIRepair","source":"SWE-bench team","sourceUrl":"https://github.com/SWE-bench/experiments/tree/main/evaluation/multimodal","measuredAt":"2025-11-17","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:medcode:overall","benchmarkName":"MedCode","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":33.791,"unit":"percent","family":"vals-ai:medcode:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medcode","measuredAt":"2026-10-29","stderr":2.021,"costPerTest":0.017605,"latency":21.062},{"benchmarkId":"epoch-ai:epoch-frontiermath-2025-02-28-public:overall","benchmarkName":"FrontierMath-2025-02-28-Public","category":"math","domain":null,"eligible":false,"ranked":true,"score":30,"unit":"percent","family":"epoch-ai:epoch-frontiermath-2025-02-28-public:overall","index":null,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":15.275252316519463,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-2025-02-28-private:overall","benchmarkName":"FrontierMath-2025-02-28-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":24.82758620689655,"unit":"percent","family":"epoch-ai:epoch-frontiermath-2025-02-28-private:overall","index":51.831410977384,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.541251077219606,"costPerTest":null,"latency":null},{"benchmarkId":"omniscienceAccuracy","benchmarkName":"Artificial Analysis Omniscience Accuracy","category":"knowledge","score":24.8,"unit":"percent","family":"omniscienceAccuracy","index":41.944298871301086,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"simpleQa","benchmarkName":"Measuring Short-Form Factuality in Large Language Models","category":"knowledge","score":20.2,"unit":"percent","family":"simpleQa","index":29.636890373538872,"aboveFit":false,"run":null,"source":"Jason Wei, Najoung Kim, Hyung Won Chung, Yu-An Chung, Siddhartha Papay, Yifeng Lu, Hannaneh Hajishirzi, Luke Zettlemoyer","sourceUrl":"https://arxiv.org/abs/2411.04368","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-simpleqa-verified:overall","benchmarkName":"SimpleQA Verified","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":18.8,"unit":"percent","family":"epoch-ai:epoch-simpleqa-verified:overall","index":39.581111939598216,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.2361586015103758,"costPerTest":null,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":16.5,"unit":"percent","family":"hle","index":40.94097460630508,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-chess-puzzles:overall","benchmarkName":"Chess Puzzles","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":13,"unit":"percent","family":"epoch-ai:epoch-chess-puzzles:overall","index":43.85187326209018,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":3.379976689896311,"costPerTest":null,"latency":null},{"benchmarkId":"arc-prize:v2-public-eval:overall","benchmarkName":"ARC-AGI-2 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":7.5200000000000005,"unit":"percent","family":"arc-prize:v2-public-eval:overall","index":null,"aboveFit":false,"run":"high effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.8822,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-private:overall","benchmarkName":"FrontierMath-Tier-4-2025-07-01-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":6.25,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-private:overall","index":49.55037447848712,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":3.530829267912701,"costPerTest":null,"latency":null},{"benchmarkId":"arc-prize:v2-semi-private:overall","benchmarkName":"ARC-AGI-2 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":6.11,"unit":"percent","family":"arc-prize:v2-semi-private:overall","index":41.99382842520686,"aboveFit":false,"run":"high effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.856,"latency":null},{"benchmarkId":"epoch-ai:epoch-mystery-game-puzzles:overall","benchmarkName":"Mystery Game Puzzles","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":5,"unit":"percent","family":"epoch-ai:epoch-mystery-game-puzzles:overall","index":42.55307268559606,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.190429135575903,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tier-4-v2-private:overall","benchmarkName":"FrontierMath-Tier-4-v2-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":4.878048780487805,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tier-4-v2-private:overall","index":50.40464614989685,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":3.4059122057973035,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:ioi-v1:overall","benchmarkName":"IOI v1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":4.834,"unit":"percent","family":"vals-ai:ioi-v1:overall","index":42.54557767270864,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/ioi-v1","measuredAt":"2026-08-09","stderr":2.355,"costPerTest":0.156077,"latency":480.192},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":0.6,"unit":"percent","family":"critpt","index":38.010646002792235,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-public:overall","benchmarkName":"FrontierMath-Tier-4-2025-07-01-Public","category":"math","domain":null,"eligible":false,"ranked":true,"score":0,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-public:overall","index":null,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":1.1,"outputPerMillion":4.4,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":122,"timeToFirstToken":6.413,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":{"elo":1353.170256392411,"eloUpper":1357.1303130512374,"eloLower":1349.210199733585,"votes":44801,"rank":209,"rankUpper":null,"rankLower":null,"arenaName":"o4-mini-2025-04-16","organization":"openai","updatedAt":"2026-11-09"},"quality":{"coveragePercent":100,"benchmarkCount":29,"categoryCount":8,"confidence":"high","state":"available"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:sierra-research","name":"Sierra Research","url":"https://github.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:swe-bench-team","name":"SWE-bench team","url":"https://github.com","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:arc-prize-foundation","name":"ARC Prize Foundation","url":"https://arcprize.org","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"deepseek-r1-0528","providerId":"deepseek","slug":"deepseek-r1-0528","name":"DeepSeek-R1-0528","provider":"DeepSeek","description":"DeepSeek-R1-0528 is a reasoning model from DeepSeek in the DeepSeek-R1 family. 11 benchmarks count toward its index, in 4 categories.","logoUrl":null,"releaseDate":"2025-05-28","contextWindowTokens":163840,"modalities":["text"],"openWeights":true,"details":{"apiId":"deepseek/deepseek-r1-0528","maxOutputTokens":32768,"knowledgeCutoff":"2025-03-31","retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.35,"weightsUrl":"https://huggingface.co/deepseek-ai/DeepSeek-R1-0528"},"reasoning":true,"overallScore":44.6182489996421,"overallScoreRange":{"low":35.74281055781966,"high":53.49368744146454},"badges":[],"capabilityScores":{"agentic":null,"coding":46.937521299605926,"reasoning":38.2205539358068,"multimodal":null,"knowledge":45.990677612881704,"multilingual":null,"instruction":null,"math":45.909698000836784},"capabilityCounts":{"knowledge":5,"math":2,"coding":2,"reasoning":2},"benchmarks":[{"benchmarkId":"mmluRedux","benchmarkName":"MMLU-Redux","category":"knowledge","score":93.4,"unit":"percent","family":"mmluRedux","index":48.6032804142874,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aime2024","benchmarkName":"American Invitational Mathematics Examination 2024","category":"math","score":91.4,"unit":"percent","family":"aime2024","index":45.33241531253401,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aime2025","benchmarkName":"American Invitational Mathematics Examination 2025","category":"math","score":87.5,"unit":"percent","family":"aime2025","index":46.486980689139564,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmluPro","benchmarkName":"Massive Multitask Language Understanding Professional","category":"knowledge","score":85,"unit":"percent","family":"mmlu-pro","index":51.57055418848201,"aboveFit":false,"run":null,"source":"Yubo Wang, Xueguang Ma, Ge Zhang, Yuansheng Ni, Abhranil Chandra, Shiguang Guo, Weiming Ren, Aaran Arulraj, Xuan He, Ziyan Jiang, Tianle Li, Max Ku, Kai Wang, Alex Zhuang, Rongqi Fan, Xiang Yue, Wenhu Chen","sourceUrl":"https://arxiv.org/abs/2406.01574","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gpqa","benchmarkName":"Graduate-Level Google-Proof Q&A","category":"knowledge","score":81,"unit":"percent","family":"gpqa-diamond","index":51.500087406352115,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"liveCodeBench","benchmarkName":"LiveCodeBench: Holistic and Contamination Free Evaluation of Large Language Models for Code","category":"coding","score":73.3,"unit":"percent","family":"livecodebench","index":50.163798589399555,"aboveFit":false,"run":null,"source":"Naman Jain, King Han, Alex Gu, Wen-Ding Li, Fanjia Yan, Tianjun Zhang, Sida Wang, Armando Solar-Lezama, Koushik Sen, Ion Stoica","sourceUrl":"https://arxiv.org/abs/2403.07974","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"sweVerified","benchmarkName":"Software Engineering Benchmark Verified","category":"coding","score":57.6,"unit":"percent","family":"swe-bench-verified","index":43.711244009812305,"aboveFit":false,"run":null,"source":"Carlos E. Jimenez, John Yang, Alexander Wettig, Shunyu Yao, Kexin Pei, Ofir Press, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2310.06770","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"simpleQa","benchmarkName":"Measuring Short-Form Factuality in Large Language Models","category":"knowledge","score":27.8,"unit":"percent","family":"simpleQa","index":36.134540154946166,"aboveFit":false,"run":null,"source":"Jason Wei, Najoung Kim, Hyung Won Chung, Yu-An Chung, Siddhartha Papay, Yifeng Lu, Hannaneh Hajishirzi, Luke Zettlemoyer","sourceUrl":"https://arxiv.org/abs/2411.04368","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"arc-prize:v1-public-eval:overall","benchmarkName":"ARC-AGI-1 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":26.979999999999997,"unit":"percent","family":"arc-prize:v1-public-eval:overall","index":null,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.047,"latency":null},{"benchmarkId":"arc-prize:v1-semi-private:overall","benchmarkName":"ARC-AGI-1 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":21.21,"unit":"percent","family":"arc-prize:v1-semi-private:overall","index":36.738361604277756,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.0464,"latency":null},{"benchmarkId":"hle","benchmarkName":"Humanity's Last Exam","category":"knowledge","score":17.7,"unit":"percent","family":"hle","index":42.14492590034084,"aboveFit":false,"run":null,"source":"Center for AI Safety, Scale AI, and thousands of expert contributors","sourceUrl":"https://lastexam.ai/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"arc-prize:v2-semi-private:overall","benchmarkName":"ARC-AGI-2 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":1.1199999999999999,"unit":"percent","family":"arc-prize:v2-semi-private:overall","index":39.70274626733584,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.0527,"latency":null},{"benchmarkId":"arc-prize:v2-public-eval:overall","benchmarkName":"ARC-AGI-2 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":0.29,"unit":"percent","family":"arc-prize:v2-public-eval:overall","index":null,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.0495,"latency":null}],"price":{"inputPerMillion":0.5,"outputPerMillion":2.1500000000000004,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":20,"timeToFirstToken":1.947,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":{"elo":1427.700886188908,"eloUpper":1433.356884378025,"eloLower":1422.0448879997912,"votes":18153,"rank":101,"rankUpper":null,"rankLower":null,"arenaName":"DeepSeek-R1-0528","organization":"deepseek","updatedAt":"2026-11-09"},"quality":{"coveragePercent":60,"benchmarkCount":11,"categoryCount":4,"confidence":"low","state":"available"},"hosts":[{"host":"deepinfra","name":"Deep Infra","kind":"host","url":"https://deepinfra.com/models","inputPerMillion":0.5,"outputPerMillion":2.15,"cacheReadPerMillion":0.35,"contextTokens":163840,"maxOutputTokens":64000,"deprecated":false},{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":0.5,"outputPerMillion":2.15,"cacheReadPerMillion":0.35,"contextTokens":163840,"maxOutputTokens":32768,"deprecated":false},{"host":"siliconflow","name":"SiliconFlow","kind":"host","url":"https://cloud.siliconflow.com/models","inputPerMillion":0.5,"outputPerMillion":2.18,"cacheReadPerMillion":null,"contextTokens":164000,"maxOutputTokens":164000,"deprecated":false},{"host":"io-net","name":"IO.NET","kind":"host","url":"https://io.net/docs/guides/intelligence/io-intelligence","inputPerMillion":0.523667,"outputPerMillion":2.205333,"cacheReadPerMillion":0.261834,"contextTokens":128000,"maxOutputTokens":4096,"deprecated":false},{"host":"novita-ai","name":"Novita AI","kind":"host","url":"https://novita.ai/docs/guides/introduction","inputPerMillion":0.7,"outputPerMillion":2.5,"cacheReadPerMillion":0.35,"contextTokens":163840,"maxOutputTokens":32768,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:arc-prize-foundation","name":"ARC Prize Foundation","url":"https://arcprize.org","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"o1","providerId":"openai","slug":"o1","name":"o1","provider":"OpenAI","description":"o1 is a reasoning model from OpenAI. 20 benchmarks count toward its index, in 7 categories.","logoUrl":null,"releaseDate":"2024-12-01","contextWindowTokens":200000,"modalities":["text","image","file"],"openWeights":false,"details":{"apiId":"openai/o1","maxOutputTokens":100000,"knowledgeCutoff":"2023-10-31","retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":7.5,"weightsUrl":null},"reasoning":true,"overallScore":44.528930359884455,"overallScoreRange":{"low":40.4132084402673,"high":48.644652279501614},"badges":[],"capabilityScores":{"agentic":44.3291169516314,"coding":31.049879638092456,"reasoning":43.63266410222036,"multimodal":55.8977412952405,"knowledge":46.78738736568096,"multilingual":null,"instruction":51.79254074340875,"math":43.8753081528079},"capabilityCounts":{"math":6,"instruction":2,"knowledge":6,"multimodal":1,"reasoning":3,"agentic":1,"coding":1},"benchmarks":[{"benchmarkId":"aaMath500","benchmarkName":"Artificial Analysis MATH-500","category":"math","score":97,"unit":"percent","family":"math-500","index":46.63232986207564,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/math-500","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:medqa:overall","benchmarkName":"MedQA","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":96.517,"unit":"percent","family":"vals-ai:medqa:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medqa","measuredAt":"2026-04-16","stderr":0.33,"costPerTest":0.05282,"latency":11.152},{"benchmarkId":"epoch-ai:epoch-math-level-5:overall","benchmarkName":"MATH level 5","category":"math","domain":null,"eligible":true,"ranked":true,"score":94.7129909365559,"unit":"percent","family":"epoch-ai:epoch-math-level-5:overall","index":45.00525582160437,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0.6152192005965585,"costPerTest":null,"latency":null},{"benchmarkId":"ifeval","benchmarkName":"Instruction-Following Eval","category":"instruction","score":92.2,"unit":"percent","family":"ifeval","index":48.86535587223853,"aboveFit":false,"run":null,"source":"Jeffrey Zhou, Tianjian Lu, Swaroop Mishra, Siddhartha Brahma, Sujoy Basu, Yi Luan, Denny Zhou, Le Hou","sourceUrl":"https://arxiv.org/abs/2311.07911","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmlu","benchmarkName":"Massive Multitask Language Understanding","category":"knowledge","score":91.8,"unit":"percent","family":"mmlu","index":41.140767140630146,"aboveFit":false,"run":null,"source":"Dan Hendrycks, Collin Burns, Steven Basart, Andy Zou, Mantas Mazeika, Dawn Song, Jacob Steinhardt","sourceUrl":"https://arxiv.org/abs/2009.03300","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:math500:overall","benchmarkName":"MATH 500","category":"math","domain":null,"eligible":true,"ranked":true,"score":90.4,"unit":"percent","family":"math-500","index":39.65772566003844,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/math500","measuredAt":"2026-01-09","stderr":2.591,"costPerTest":0.095269,"latency":23.537},{"benchmarkId":"vals-ai:mgsm:overall","benchmarkName":"MGSM","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":89.309,"unit":"percent","family":"vals-ai:mgsm:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mgsm","measuredAt":"2026-01-09","stderr":1.779,"costPerTest":0.057248,"latency":11.207},{"benchmarkId":"vals-ai:mmlu_pro:overall","benchmarkName":"MMLU Pro","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":83.488,"unit":"percent","family":"mmlu-pro","index":49.66561517022978,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","measuredAt":"2026-09-01","stderr":0.364,"costPerTest":0.116223,"latency":26.867},{"benchmarkId":"vals-ai:legal_bench:overall","benchmarkName":"LegalBench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":80.393,"unit":"percent","family":"vals-ai:legal_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","measuredAt":"2026-10-13","stderr":0.578,"costPerTest":null,"latency":8.338},{"benchmarkId":"vals-ai:mmmu:overall","benchmarkName":"MMMU Pro","category":"multimodal","domain":null,"eligible":true,"ranked":true,"score":77.412,"unit":"percent","family":"mmmu-pro","index":55.8977412952405,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmmu","measuredAt":"2026-09-01","stderr":1.005,"costPerTest":0.14586,"latency":26.415},{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":76.76767676767676,"unit":"percent","family":"gpqa-diamond","index":47.61831618623706,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":3.0088629490217444,"costPerTest":null,"latency":null},{"benchmarkId":"gpqa","benchmarkName":"Graduate-Level Google-Proof Q&A","category":"knowledge","score":75.7,"unit":"percent","family":"gpqa-diamond","index":46.63907222927726,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":74.7,"unit":"percent","family":"gpqa-diamond","index":45.72189955435748,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:tax_eval_v2:overall","benchmarkName":"TaxEval v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":74.284,"unit":"percent","family":"vals-ai:tax_eval_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","measuredAt":"2026-09-01","stderr":0.861,"costPerTest":0.12173,"latency":19.477},{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":73.33333333333333,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":51.583987563657715,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":6.666666666666667,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:gpqa:overall","benchmarkName":"GPQA Diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":73.232,"unit":"percent","family":"gpqa-diamond","index":44.37549006757523,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","measuredAt":"2026-09-01","stderr":2.231,"costPerTest":0.252632,"latency":40.176},{"benchmarkId":"vals-ai:aime:overall","benchmarkName":"AIME","category":"math","domain":null,"eligible":true,"ranked":true,"score":71.458,"unit":"percent","family":"vals-ai:aime:overall","index":46.73298193934151,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/aime","measuredAt":"2026-04-16","stderr":1.458,"costPerTest":0.658977,"latency":177.028},{"benchmarkId":"aaIfBench","benchmarkName":"Artificial Analysis IFBench","category":"instruction","score":70.3,"unit":"percent","family":"ifbench-family","index":54.71972561457896,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":65,"unit":"percent","family":"lcr","index":51.89328362223862,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"tau2Bench","benchmarkName":"τ²-Bench Tool-Agent-User Evaluation","category":"agentic","score":62.6,"unit":"percent","family":"tau2-bench","index":44.3291169516314,"aboveFit":false,"run":null,"source":"Victor Barres, Honghua Dong, Soham Ray, Xujie Si, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2506.07982","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:lcb:overall","benchmarkName":"LiveCodeBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":50.264,"unit":"percent","family":"livecodebench","index":31.049879638092456,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","measuredAt":"2026-09-01","stderr":1.393,"costPerTest":0.328067,"latency":92.082},{"benchmarkId":"epoch-ai:epoch-simpleqa-verified:overall","benchmarkName":"SimpleQA Verified","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":41.099999999999994,"unit":"percent","family":"epoch-ai:epoch-simpleqa-verified:overall","index":58.158980546207815,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.556667341859915,"costPerTest":null,"latency":null},{"benchmarkId":"omniscienceAccuracy","benchmarkName":"Artificial Analysis Omniscience Accuracy","category":"knowledge","score":34.5,"unit":"percent","family":"omniscienceAccuracy","index":54.16878191667858,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tiers-1-3-v2-private:overall","benchmarkName":"FrontierMath-Tiers-1-3-v2-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":14.736842105263156,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tiers-1-3-v2-private:overall","index":38.65145836656703,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.1034091168852487,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-chess-puzzles:overall","benchmarkName":"Chess Puzzles","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":10,"unit":"percent","family":"epoch-ai:epoch-chess-puzzles:overall","index":41.574827676639885,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":3.015113445777636,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-2025-02-28-private:overall","benchmarkName":"FrontierMath-2025-02-28-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":9.310344827586208,"unit":"percent","family":"epoch-ai:epoch-frontiermath-2025-02-28-private:overall","index":38.13313746461973,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.7092785280147835,"costPerTest":null,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":7,"unit":"percent","family":"hle","index":31.409693528522038,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":0.3,"unit":"percent","family":"critpt","index":37.42988100778257,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-2025-02-28-public:overall","benchmarkName":"FrontierMath-2025-02-28-Public","category":"math","domain":null,"eligible":false,"ranked":true,"score":0,"unit":"percent","family":"epoch-ai:epoch-frontiermath-2025-02-28-public:overall","index":null,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":15,"outputPerMillion":60,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":97,"timeToFirstToken":1.393,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":{"elo":1365.8268013074762,"eloUpper":1370.3030838942814,"eloLower":1361.350518720671,"votes":27807,"rank":196,"rankUpper":null,"rankLower":null,"arenaName":"o1-2024-12-17","organization":"openai","updatedAt":"2026-11-09"},"quality":{"coveragePercent":95,"benchmarkCount":20,"categoryCount":7,"confidence":"medium","state":"available"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gemma-4-26b-a4b","providerId":"google","slug":"gemma-4-26b-a4b","name":"Gemma 4 26B A4B","provider":"Google","description":"Gemma 4 26B A4B is a reasoning model from Google in the Gemma 4 family. 17 benchmarks count toward its index, in 8 categories.","logoUrl":null,"releaseDate":"2026-04-02","contextWindowTokens":256000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":true,"overallScore":44.45066512200727,"overallScoreRange":{"low":40.52886406516977,"high":48.372466178844775},"badges":[],"capabilityScores":{"agentic":34.021464189863764,"coding":45.309496335470726,"reasoning":42.139438728320755,"multimodal":49.57524369301413,"knowledge":46.67976586013999,"multilingual":36.290867691610515,"instruction":56.682197905720926,"math":56.75549019239625},"capabilityCounts":{"knowledge":3,"math":1,"coding":3,"multimodal":3,"instruction":1,"reasoning":3,"multilingual":1,"agentic":2},"benchmarks":[{"benchmarkId":"mmluPro","benchmarkName":"Massive Multitask Language Understanding Professional","category":"knowledge","score":82.6,"unit":"percent","family":"mmlu-pro","index":48.54684146109751,"aboveFit":false,"run":null,"source":"Yubo Wang, Xueguang Ma, Ge Zhang, Yuansheng Ni, Abhranil Chandra, Shiguang Guo, Weiming Ren, Aaran Arulraj, Xuan He, Ziyan Jiang, Tianle Li, Max Ku, Kai Wang, Alex Zhuang, Rongqi Fan, Xiang Yue, Wenhu Chen","sourceUrl":"https://arxiv.org/abs/2406.01574","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gpqa","benchmarkName":"Graduate-Level Google-Proof Q&A","category":"knowledge","score":82.3,"unit":"percent","family":"gpqa-diamond","index":52.69241188374784,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":82.22222222222221,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":56.75549019239625,"aboveFit":false,"run":"minimal effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":5.763774795025093,"costPerTest":null,"latency":null},{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":79.2,"unit":"percent","family":"gpqa-diamond","index":49.84917659149651,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"liveCodeBenchV6","benchmarkName":"LiveCodeBench v6","category":"coding","score":77.1,"unit":"percent","family":"liveCodeBenchV6","index":47.72057714762266,"aboveFit":false,"run":null,"source":"LiveCodeBench maintainers","sourceUrl":"https://github.com/LiveCodeBench/LiveCodeBench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmmuPro","benchmarkName":"Massive Multi-discipline Multimodal Understanding Pro","category":"multimodal","score":73.8,"unit":"percent","family":"mmmu-pro","index":51.197040193630066,"aboveFit":false,"run":null,"source":"MMMU-Pro authors","sourceUrl":"https://arxiv.org/abs/2409.02813","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":73.23232323232324,"unit":"percent","family":"gpqa-diamond","index":44.37578652742975,"aboveFit":false,"run":"minimal effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":3.1544498882702823,"costPerTest":null,"latency":null},{"benchmarkId":"aaIfBench","benchmarkName":"Artificial Analysis IFBench","category":"instruction","score":72.4,"unit":"percent","family":"ifbench-family","index":56.682197905720926,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"extractBenchLlamaIndex","benchmarkName":"ExtractBench: schema-guided enterprise document extraction (LlamaIndex leaderboard)","category":"multimodal","score":70.27,"unit":"percent","family":"extractBenchLlamaIndex","index":45.18166309964661,"aboveFit":false,"run":null,"source":"Boyang Zhang, Adrian Lyjak, Eli Stewart, Zhaoqi Li and Simon Suo (LlamaIndex)","sourceUrl":"https://arxiv.org/abs/2607.29677","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaMmmuPro","benchmarkName":"Artificial Analysis MMMU-Pro","category":"multimodal","score":69.2,"unit":"percent","family":"mmmu-pro","index":45.21054377408191,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmmu-pro","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":65.7,"unit":"percent","family":"lcr","index":52.37353999014858,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-french:overall","benchmarkName":"EuroEval French","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":65.22625000000001,"unit":"percent","family":"euroeval","index":60.71724857596087,"aboveFit":false,"run":"gemma-4-26B-A4B-it","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/french","measuredAt":null,"stderr":0.4842761996010128,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-portuguese:overall","benchmarkName":"EuroEval Portuguese","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":61.96625000000001,"unit":"percent","family":"euroeval","index":56.29417868059064,"aboveFit":false,"run":"gemma-4-26B-A4B-it","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/portuguese","measuredAt":null,"stderr":0.4920540493685628,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-italian:overall","benchmarkName":"EuroEval Italian","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":61.095000000000006,"unit":"percent","family":"euroeval","index":55.11209289950435,"aboveFit":false,"run":"gemma-4-26B-A4B-it","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/italian","measuredAt":null,"stderr":0.515935134004266,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":58.815,"unit":"percent","family":"euroeval","index":52.01865751255828,"aboveFit":false,"run":"gemma-4-26B-A4B-it","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":0.644843731069784,"costPerTest":null,"latency":null},{"benchmarkId":"parseBenchLlamaIndex","benchmarkName":"ParseBench: document parsing for AI agents (LlamaIndex leaderboard)","category":"multimodal","score":58.5,"unit":"percent","family":"parseBenchLlamaIndex","index":55.3402759955398,"aboveFit":false,"run":null,"source":"Boyang Zhang, Sebastián G. Acosta, Preston Carlson, Sacha Bron, Pierre-Loïc Doulcet, Daniel B. Ospina and Simon Suo (LlamaIndex)","sourceUrl":"https://arxiv.org/abs/2604.08538","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-dutch:overall","benchmarkName":"EuroEval Dutch","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":56.35888888888889,"unit":"percent","family":"euroeval","index":48.686279869846665,"aboveFit":false,"run":"gemma-4-26B-A4B-it","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/dutch","measuredAt":null,"stderr":0.5039204326767628,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":55.48285714285714,"unit":"percent","family":"euroeval","index":47.4977063596437,"aboveFit":false,"run":"gemma-4-26B-A4B-it","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.7036972331325222,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-spanish:overall","benchmarkName":"EuroEval Spanish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":49.43375,"unit":"percent","family":"euroeval","index":39.2904598266865,"aboveFit":false,"run":"gemma-4-26B-A4B-it","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/spanish","measuredAt":null,"stderr":0.5402545696243577,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":49.142500000000005,"unit":"percent","family":"euroeval","index":38.895300591402275,"aboveFit":false,"run":"gemma-4-26B-A4B-it","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.6085061113086704,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-dutch:overall","benchmarkName":"EuroEval Dutch","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":45.30333333333333,"unit":"percent","family":"euroeval","index":33.68643479181876,"aboveFit":false,"run":"gemma-4-26B-A4B","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/dutch","measuredAt":null,"stderr":0.6977530073451919,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-french:overall","benchmarkName":"EuroEval French","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":44.443749999999994,"unit":"percent","family":"euroeval","index":32.52017799297561,"aboveFit":false,"run":"gemma-4-26B-A4B","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/french","measuredAt":null,"stderr":0.9847874136076273,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":44.443749999999994,"unit":"percent","family":"euroeval","index":32.52017799297561,"aboveFit":false,"run":"gemma-4-26B-A4B","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":0.9404362352121487,"costPerTest":null,"latency":null},{"benchmarkId":"mrcrv2","benchmarkName":"MRCRv2","category":"reasoning","score":44.1,"unit":"percent","family":"mrcrv2","index":37.19566018204078,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"tau2Bench","benchmarkName":"τ²-Bench Tool-Agent-User Evaluation","category":"agentic","score":43.6,"unit":"percent","family":"tau2-bench","index":31.72122782512089,"aboveFit":false,"run":null,"source":"Victor Barres, Honghua Dong, Soham Ray, Xujie Si, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2506.07982","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-portuguese:overall","benchmarkName":"EuroEval Portuguese","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":43.5475,"unit":"percent","family":"euroeval","index":31.304172964225444,"aboveFit":false,"run":"gemma-4-26B-A4B","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/portuguese","measuredAt":null,"stderr":0.8986404662043658,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-italian:overall","benchmarkName":"EuroEval Italian","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":40.980000000000004,"unit":"percent","family":"euroeval","index":27.82066622914475,"aboveFit":false,"run":"gemma-4-26B-A4B","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/italian","measuredAt":null,"stderr":0.9092742641799558,"costPerTest":null,"latency":null},{"benchmarkId":"aaSciCode","benchmarkName":"Artificial Analysis SciCode","category":"coding","score":40,"unit":"percent","family":"scicode-family","index":47.987444299100595,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/scicode","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":38.48428571428572,"unit":"percent","family":"euroeval","index":24.434556190927484,"aboveFit":false,"run":"gemma-4-26B-A4B","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.8545783063199377,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":37.70625,"unit":"percent","family":"euroeval","index":23.378940747559373,"aboveFit":false,"run":"gemma-4-26B-A4B","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":1.009246315326442,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-spanish:overall","benchmarkName":"EuroEval Spanish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":35.385000000000005,"unit":"percent","family":"euroeval","index":20.229538601967924,"aboveFit":false,"run":"gemma-4-26B-A4B","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/spanish","measuredAt":null,"stderr":0.6154012918413481,"costPerTest":null,"latency":null},{"benchmarkId":"scarfBench","benchmarkName":"ScarfBench: Cross-Framework Application Migration in Enterprise Java","category":"coding","score":24.4,"unit":"percent","family":"scarfBench","index":40.22046755968893,"aboveFit":false,"run":null,"source":"Advait Pavuluri, Bridget McGinn, Ashita Saxena, George Safta, Srikanth Tamilselvam, Raju Pavuluri, Michele Merler, Baishakhi Ray, Rahul Krishna (IBM Research)","sourceUrl":"https://arxiv.org/abs/2605.06754","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":19.3,"unit":"percent","family":"hle","index":43.75019429238851,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hle","benchmarkName":"Humanity's Last Exam","category":"knowledge","score":17.2,"unit":"percent","family":"hle","index":41.64327952782594,"aboveFit":false,"run":null,"source":"Center for AI Safety, Scale AI, and thousands of expert contributors","sourceUrl":"https://lastexam.ai/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hleNoTools","benchmarkName":"Humanity's Last Exam without tools","category":"knowledge","score":8.7,"unit":"percent","family":"hle","index":33.115291195072686,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gdpvalAaNormalized","benchmarkName":"GDPval-AA normalized","category":"agentic","score":3.2,"unit":"percent","family":"gdpvalAaNormalized","index":36.32170055460665,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":0,"unit":"percent","family":"critpt","index":36.8491160127729,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":0,"outputPerMillion":0,"batch":null,"freeTier":true,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":{"elo":1434.0558280701962,"eloUpper":1441.4876467248037,"eloLower":1426.6240094155892,"votes":6075,"rank":92,"rankUpper":null,"rankLower":null,"arenaName":"Gemma 4 26B A4B","organization":"google","updatedAt":"2026-11-09"},"quality":{"coveragePercent":100,"benchmarkCount":17,"categoryCount":8,"confidence":"medium","state":"available"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"grok-3-mini","providerId":"xai","slug":"grok-3-mini","name":"Grok 3 Mini","provider":"xAI","description":"Grok 3 Mini is a reasoning model from xAI. 8 benchmarks count toward its index, in 4 categories.","logoUrl":null,"releaseDate":"2025-02-19","contextWindowTokens":128000,"modalities":["text"],"openWeights":false,"details":null,"reasoning":true,"overallScore":44.27227081063106,"overallScoreRange":{"low":34.555312484737826,"high":53.9892291365243},"badges":[],"capabilityScores":{"agentic":null,"coding":51.408410432903665,"reasoning":36.97717826796044,"multimodal":null,"knowledge":48.65916820671421,"multilingual":null,"instruction":null,"math":43.87144274238368},"capabilityCounts":{"math":4,"knowledge":1,"coding":1,"reasoning":2},"benchmarks":[{"benchmarkId":"aime2024","benchmarkName":"American Invitational Mathematics Examination 2024","category":"math","score":89.5,"unit":"percent","family":"aime2024","index":44.4502851811599,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-math-level-5:overall","benchmarkName":"MATH level 5","category":"math","domain":null,"eligible":true,"ranked":true,"score":88.06646525679758,"unit":"percent","family":"epoch-ai:epoch-math-level-5:overall","index":41.776670226342375,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0.8912712785411623,"costPerTest":null,"latency":null},{"benchmarkId":"gpqa","benchmarkName":"Graduate-Level Google-Proof Q&A","category":"knowledge","score":80.3,"unit":"percent","family":"gpqa-diamond","index":50.858066533908264,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":77.77777777777779,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":54.16973887802699,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":6.267511942419625,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":75.5050505050505,"unit":"percent","family":"gpqa-diamond","index":46.46026987952017,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.767552182004143,"costPerTest":null,"latency":null},{"benchmarkId":"liveCodeBench","benchmarkName":"LiveCodeBench: Holistic and Contamination Free Evaluation of Large Language Models for Code","category":"coding","score":74.8,"unit":"percent","family":"livecodebench","index":51.408410432903665,"aboveFit":false,"run":null,"source":"Naman Jain, King Han, Alex Gu, Wen-Ding Li, Fanjia Yan, Tianjun Zhang, Sida Wang, Armando Solar-Lezama, Koushik Sen, Ion Stoica","sourceUrl":"https://arxiv.org/abs/2403.07974","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"arc-prize:v1-public-eval:overall","benchmarkName":"ARC-AGI-1 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":17.62,"unit":"percent","family":"arc-prize:v1-public-eval:overall","index":null,"aboveFit":false,"run":"low effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.0087,"latency":null},{"benchmarkId":"arc-prize:v1-semi-private:overall","benchmarkName":"ARC-AGI-1 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":16.5,"unit":"percent","family":"arc-prize:v1-semi-private:overall","index":34.57300455926835,"aboveFit":false,"run":"low effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.0099,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-2025-02-28-private:overall","benchmarkName":"FrontierMath-2025-02-28-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":5.862068965517241,"unit":"percent","family":"epoch-ai:epoch-frontiermath-2025-02-28-private:overall","index":35.08907668400545,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.3818435156390343,"costPerTest":null,"latency":null},{"benchmarkId":"arc-prize:v2-semi-private:overall","benchmarkName":"ARC-AGI-2 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":0.42,"unit":"percent","family":"arc-prize:v2-semi-private:overall","index":39.38135197665253,"aboveFit":false,"run":"low effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.0131,"latency":null},{"benchmarkId":"arc-prize:v2-public-eval:overall","benchmarkName":"ARC-AGI-2 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":0,"unit":"percent","family":"arc-prize:v2-public-eval:overall","index":null,"aboveFit":false,"run":"low effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.013,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-public:overall","benchmarkName":"FrontierMath-Tier-4-2025-07-01-Public","category":"math","domain":null,"eligible":false,"ranked":true,"score":0,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-public:overall","index":null,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-2025-02-28-public:overall","benchmarkName":"FrontierMath-2025-02-28-Public","category":"math","domain":null,"eligible":false,"ranked":true,"score":0,"unit":"percent","family":"epoch-ai:epoch-frontiermath-2025-02-28-public:overall","index":null,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":0.3,"outputPerMillion":0.5,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":65,"timeToFirstToken":32.38,"source":"BenchLM","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":{"elo":1369.5257154154374,"eloUpper":1374.969098737741,"eloLower":1364.082332093134,"votes":15574,"rank":190,"rankUpper":null,"rankLower":null,"arenaName":"Grok 3 Mini-high","organization":"xai","updatedAt":"2026-11-09"},"quality":{"coveragePercent":60,"benchmarkCount":8,"categoryCount":4,"confidence":"low","state":"available"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:arc-prize-foundation","name":"ARC Prize Foundation","url":"https://arcprize.org","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"qwen3-5-4b","providerId":"alibaba","slug":"qwen3-5-4b","name":"Qwen3.5-4B","provider":"Alibaba","description":"Qwen3.5-4B is a reasoning model from Alibaba. 9 benchmarks count toward its index, in 7 categories.","logoUrl":null,"releaseDate":null,"contextWindowTokens":262000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":true,"overallScore":44.26107506773611,"overallScoreRange":{"low":36.87107506773611,"high":51.65107506773611},"badges":[],"capabilityScores":{"agentic":63.90452375331877,"coding":null,"reasoning":43.6851192919201,"multimodal":40.26517716662909,"knowledge":41.1211782316367,"multilingual":35.537012412280745,"instruction":37.61818136319906,"math":41.40259176332871},"capabilityCounts":{"agentic":1,"knowledge":2,"multimodal":1,"reasoning":2,"math":1,"multilingual":1,"instruction":1},"benchmarks":[{"benchmarkId":"tau2Bench","benchmarkName":"τ²-Bench Tool-Agent-User Evaluation","category":"agentic","score":92.1,"unit":"percent","family":"tau2-bench","index":63.90452375331877,"aboveFit":true,"run":null,"source":"Victor Barres, Honghua Dong, Soham Ray, Xujie Si, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2506.07982","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":77.1,"unit":"percent","family":"gpqa-diamond","index":47.92311397416495,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaMmmuPro","benchmarkName":"Artificial Analysis MMMU-Pro","category":"multimodal","score":65.4,"unit":"percent","family":"mmmu-pro","index":40.26517716662909,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmmu-pro","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":63,"unit":"percent","family":"lcr","index":50.521122571067295,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":55.833333333333336,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":41.40259176332871,"aboveFit":false,"run":"none effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":6.16226296489243,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-italian:overall","benchmarkName":"EuroEval Italian","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":53.315000000000005,"unit":"percent","family":"euroeval","index":44.55642302650421,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/italian","measuredAt":null,"stderr":0.6883540150242461,"costPerTest":null,"latency":null},{"benchmarkId":"aaIfBench","benchmarkName":"Artificial Analysis IFBench","category":"instruction","score":52,"unit":"percent","family":"ifbench-family","index":37.61818136319906,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-french:overall","benchmarkName":"EuroEval French","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":51.50625,"unit":"percent","family":"euroeval","index":42.102365458022334,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/french","measuredAt":null,"stderr":0.5316601358010585,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":48.87624999999999,"unit":"percent","family":"euroeval","index":38.53406060378192,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":1.061966777964358,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-portuguese:overall","benchmarkName":"EuroEval Portuguese","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":48.69125,"unit":"percent","family":"euroeval","index":38.28305817106919,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/portuguese","measuredAt":null,"stderr":0.6118542514357485,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-dutch:overall","benchmarkName":"EuroEval Dutch","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":44.64333333333334,"unit":"percent","family":"euroeval","index":32.79096665349228,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/dutch","measuredAt":null,"stderr":0.7110659707895005,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":44.2,"unit":"percent","family":"euroeval","index":32.18946532825277,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.8144937077718895,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":39.7125,"unit":"percent","family":"euroeval","index":26.100960372585917,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.7876478035899039,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-spanish:overall","benchmarkName":"EuroEval Spanish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":36.79875,"unit":"percent","family":"euroeval","index":22.147672057360452,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/spanish","measuredAt":null,"stderr":0.6809873346839865,"costPerTest":null,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":9.9,"unit":"percent","family":"hle","index":34.31924248910844,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":0,"unit":"percent","family":"critpt","index":36.8491160127729,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":20,"timeToFirstToken":102.46,"source":"BenchLM","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":80,"benchmarkCount":9,"categoryCount":7,"confidence":"low","state":"available"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"deepseek-v3-2","providerId":"deepseek","slug":"deepseek-v3-2","name":"DeepSeek V3.2","provider":"DeepSeek","description":"DeepSeek V3.2 is a standard model from DeepSeek. 23 benchmarks count toward its index, in 6 categories.","logoUrl":null,"releaseDate":"2025-12-01","contextWindowTokens":163840,"modalities":["text"],"openWeights":true,"details":{"apiId":"deepseek/deepseek-v3.2","maxOutputTokens":147456,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.13,"weightsUrl":"https://huggingface.co/deepseek-ai/DeepSeek-V3.2"},"reasoning":false,"overallScore":44.20035112269002,"overallScoreRange":{"low":39.32867304738712,"high":49.072029197992926},"badges":[],"capabilityScores":{"agentic":43.08778573293934,"coding":48.766327271008024,"reasoning":42.86862136328628,"multimodal":null,"knowledge":43.209101116564575,"multilingual":null,"instruction":34.814649518710546,"math":46.816261505864254},"capabilityCounts":{"agentic":6,"knowledge":4,"coding":4,"math":4,"reasoning":4,"instruction":1},"benchmarks":[{"benchmarkId":"tau2-bench:telecom:overall","benchmarkName":"τ²-bench Telecom","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":96.2,"unit":"percent","family":"tau2-bench","index":66.62517351219736,"aboveFit":true,"run":"DeepSeek","source":"Sierra Research","sourceUrl":"https://github.com/sierra-research/tau2-bench","measuredAt":"2026-03-02","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:mgsm:overall","benchmarkName":"MGSM","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":90.873,"unit":"percent","family":"vals-ai:mgsm:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mgsm","measuredAt":"2026-01-09","stderr":1.768,"costPerTest":0.00015,"latency":8.966},{"benchmarkId":"vals-ai:medqa:overall","benchmarkName":"MedQA","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":89.45,"unit":"percent","family":"vals-ai:medqa:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medqa","measuredAt":"2026-04-16","stderr":0.28,"costPerTest":0.000234,"latency":13.297},{"benchmarkId":"vals-ai:mmlu_pro:overall","benchmarkName":"MMLU Pro","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":83.056,"unit":"percent","family":"mmlu-pro","index":49.12134687930057,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","measuredAt":"2026-09-01","stderr":0.381,"costPerTest":0.000283,"latency":40.772},{"benchmarkId":"tau2-bench:retail:overall","benchmarkName":"τ²-bench Retail","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":81.1,"unit":"percent","family":"tau2-bench","index":56.6052195221811,"aboveFit":false,"run":"DeepSeek","source":"Sierra Research","sourceUrl":"https://github.com/sierra-research/tau2-bench","measuredAt":"2026-04-30","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"tau2Bench","benchmarkName":"τ²-Bench Tool-Agent-User Evaluation","category":"agentic","score":78.9,"unit":"percent","family":"tau2-bench","index":55.14535867595358,"aboveFit":false,"run":null,"source":"Victor Barres, Honghua Dong, Soham Ray, Xujie Si, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2506.07982","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":77.31782106782109,"unit":"percent","family":"gpqa-diamond","index":48.12289350559231,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.01537212608744,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:gpqa:overall","benchmarkName":"GPQA Diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":76.263,"unit":"percent","family":"gpqa-diamond","index":47.1554404452571,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","measuredAt":"2026-09-01","stderr":2.161,"costPerTest":0.00065,"latency":46.828},{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":75.1,"unit":"percent","family":"gpqa-diamond","index":46.08876862432538,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"reactNativeEvals","benchmarkName":"React Native Evals","category":"coding","score":71.5,"unit":"percent","family":"reactNativeEvals","index":47.58901136696616,"aboveFit":false,"run":null,"source":"Callstack","sourceUrl":"https://rn-evals.vercel.app/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"swebench-experiments:swe-bench-verified:overall","benchmarkName":"SWE-bench Verified","category":"coding","domain":null,"eligible":true,"ranked":true,"score":70,"unit":"percent","family":"swe-bench-verified","index":52.85695797433008,"aboveFit":false,"run":"high effort · mini-SWE-agent","source":"SWE-bench team","sourceUrl":"https://github.com/SWE-bench/experiments/tree/main/evaluation/verified","measuredAt":"2026-09-01","stderr":null,"costPerTest":0.4478464965,"latency":null},{"benchmarkId":"vals-ai:lcb:overall","benchmarkName":"LiveCodeBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":69.856,"unit":"percent","family":"livecodebench","index":47.306169796714116,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","measuredAt":"2026-09-01","stderr":1.198,"costPerTest":0.000677,"latency":39.343},{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":68.3531746031746,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":48.68656086764572,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":7.53592220347252,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:legal_bench:overall","benchmarkName":"LegalBench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":65.962,"unit":"percent","family":"vals-ai:legal_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","measuredAt":"2026-10-13","stderr":0.419,"costPerTest":0.000086,"latency":2.185},{"benchmarkId":"vals-ai:aime:overall","benchmarkName":"AIME","category":"math","domain":null,"eligible":true,"ranked":true,"score":64.792,"unit":"percent","family":"vals-ai:aime:overall","index":43.81963490028354,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/aime","measuredAt":"2026-04-16","stderr":2.046,"costPerTest":0.001399,"latency":119.555},{"benchmarkId":"tau2-bench:airline:overall","benchmarkName":"τ²-bench Airline","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":63.8,"unit":"percent","family":"tau2-bench","index":45.125404685937326,"aboveFit":false,"run":"DeepSeek","source":"Sierra Research","sourceUrl":"https://github.com/sierra-research/tau2-bench","measuredAt":"2026-03-02","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"arc-prize:v1-public-eval:overall","benchmarkName":"ARC-AGI-1 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":61.62,"unit":"percent","family":"arc-prize:v1-public-eval:overall","index":null,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.07,"latency":null},{"benchmarkId":"sweRebench","benchmarkName":"SWE-Rebench","category":"coding","score":60.9,"unit":"percent","family":"sweRebench","index":null,"aboveFit":false,"run":null,"source":"Nebius","sourceUrl":"https://swe-rebench.com","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"swe-bench:swe-bench-multilingual:overall","benchmarkName":"SWE-bench Multilingual","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":59,"unit":"percent","family":"swe-bench-multilingual","index":null,"aboveFit":false,"run":"mini-SWE-agent","source":"SWE-bench team","sourceUrl":"https://www.swebench.com/","measuredAt":"2026-02-20","stderr":null,"costPerTest":0.3838255249,"latency":null},{"benchmarkId":"swebench-experiments:swe-bench-multilingual:overall","benchmarkName":"SWE-bench Multilingual","category":"coding","domain":null,"eligible":true,"ranked":true,"score":59,"unit":"percent","family":"swe-bench-multilingual","index":null,"aboveFit":false,"run":"mini-SWE-agent","source":"SWE-bench team","sourceUrl":"https://github.com/SWE-bench/experiments/tree/main/evaluation/multilingual","measuredAt":"2026-09-02","stderr":null,"costPerTest":0.3838255249,"latency":null},{"benchmarkId":"arc-prize:v1-semi-private:overall","benchmarkName":"ARC-AGI-1 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":56.99999999999999,"unit":"percent","family":"arc-prize:v1-semi-private:overall","index":53.19231672973139,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.08,"latency":null},{"benchmarkId":"vals-ai:terminal-bench:overall","benchmarkName":"Terminal-Bench 1.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":50,"unit":"percent","family":"vals-ai:terminal-bench:overall","index":54.402056932447785,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench","measuredAt":"2026-01-12","stderr":5.59,"costPerTest":0.024996,"latency":399.137},{"benchmarkId":"aaIfBench","benchmarkName":"Artificial Analysis IFBench","category":"instruction","score":49,"unit":"percent","family":"ifbench-family","index":34.814649518710546,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:corp_fin_v2:overall","benchmarkName":"CorpFin v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":47.941,"unit":"percent","family":"vals-ai:corp_fin_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","measuredAt":"2026-08-12","stderr":0.98,"costPerTest":0.036826,"latency":133.787},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":45.7,"unit":"percent","family":"lcr","index":38.65192947843538,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"clawEval","benchmarkName":"Claw-Eval","category":"agentic","score":40.2,"unit":"percent","family":"clawEval","index":23.225249230988076,"aboveFit":false,"run":null,"source":"Bowen Ye, Rang Li, Qibin Yang, Yuanxin Liu, Linli Yao, Hanglong Lv, Zhihui Xie, Chenxin An, Lei Li, Lingpeng Kong, Qi Liu, Zhifang Sui, Tong Yang","sourceUrl":"https://arxiv.org/abs/2604.06132","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:terminal-bench-2:overall","benchmarkName":"Terminal-Bench 2.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":34.831,"unit":"percent","family":"vals-ai:terminal-bench-2:overall","index":47.523401659939296,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2","measuredAt":"2026-06-04","stderr":5.079,"costPerTest":0.063298,"latency":804.541},{"benchmarkId":"gertLabs","benchmarkName":"Gert Labs Composite Game Benchmark","category":"agentic","score":29.57,"unit":"percent","family":"gertLabs","index":39.490082267621766,"aboveFit":false,"run":null,"source":"Gert Labs","sourceUrl":"https://gertlabs.com/rankings","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"omniscienceAccuracy","benchmarkName":"Artificial Analysis Omniscience Accuracy","category":"knowledge","score":24,"unit":"percent","family":"omniscienceAccuracy","index":40.93609408405345,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-2025-02-28-private:overall","benchmarkName":"FrontierMath-2025-02-28-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":22.1,"unit":"percent","family":"epoch-ai:epoch-frontiermath-2025-02-28-private:overall","index":49.423558899918106,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":4.1000000000000005,"costPerTest":null,"latency":null},{"benchmarkId":"vitaBench","benchmarkName":"VITA-Bench","category":"agentic","score":18.5,"unit":"percent","family":"vitaBench","index":38.01063520757175,"aboveFit":false,"run":null,"source":"Meituan LongCat Team","sourceUrl":"https://vitabench.github.io/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:ioi-v1:overall","benchmarkName":"IOI v1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":14.416,"unit":"percent","family":"vals-ai:ioi-v1:overall","index":47.31316994602174,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/ioi-v1","measuredAt":"2026-08-09","stderr":4.496,"costPerTest":0.05196,"latency":2384.516},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":11.2,"unit":"percent","family":"hle","index":35.623523057647176,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"arc-prize:v2-semi-private:overall","benchmarkName":"ARC-AGI-2 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":4.03,"unit":"percent","family":"arc-prize:v2-semi-private:overall","index":41.038828247176454,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.12,"latency":null},{"benchmarkId":"arc-prize:v2-public-eval:overall","benchmarkName":"ARC-AGI-2 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":3.8899999999999997,"unit":"percent","family":"arc-prize:v2-public-eval:overall","index":null,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.13,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-private:overall","benchmarkName":"FrontierMath-Tier-4-2025-07-01-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":2.1,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-private:overall","index":45.33529135560964,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.1,"costPerTest":null,"latency":null},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":0.9,"unit":"percent","family":"critpt","index":38.5914109978019,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:finance_agent:overall","benchmarkName":"Finance Agent v1.1","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":0,"unit":"percent","family":"vals-ai:finance_agent:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/finance_agent","measuredAt":"2026-06-04","stderr":0,"costPerTest":0.03968,"latency":100.617}],"price":{"inputPerMillion":0.26,"outputPerMillion":0.38,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":19,"timeToFirstToken":0.98,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":{"elo":1424.5298016982538,"eloUpper":1428.0031861851226,"eloLower":1421.056417211385,"votes":47790,"rank":109,"rankUpper":null,"rankLower":null,"arenaName":"DeepSeek-V3.2","organization":"deepseek","updatedAt":"2026-11-09"},"quality":{"coveragePercent":85,"benchmarkCount":23,"categoryCount":6,"confidence":"medium","state":"available"},"hosts":[{"host":"deepinfra","name":"Deep Infra","kind":"host","url":"https://deepinfra.com/models","inputPerMillion":0.26,"outputPerMillion":0.38,"cacheReadPerMillion":0.13,"contextTokens":163840,"maxOutputTokens":64000,"deprecated":false},{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":0.26,"outputPerMillion":0.38,"cacheReadPerMillion":0.13,"contextTokens":163840,"maxOutputTokens":147456,"deprecated":false},{"host":"siliconflow","name":"SiliconFlow","kind":"host","url":"https://cloud.siliconflow.com/models","inputPerMillion":0.27,"outputPerMillion":0.42,"cacheReadPerMillion":0.135,"contextTokens":164000,"maxOutputTokens":164000,"deprecated":false},{"host":"chutes","name":"Chutes","kind":"host","url":"https://llm.chutes.ai/v1/models","inputPerMillion":1,"outputPerMillion":1,"cacheReadPerMillion":0.1,"contextTokens":131072,"maxOutputTokens":65536,"deprecated":false},{"host":"digitalocean","name":"DigitalOcean","kind":"cloud","url":"https://docs.digitalocean.com/products/gradient-ai-platform/details/models/","inputPerMillion":0.5,"outputPerMillion":1.6,"cacheReadPerMillion":0.15,"contextTokens":163840,"maxOutputTokens":163840,"deprecated":false},{"host":"amazon-bedrock","name":"Amazon Bedrock","kind":"cloud","url":"https://docs.aws.amazon.com/bedrock/latest/userguide/models-supported.html","inputPerMillion":0.62,"outputPerMillion":1.85,"cacheReadPerMillion":null,"contextTokens":163840,"maxOutputTokens":81920,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:sierra-research","name":"Sierra Research","url":"https://github.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:swe-bench-team","name":"SWE-bench team","url":"https://github.com","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:arc-prize-foundation","name":"ARC Prize Foundation","url":"https://arcprize.org","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"claudeopus4","providerId":"anthropic","slug":"anthropic-claude-opus-4","name":"Claude Opus 4","provider":"Anthropic","description":"Claude Opus 4 is a model from Anthropic. 14 benchmarks count toward its index, in 6 categories.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":43.87289488380363,"overallScoreRange":{"low":38.55128060207973,"high":49.194509165527535},"badges":[],"capabilityScores":{"agentic":49.57134453581209,"coding":47.32711464139134,"reasoning":38.55151882479849,"multimodal":50.55934818372168,"knowledge":48.2444226783885,"multilingual":null,"instruction":null,"math":39.39365114196514},"capabilityCounts":{"math":6,"knowledge":2,"agentic":1,"multimodal":1,"coding":2,"reasoning":2},"benchmarks":[{"benchmarkId":"vals-ai:mgsm:overall","benchmarkName":"MGSM","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":93.782,"unit":"percent","family":"vals-ai:mgsm:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mgsm","measuredAt":"2026-01-09","stderr":1.499,"costPerTest":0.027191,"latency":10.1},{"benchmarkId":"vals-ai:medqa:overall","benchmarkName":"MedQA","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":92.867,"unit":"percent","family":"vals-ai:medqa:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medqa","measuredAt":"2026-04-16","stderr":0.235,"costPerTest":0.037936,"latency":11.929},{"benchmarkId":"vals-ai:math500:overall","benchmarkName":"MATH 500","category":"math","domain":null,"eligible":true,"ranked":true,"score":90.4,"unit":"percent","family":"math-500","index":39.65772566003844,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/math500","measuredAt":"2026-01-09","stderr":1.318,"costPerTest":0.045429,"latency":14.812},{"benchmarkId":"vals-ai:mmlu_pro:overall","benchmarkName":"MMLU Pro","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":86.166,"unit":"percent","family":"mmlu-pro","index":53.03957462186964,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","measuredAt":"2026-09-01","stderr":0.337,"costPerTest":0.049652,"latency":10.405},{"benchmarkId":"epoch-ai:epoch-math-level-5:overall","benchmarkName":"MATH level 5","category":"math","domain":null,"eligible":true,"ranked":true,"score":85.04531722054381,"unit":"percent","family":"epoch-ai:epoch-math-level-5:overall","index":40.309131319405104,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0.9804690367660946,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:legal_bench:overall","benchmarkName":"LegalBench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":83.071,"unit":"percent","family":"vals-ai:legal_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","measuredAt":"2026-10-13","stderr":0.439,"costPerTest":null,"latency":3.733},{"benchmarkId":"tau2-bench:retail:overall","benchmarkName":"τ²-bench Retail","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":81.4,"unit":"percent","family":"tau2-bench","index":56.80429145575759,"aboveFit":false,"run":"Anthropic","source":"Sierra Research","sourceUrl":"https://github.com/sierra-research/tau2-bench","measuredAt":"2026-04-30","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:mmmu:overall","benchmarkName":"MMMU Pro","category":"multimodal","domain":null,"eligible":true,"ranked":true,"score":73.31,"unit":"percent","family":"mmmu-pro","index":50.55934818372168,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmmu","measuredAt":"2026-09-01","stderr":1.063,"costPerTest":0.061801,"latency":14.023},{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":72.72727272727273,"unit":"percent","family":"gpqa-diamond","index":43.912568004742994,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":3.031371053819892,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:tax_eval_v2:overall","benchmarkName":"TaxEval v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":71.914,"unit":"percent","family":"vals-ai:tax_eval_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","measuredAt":"2026-09-01","stderr":0.879,"costPerTest":0.03694,"latency":15.23},{"benchmarkId":"vals-ai:gpqa:overall","benchmarkName":"GPQA Diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":71.717,"unit":"percent","family":"gpqa-diamond","index":42.98597346507175,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","measuredAt":"2026-09-01","stderr":2.261,"costPerTest":0.066808,"latency":19.13},{"benchmarkId":"epoch-ai:epoch-swe-bench-verified:overall","benchmarkName":"SWE-Bench verified","category":"coding","domain":null,"eligible":true,"ranked":true,"score":70.66115702479338,"unit":"percent","family":"swe-bench-verified","index":53.34459934807136,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.071754535128056,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:lcb:overall","benchmarkName":"LiveCodeBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":62.629,"unit":"percent","family":"livecodebench","index":41.309629934711324,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","measuredAt":"2026-09-01","stderr":1.117,"costPerTest":0.055697,"latency":14.777},{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":60,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":43.8267336205499,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":7.385489458759964,"costPerTest":null,"latency":null},{"benchmarkId":"tau2-bench:airline:overall","benchmarkName":"τ²-bench Airline","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":59.6,"unit":"percent","family":"tau2-bench","index":42.33839761586658,"aboveFit":false,"run":"Anthropic","source":"Sierra Research","sourceUrl":"https://github.com/sierra-research/tau2-bench","measuredAt":"2026-03-02","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:mortgage_tax:overall","benchmarkName":"MortgageTax","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":58.586,"unit":"percent","family":"vals-ai:mortgage_tax:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mortgage_tax","measuredAt":"2026-09-01","stderr":0.982,"costPerTest":0.08824,"latency":13.758},{"benchmarkId":"vals-ai:aime:overall","benchmarkName":"AIME","category":"math","domain":null,"eligible":true,"ranked":true,"score":41.25,"unit":"percent","family":"vals-ai:aime:overall","index":33.53070360812892,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/aime","measuredAt":"2026-04-16","stderr":1.152,"costPerTest":0.110384,"latency":37.029},{"benchmarkId":"arc-prize:v1-public-eval:overall","benchmarkName":"ARC-AGI-1 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":35.5,"unit":"percent","family":"arc-prize:v1-public-eval:overall","index":null,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.3485,"latency":null},{"benchmarkId":"arc-prize:v1-semi-private:overall","benchmarkName":"ARC-AGI-1 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":22.5,"unit":"percent","family":"arc-prize:v1-semi-private:overall","index":37.33142117711473,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.4036,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-2025-02-28-private:overall","benchmarkName":"FrontierMath-2025-02-28-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":4.310344827586206,"unit":"percent","family":"epoch-ai:epoch-frontiermath-2025-02-28-private:overall","index":33.71924933272902,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.1715642254112493,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-private:overall","benchmarkName":"FrontierMath-Tier-4-2025-07-01-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":2.083333333333333,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-private:overall","index":45.31836331093945,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0,"costPerTest":null,"latency":null},{"benchmarkId":"arc-prize:v2-semi-private:overall","benchmarkName":"ARC-AGI-2 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":1.27,"unit":"percent","family":"arc-prize:v2-semi-private:overall","index":39.771616472482265,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.6388,"latency":null},{"benchmarkId":"arc-prize:v2-public-eval:overall","benchmarkName":"ARC-AGI-2 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":1.25,"unit":"percent","family":"arc-prize:v2-public-eval:overall","index":null,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.6634,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-public:overall","benchmarkName":"FrontierMath-Tier-4-2025-07-01-Public","category":"math","domain":null,"eligible":false,"ranked":true,"score":0,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-public:overall","index":null,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-2025-02-28-public:overall","benchmarkName":"FrontierMath-2025-02-28-Public","category":"math","domain":null,"eligible":false,"ranked":true,"score":0,"unit":"percent","family":"epoch-ai:epoch-frontiermath-2025-02-28-public:overall","index":null,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":{"elo":1366.0411018412906,"eloUpper":1370.378933844508,"eloLower":1361.703269838073,"votes":42967,"rank":195,"rankUpper":null,"rankLower":null,"arenaName":"Claude Opus 4-20250514","organization":"anthropic","updatedAt":"2026-11-09"},"quality":{"coveragePercent":90,"benchmarkCount":14,"categoryCount":6,"confidence":"medium","state":"available"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:sierra-research","name":"Sierra Research","url":"https://github.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:arc-prize-foundation","name":"ARC Prize Foundation","url":"https://arcprize.org","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"claude-haiku-4-5-thinking","providerId":"anthropic","slug":"claude-haiku-4-5-thinking","name":"Claude Haiku 4.5 Thinking","provider":"Anthropic","description":"Claude Haiku 4.5 Thinking is a reasoning model from Anthropic in the Claude Haiku 4.5 family. 13 benchmarks count toward its index, in 5 categories.","logoUrl":null,"releaseDate":"2025-10-15","contextWindowTokens":200000,"modalities":["text"],"openWeights":false,"details":null,"reasoning":true,"overallScore":43.534088395738486,"overallScoreRange":{"low":36.54933730925751,"high":50.51883948221946},"badges":[],"capabilityScores":{"agentic":50.24155297120215,"coding":42.52535498953347,"reasoning":null,"multimodal":30.137877837290034,"knowledge":43.550676074775055,"multilingual":null,"instruction":null,"math":51.64974674423232},"capabilityCounts":{"math":1,"knowledge":2,"coding":5,"multimodal":2,"agentic":3},"benchmarks":[{"benchmarkId":"vals-ai:mgsm:overall","benchmarkName":"MGSM","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":92.146,"unit":"percent","family":"vals-ai:mgsm:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mgsm","measuredAt":"2026-01-09","stderr":1.649,"costPerTest":0.006874,"latency":9.624},{"benchmarkId":"vals-ai:medscribe:overall","benchmarkName":"MedScribe","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":85.23,"unit":"percent","family":"vals-ai:medscribe:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medscribe","measuredAt":"2026-10-29","stderr":1.899,"costPerTest":0.042375,"latency":66.197},{"benchmarkId":"vals-ai:aime:overall","benchmarkName":"AIME","category":"math","domain":null,"eligible":true,"ranked":true,"score":82.708,"unit":"percent","family":"vals-ai:aime:overall","index":51.64974674423232,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/aime","measuredAt":"2026-04-16","stderr":0.915,"costPerTest":0.173775,"latency":158.25},{"benchmarkId":"vals-ai:legal_bench:overall","benchmarkName":"LegalBench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":81.238,"unit":"percent","family":"vals-ai:legal_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","measuredAt":"2026-10-13","stderr":0.487,"costPerTest":null,"latency":10.372},{"benchmarkId":"vals-ai:medqa:overall","benchmarkName":"MedQA","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":79.567,"unit":"percent","family":"vals-ai:medqa:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medqa","measuredAt":"2026-04-16","stderr":0.368,"costPerTest":0.014093,"latency":23.905},{"benchmarkId":"vals-ai:mmlu_pro:overall","benchmarkName":"MMLU Pro","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":78.715,"unit":"percent","family":"mmlu-pro","index":43.65220648364387,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","measuredAt":"2026-09-01","stderr":0.477,"costPerTest":0.018312,"latency":25.784},{"benchmarkId":"vals-ai:gpqa:overall","benchmarkName":"GPQA Diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":72.222,"unit":"percent","family":"gpqa-diamond","index":43.44914566590624,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","measuredAt":"2026-09-01","stderr":2.474,"costPerTest":0.055064,"latency":83.203},{"benchmarkId":"vals-ai:tax_eval_v2:overall","benchmarkName":"TaxEval v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":67.539,"unit":"percent","family":"vals-ai:tax_eval_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","measuredAt":"2026-09-01","stderr":0.914,"costPerTest":0.02743,"latency":47.524},{"benchmarkId":"vals-ai:swebench:overall","benchmarkName":"SWE-bench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":66.6,"unit":"percent","family":"swe-bench-verified","index":50.34926220986553,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/swebench","measuredAt":"2026-09-01","stderr":2.111,"costPerTest":0.366229,"latency":242.166},{"benchmarkId":"vals-ai:mortgage_tax:overall","benchmarkName":"MortgageTax","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":62.162,"unit":"percent","family":"vals-ai:mortgage_tax:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mortgage_tax","measuredAt":"2026-09-01","stderr":0.962,"costPerTest":0.020742,"latency":29.792},{"benchmarkId":"vals-ai:corp_fin_v2:overall","benchmarkName":"CorpFin v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":60.606,"unit":"percent","family":"vals-ai:corp_fin_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","measuredAt":"2026-08-12","stderr":0.962,"costPerTest":0.089265,"latency":14.242},{"benchmarkId":"vals-ai:case_law_v2:overall","benchmarkName":"CaseLaw v2","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":56.484,"unit":"percent","family":"vals-ai:case_law_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/case_law_v2","measuredAt":"2026-05-04","stderr":0.916,"costPerTest":0.049552,"latency":32.919},{"benchmarkId":"parseBenchLlamaIndex","benchmarkName":"ParseBench: document parsing for AI agents (LlamaIndex leaderboard)","category":"multimodal","score":51.21,"unit":"percent","family":"parseBenchLlamaIndex","index":45.16817900496956,"aboveFit":false,"run":null,"source":"Boyang Zhang, Sebastián G. Acosta, Preston Carlson, Sacha Bron, Pierre-Loïc Doulcet, Daniel B. Ospina and Simon Suo (LlamaIndex)","sourceUrl":"https://arxiv.org/abs/2604.08538","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:terminal-bench:overall","benchmarkName":"Terminal-Bench 1.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":50,"unit":"percent","family":"vals-ai:terminal-bench:overall","index":54.402056932447785,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench","measuredAt":"2026-01-12","stderr":5.59,"costPerTest":0.259805,"latency":349.265},{"benchmarkId":"vals-ai:finance_agent:overall","benchmarkName":"Finance Agent v1.1","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":46.931,"unit":"percent","family":"vals-ai:finance_agent:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/finance_agent","measuredAt":"2026-06-04","stderr":2.849,"costPerTest":0.377285,"latency":117.937},{"benchmarkId":"vals-ai:mmmu:overall","benchmarkName":"MMMU Pro","category":"multimodal","domain":null,"eligible":true,"ranked":true,"score":46.069,"unit":"percent","family":"mmmu-pro","index":15.10757666961051,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmmu","measuredAt":"2026-09-01","stderr":1.198,"costPerTest":0.027476,"latency":40.481},{"benchmarkId":"vals-ai:terminal-bench-2-1:overall","benchmarkName":"Terminal-Bench 2.1","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":43.82,"unit":"percent","family":"terminal-bench-2-1","index":46.7694832785609,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2-1","measuredAt":"2026-10-01","stderr":1.946,"costPerTest":0.363822,"latency":649.489},{"benchmarkId":"vals-ai:lcb:overall","benchmarkName":"LiveCodeBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":41.175,"unit":"percent","family":"livecodebench","index":23.508361607686552,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","measuredAt":"2026-09-01","stderr":1.109,"costPerTest":0.059767,"latency":81.561},{"benchmarkId":"vals-ai:terminal-bench-2:overall","benchmarkName":"Terminal-Bench 2.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":38.202,"unit":"percent","family":"vals-ai:terminal-bench-2:overall","index":49.55311870259774,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2","measuredAt":"2026-06-04","stderr":5.18,"costPerTest":0.327722,"latency":655.044},{"benchmarkId":"vals-ai:medcode:overall","benchmarkName":"MedCode","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":32.678,"unit":"percent","family":"vals-ai:medcode:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medcode","measuredAt":"2026-10-29","stderr":1.998,"costPerTest":0.020099,"latency":34.291},{"benchmarkId":"vals-ai:fabv2:overall","benchmarkName":"Finance Agent (v2)","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":31.01,"unit":"percent","family":"vals-ai:fabv2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/fabv2","measuredAt":"2026-10-29","stderr":0.564,"costPerTest":0.602129,"latency":176.769},{"benchmarkId":"vals-ai:emb:overall","benchmarkName":"EMB","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":22.664,"unit":"percent","family":"vals-ai:emb:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/emb","measuredAt":"2026-10-20","stderr":2.392,"costPerTest":1.030784,"latency":718.845},{"benchmarkId":"vals-ai:vibe-code:overall","benchmarkName":"Vibe Code Bench v1.1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":11.393,"unit":"percent","family":"vals-ai:vibe-code:overall","index":46.36029598841619,"aboveFit":false,"run":"OpenHands","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","measuredAt":"2026-11-09","stderr":3.129,"costPerTest":1.306474,"latency":775.882},{"benchmarkId":"vals-ai:legal_research:overall","benchmarkName":"Legal Research Bench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":10.577,"unit":"percent","family":"vals-ai:legal_research:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_research","measuredAt":"2026-11-09","stderr":2.138,"costPerTest":0.26891,"latency":243.38},{"benchmarkId":"vals-ai:code-migration:overall","benchmarkName":"Code Migration","category":"coding","domain":null,"eligible":true,"ranked":true,"score":7.551,"unit":"percent","family":"vals-ai:code-migration:overall","index":49.20053139217421,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/code-migration","measuredAt":"2026-10-29","stderr":1.056,"costPerTest":0.981576,"latency":593.715},{"benchmarkId":"vals-ai:ioi-v1:overall","benchmarkName":"IOI v1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":6.166,"unit":"percent","family":"vals-ai:ioi-v1:overall","index":43.208323749524865,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/ioi-v1","measuredAt":"2026-08-09","stderr":2.515,"costPerTest":2.133406,"latency":7415.352},{"benchmarkId":"vals-ai:tax_agent_bench:overall","benchmarkName":"Tax Agent Bench","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":5.855,"unit":"percent","family":"vals-ai:tax_agent_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench","measuredAt":"2026-11-09","stderr":1.537,"costPerTest":0.226416,"latency":155.016},{"benchmarkId":"vals-ai:programbench:overall","benchmarkName":"ProgramBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":0,"unit":"percent","family":"vals-ai:programbench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/programbench","measuredAt":"2026-11-09","stderr":0,"costPerTest":null,"latency":457.113}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":75,"benchmarkCount":13,"categoryCount":5,"confidence":"low","state":"available"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"trinity-large-thinking","providerId":"arcee-ai","slug":"trinity-large-thinking","name":"Trinity-Large-Thinking","provider":"Arcee AI","description":"Trinity-Large-Thinking is a reasoning model from Arcee AI in the Trinity Large family. 12 benchmarks count toward its index, in 5 categories.","logoUrl":null,"releaseDate":"2026-03-10","contextWindowTokens":262144,"modalities":["text"],"openWeights":true,"details":{"apiId":"arcee-ai/trinity-large-thinking","maxOutputTokens":80000,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":false,"cacheReadPerMillion":0.06,"weightsUrl":"https://huggingface.co/arcee-ai/Trinity-Large-Thinking"},"reasoning":true,"overallScore":43.48151899345726,"overallScoreRange":{"low":36.313642067059135,"high":50.64939591985538},"badges":[],"capabilityScores":{"agentic":43.97255285176888,"coding":48.305612716175204,"reasoning":35.98026021461385,"multimodal":null,"knowledge":45.49278208378883,"multilingual":null,"instruction":39.76755577730691,"math":null},"capabilityCounts":{"agentic":4,"knowledge":3,"coding":2,"instruction":1,"reasoning":2},"benchmarks":[{"benchmarkId":"aime2025Arcee","benchmarkName":"AIME25 first-party comparison snapshot","category":"math","score":96.3,"unit":"percent","family":"aime2025Arcee","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"tau2Bench","benchmarkName":"τ²-Bench Tool-Agent-User Evaluation","category":"agentic","score":90.1,"unit":"percent","family":"tau2-bench","index":62.577377529475555,"aboveFit":true,"run":null,"source":"Victor Barres, Honghua Dong, Soham Ray, Xujie Si, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2506.07982","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"tau2Airline","benchmarkName":"τ²-Bench Airline Domain","category":"agentic","score":88,"unit":"percent","family":"tau2Airline","index":null,"aboveFit":false,"run":null,"source":"Victor Barres, Honghua Dong, Soham Ray, Xujie Si, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2506.07982","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmluProArcee","benchmarkName":"MMLU-Pro first-party comparison snapshot","category":"knowledge","score":83.4,"unit":"percent","family":"mmlu-pro","index":49.55474570355902,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gpqaDiamond","benchmarkName":"GPQA Diamond","category":"knowledge","score":76.3,"unit":"percent","family":"gpqa-diamond","index":47.189375834229125,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":75.2,"unit":"percent","family":"gpqa-diamond","index":46.18048589181737,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"sweVerifiedArcee","benchmarkName":"SWE-bench Verified (mini-swe-agent-v2)","category":"coding","score":63.2,"unit":"percent","family":"swe-bench-verified","index":47.84156644540098,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:case_law_v2:overall","benchmarkName":"CaseLaw v2","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":57.878,"unit":"percent","family":"vals-ai:case_law_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/case_law_v2","measuredAt":"2026-05-04","stderr":1.856,"costPerTest":0.008867,"latency":14.554},{"benchmarkId":"aaIfBench","benchmarkName":"Artificial Analysis IFBench","category":"instruction","score":56.3,"unit":"percent","family":"ifbench-family","index":41.636577006965915,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:corp_fin_v2:overall","benchmarkName":"CorpFin v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":54.662,"unit":"percent","family":"vals-ai:corp_fin_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","measuredAt":"2026-08-12","stderr":0.977,"costPerTest":0.01939,"latency":12.482},{"benchmarkId":"ifBench","benchmarkName":"Instruction Following Benchmark","category":"instruction","score":52.3,"unit":"percent","family":"ifbench-family","index":37.898534547647905,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaSciCode","benchmarkName":"Artificial Analysis SciCode","category":"coding","score":40.6,"unit":"percent","family":"scicode-family","index":48.76965898694944,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/scicode","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":38,"unit":"percent","family":"lcr","index":33.36910943142579,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gertLabs","benchmarkName":"Gert Labs Composite Game Benchmark","category":"agentic","score":32.55,"unit":"percent","family":"gertLabs","index":41.903038291248926,"aboveFit":false,"run":null,"source":"Gert Labs","sourceUrl":"https://gertlabs.com/rankings","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:terminal-bench-2:overall","benchmarkName":"Terminal-Bench 2.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":17.978,"unit":"percent","family":"vals-ai:terminal-bench-2:overall","index":37.37602066915826,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2","measuredAt":"2026-06-04","stderr":4.093,"costPerTest":0.325546,"latency":1015.224},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":15.8,"unit":"percent","family":"hle","index":40.23866968478423,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":0.9,"unit":"percent","family":"critpt","index":38.5914109978019,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gdpvalAaNormalized","benchmarkName":"GDPval-AA normalized","category":"agentic","score":0,"unit":"percent","family":"gdpvalAaNormalized","index":34.03377491719279,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":0.25,"outputPerMillion":0.7999999999999999,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":138,"timeToFirstToken":0.881,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":{"elo":1340.190316662844,"eloUpper":1344.8384242930056,"eloLower":1335.542209032682,"votes":30045,"rank":218,"rankUpper":null,"rankLower":null,"arenaName":"Trinity Large Thinking","organization":"","updatedAt":"2026-11-09"},"quality":{"coveragePercent":75,"benchmarkCount":12,"categoryCount":5,"confidence":"low","state":"available"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"mimo-v2-flash","providerId":"xiaomi","slug":"mimo-v2-flash","name":"MiMo-V2-Flash","provider":"Xiaomi","description":"MiMo-V2-Flash is a reasoning model from Xiaomi. 13 benchmarks count toward its index, in 6 categories.","logoUrl":null,"releaseDate":"2026-03-15","contextWindowTokens":256000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":true,"overallScore":43.42708204679931,"overallScoreRange":{"low":37.44233096031834,"high":49.41183313328029},"badges":[],"capabilityScores":{"agentic":48.53642521379106,"coding":52.979465450552695,"reasoning":40.416599104311906,"multimodal":null,"knowledge":38.0462750273212,"multilingual":null,"instruction":26.310602923762065,"math":50.0683110769799},"capabilityCounts":{"math":1,"knowledge":4,"agentic":2,"coding":2,"reasoning":3,"instruction":1},"benchmarks":[{"benchmarkId":"aime2025","benchmarkName":"American Invitational Mathematics Examination 2025","category":"math","score":94.1,"unit":"percent","family":"aime2025","index":50.0683110769799,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmluPro","benchmarkName":"Massive Multitask Language Understanding Professional","category":"knowledge","score":84.9,"unit":"percent","family":"mmlu-pro","index":51.44456615817433,"aboveFit":false,"run":null,"source":"Yubo Wang, Xueguang Ma, Ge Zhang, Yuansheng Ni, Abhranil Chandra, Shiguang Guo, Weiming Ren, Aaran Arulraj, Xuan He, Ziyan Jiang, Tianle Li, Max Ku, Kai Wang, Alex Zhuang, Rongqi Fan, Xiang Yue, Wenhu Chen","sourceUrl":"https://arxiv.org/abs/2406.01574","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"tau2Bench","benchmarkName":"τ²-Bench Tool-Agent-User Evaluation","category":"agentic","score":83.9,"unit":"percent","family":"tau2-bench","index":58.46322423556161,"aboveFit":false,"run":null,"source":"Victor Barres, Honghua Dong, Soham Ray, Xujie Si, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2506.07982","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gpqa","benchmarkName":"Graduate-Level Google-Proof Q&A","category":"knowledge","score":83.7,"unit":"percent","family":"gpqa-diamond","index":53.97645362863554,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"liveCodeBenchV6","benchmarkName":"LiveCodeBench v6","category":"coding","score":80.6,"unit":"percent","family":"liveCodeBenchV6","index":50.59427716231075,"aboveFit":false,"run":null,"source":"LiveCodeBench maintainers","sourceUrl":"https://github.com/LiveCodeBench/LiveCodeBench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"sweVerified","benchmarkName":"Software Engineering Benchmark Verified","category":"coding","score":73.4,"unit":"percent","family":"swe-bench-verified","index":55.36465373879464,"aboveFit":false,"run":null,"source":"Carlos E. Jimenez, John Yang, Alexander Wettig, Shunyu Yao, Kexin Pei, Ofir Press, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2310.06770","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":65.6,"unit":"percent","family":"gpqa-diamond","index":37.37562821258742,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"longBenchV2","benchmarkName":"LongBench v2","category":"reasoning","score":60.6,"unit":"percent","family":"longBenchV2","index":52.19790876223265,"aboveFit":false,"run":null,"source":"LongBench v2 authors","sourceUrl":"https://arxiv.org/abs/2412.15204","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:gpqa:overall","benchmarkName":"GPQA Diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":59.344,"unit":"percent","family":"gpqa-diamond","index":31.637795958289253,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","measuredAt":"2026-09-01","stderr":2.496,"costPerTest":0.001212,"latency":20.902},{"benchmarkId":"aaIfBench","benchmarkName":"Artificial Analysis IFBench","category":"instruction","score":39.9,"unit":"percent","family":"ifbench-family","index":26.310602923762065,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":36.3,"unit":"percent","family":"lcr","index":32.20277253793017,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"omniscienceAccuracy","benchmarkName":"Artificial Analysis Omniscience Accuracy","category":"knowledge","score":15.6,"unit":"percent","family":"omniscienceAccuracy","index":30.34994381795335,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":8.6,"unit":"percent","family":"hle","index":33.01496192056971,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gdpvalAaNormalized","benchmarkName":"GDPval-AA normalized","category":"agentic","score":6.4,"unit":"percent","family":"gdpvalAaNormalized","index":38.609626192020514,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":0,"unit":"percent","family":"critpt","index":36.8491160127729,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":129,"timeToFirstToken":2.14,"source":"BenchLM","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":85,"benchmarkCount":13,"categoryCount":6,"confidence":"medium","state":"available"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gpt-oss-120b","providerId":"openai","slug":"gpt-oss-120b","name":"GPT-OSS 120B","provider":"OpenAI","description":"GPT-OSS 120B is a standard model from OpenAI in the GPT-OSS family. 25 benchmarks count toward its index, in 7 categories.","logoUrl":null,"releaseDate":"2025-08-05","contextWindowTokens":131072,"modalities":["text"],"openWeights":true,"details":{"apiId":"openai/gpt-oss-120b","maxOutputTokens":65536,"knowledgeCutoff":"2024-06-30","retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.075,"weightsUrl":"https://huggingface.co/openai/gpt-oss-120b"},"reasoning":false,"overallScore":43.3697357422665,"overallScoreRange":{"low":39.1357357422665,"high":47.6037357422665},"badges":[],"capabilityScores":{"agentic":40.104797308455005,"coding":42.842577530460744,"reasoning":40.2112742176123,"multimodal":null,"knowledge":42.40580777947218,"multilingual":51.05469133797723,"instruction":53.504861815300615,"math":51.497812205927715},"capabilityCounts":{"math":5,"knowledge":5,"coding":4,"instruction":1,"agentic":6,"multilingual":1,"reasoning":3},"benchmarks":[{"benchmarkId":"aime2024","benchmarkName":"American Invitational Mathematics Examination 2024","category":"math","score":95.8,"unit":"percent","family":"aime2024","index":47.37524298518983,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:math500:overall","benchmarkName":"MATH 500","category":"math","domain":null,"eligible":true,"ranked":true,"score":94.8,"unit":"percent","family":"math-500","index":44.3074617947299,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/math500","measuredAt":"2026-01-09","stderr":0.993,"costPerTest":0.001381,"latency":16.275},{"benchmarkId":"aaAime2025","benchmarkName":"Artificial Analysis AIME 2025","category":"math","score":93.4,"unit":"percent","family":"aaAime2025","index":null,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/aime-2025","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:aime:overall","benchmarkName":"AIME","category":"math","domain":null,"eligible":true,"ranked":true,"score":92.598,"unit":"percent","family":"vals-ai:aime:overall","index":55.97212931493189,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/aime","measuredAt":"2026-04-16","stderr":1.225,"costPerTest":0.010472,"latency":156.629},{"benchmarkId":"aime2025","benchmarkName":"American Invitational Mathematics Examination 2025","category":"math","score":92.5,"unit":"percent","family":"aime2025","index":49.2001097708368,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:mgsm:overall","benchmarkName":"MGSM","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":92.036,"unit":"percent","family":"vals-ai:mgsm:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mgsm","measuredAt":"2026-01-09","stderr":1.642,"costPerTest":0.000659,"latency":5.793},{"benchmarkId":"vals-ai:medqa:overall","benchmarkName":"MedQA","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":91.36,"unit":"percent","family":"vals-ai:medqa:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medqa","measuredAt":"2026-04-16","stderr":0.24,"costPerTest":0.000607,"latency":12.383},{"benchmarkId":"mmlu","benchmarkName":"Massive Multitask Language Understanding","category":"knowledge","score":90,"unit":"percent","family":"mmlu","index":38.991587985272176,"aboveFit":false,"run":null,"source":"Dan Hendrycks, Collin Burns, Steven Basart, Andy Zou, Mantas Mazeika, Dawn Song, Jacob Steinhardt","sourceUrl":"https://arxiv.org/abs/2009.03300","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":88.88888888888889,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":60.634117163950165,"aboveFit":true,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":4.463343655613273,"costPerTest":null,"latency":null},{"benchmarkId":"aaLiveCodeBench","benchmarkName":"Artificial Analysis LiveCodeBench","category":"coding","score":87.8,"unit":"percent","family":"livecodebench","index":62.19504640993927,"aboveFit":true,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/livecodebench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:lcb:overall","benchmarkName":"LiveCodeBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":83.234,"unit":"percent","family":"livecodebench","index":58.40644795831277,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","measuredAt":"2026-09-01","stderr":1.055,"costPerTest":0.008486,"latency":81.703},{"benchmarkId":"gpqa","benchmarkName":"Graduate-Level Google-Proof Q&A","category":"knowledge","score":80.1,"unit":"percent","family":"gpqa-diamond","index":50.674631998924305,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:mmlu_pro:overall","benchmarkName":"MMLU Pro","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":79.166,"unit":"percent","family":"mmlu-pro","index":44.220412500331534,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","measuredAt":"2026-09-01","stderr":0.395,"costPerTest":0.001438,"latency":35.41},{"benchmarkId":"vals-ai:gpqa:overall","benchmarkName":"GPQA Diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":78.536,"unit":"percent","family":"gpqa-diamond","index":49.240173935349766,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","measuredAt":"2026-09-01","stderr":2.253,"costPerTest":0.006212,"latency":56.176},{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":78.2,"unit":"percent","family":"gpqa-diamond","index":48.932003916576726,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:legal_bench:overall","benchmarkName":"LegalBench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":75.938,"unit":"percent","family":"vals-ai:legal_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","measuredAt":"2026-10-13","stderr":0.517,"costPerTest":null,"latency":7.852},{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":75.75757575757575,"unit":"percent","family":"gpqa-diamond","index":46.69187914086354,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.7437163443883765,"costPerTest":null,"latency":null},{"benchmarkId":"reactNativeEvals","benchmarkName":"React Native Evals","category":"coding","score":71.6,"unit":"percent","family":"reactNativeEvals","index":47.69712216967908,"aboveFit":false,"run":null,"source":"Callstack","sourceUrl":"https://rn-evals.vercel.app/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:tax_eval_v2:overall","benchmarkName":"TaxEval v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":71.586,"unit":"percent","family":"vals-ai:tax_eval_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","measuredAt":"2026-09-01","stderr":0.878,"costPerTest":0.002958,"latency":47.661},{"benchmarkId":"aaIfBench","benchmarkName":"Artificial Analysis IFBench","category":"instruction","score":69,"unit":"percent","family":"ifbench-family","index":53.504861815300615,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"tau2Bench","benchmarkName":"τ²-Bench Tool-Agent-User Evaluation","category":"agentic","score":65.8,"unit":"percent","family":"tau2-bench","index":46.452550909780534,"aboveFit":false,"run":null,"source":"Victor Barres, Honghua Dong, Soham Ray, Xujie Si, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2506.07982","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"sweVerified","benchmarkName":"Software Engineering Benchmark Verified","category":"coding","score":62.4,"unit":"percent","family":"swe-bench-verified","index":47.251520383174025,"aboveFit":false,"run":null,"source":"Carlos E. Jimenez, John Yang, Alexander Wettig, Shunyu Yao, Kexin Pei, Ofir Press, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2310.06770","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":61.262499999999996,"unit":"percent","family":"euroeval","index":55.33935185885234,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":0.8705126004257492,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-portuguese:overall","benchmarkName":"EuroEval Portuguese","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":59.292500000000004,"unit":"percent","family":"euroeval","index":52.66651514293844,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/portuguese","measuredAt":null,"stderr":0.5113523736133431,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-italian:overall","benchmarkName":"EuroEval Italian","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":58.4275,"unit":"percent","family":"euroeval","index":51.49290917376811,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/italian","measuredAt":null,"stderr":0.573895678673398,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-french:overall","benchmarkName":"EuroEval French","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":58.38125,"unit":"percent","family":"euroeval","index":51.43015856558993,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/french","measuredAt":null,"stderr":0.4211294337849113,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:corp_fin_v2:overall","benchmarkName":"CorpFin v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":58.236,"unit":"percent","family":"vals-ai:corp_fin_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","measuredAt":"2026-08-12","stderr":0.968,"costPerTest":0.009601,"latency":42.722},{"benchmarkId":"euroeval:euroeval-dutch:overall","benchmarkName":"EuroEval Dutch","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":57.82777777777778,"unit":"percent","family":"euroeval","index":50.679224110364544,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/dutch","measuredAt":null,"stderr":0.4356419982247854,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":57.58428571428572,"unit":"percent","family":"euroeval","index":50.34886140613346,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.5376821001449895,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-spanish:overall","benchmarkName":"EuroEval Spanish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":53.504999999999995,"unit":"percent","family":"euroeval","index":44.8142093087497,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/spanish","measuredAt":null,"stderr":0.4363484845854286,"costPerTest":null,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":52,"unit":"percent","family":"lcr","index":42.97423678962504,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":51.15625000000001,"unit":"percent","family":"euroeval","index":41.627495990727994,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.5813292956664062,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:case_law_v2:overall","benchmarkName":"CaseLaw v2","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":48.767,"unit":"percent","family":"vals-ai:case_law_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/case_law_v2","measuredAt":"2026-05-04","stderr":0.877,"costPerTest":0.004339,"latency":68.348},{"benchmarkId":"aaSciCode","benchmarkName":"Artificial Analysis SciCode","category":"coding","score":34,"unit":"percent","family":"scicode-family","index":40.1652974206122,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/scicode","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:swebench:overall","benchmarkName":"SWE-bench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":33.6,"unit":"percent","family":"swe-bench-verified","index":26.009862143003698,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/swebench","measuredAt":"2026-09-01","stderr":2.114,"costPerTest":0.171583,"latency":256.464},{"benchmarkId":"gertLabs","benchmarkName":"Gert Labs Composite Game Benchmark","category":"agentic","score":29.61,"unit":"percent","family":"gertLabs","index":39.52247093907985,"aboveFit":false,"run":null,"source":"Gert Labs","sourceUrl":"https://gertlabs.com/rankings","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"swe-bench:swe-bench-verified:overall","benchmarkName":"SWE-bench Verified","category":"coding","domain":null,"eligible":true,"ranked":true,"score":26,"unit":"percent","family":"swe-bench-verified","index":20.40442455184764,"aboveFit":false,"run":"mini-SWE-agent","source":"SWE-bench team","sourceUrl":"https://www.swebench.com/","measuredAt":"2026-02-26","stderr":null,"costPerTest":0.05711905579999999,"latency":null},{"benchmarkId":"swebench-experiments:swe-bench-verified:overall","benchmarkName":"SWE-bench Verified","category":"coding","domain":null,"eligible":true,"ranked":true,"score":26,"unit":"percent","family":"swe-bench-verified","index":20.40442455184764,"aboveFit":false,"run":"mini-SWE-agent","source":"SWE-bench team","sourceUrl":"https://github.com/SWE-bench/experiments/tree/main/evaluation/verified","measuredAt":"2026-09-01","stderr":null,"costPerTest":0.05711905579999999,"latency":null},{"benchmarkId":"vals-ai:terminal-bench:overall","benchmarkName":"Terminal-Bench 1.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":22.5,"unit":"percent","family":"vals-ai:terminal-bench:overall","index":34.500767275227226,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench","measuredAt":"2026-01-12","stderr":4.669,"costPerTest":0.038818,"latency":53.75},{"benchmarkId":"omniscienceAccuracy","benchmarkName":"Artificial Analysis Omniscience Accuracy","category":"knowledge","score":21.7,"unit":"percent","family":"omniscienceAccuracy","index":38.03750532071652,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:finance_agent:overall","benchmarkName":"Finance Agent v1.1","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":21.541,"unit":"percent","family":"vals-ai:finance_agent:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/finance_agent","measuredAt":"2026-06-04","stderr":2.238,"costPerTest":0.064574,"latency":185.074},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":19.6,"unit":"percent","family":"hle","index":44.05118211589745,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:terminal-bench-2:overall","benchmarkName":"Terminal-Bench 2.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":19.101,"unit":"percent","family":"vals-ai:terminal-bench-2:overall","index":38.052191609207334,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2","measuredAt":"2026-06-04","stderr":4.19,"costPerTest":0.131646,"latency":490.948},{"benchmarkId":"hle","benchmarkName":"Humanity's Last Exam","category":"knowledge","score":14.9,"unit":"percent","family":"hle","index":39.335706214257414,"aboveFit":false,"run":null,"source":"Center for AI Safety, Scale AI, and thousands of expert contributors","sourceUrl":"https://lastexam.ai/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gdpvalAaNormalized","benchmarkName":"GDPval-AA normalized","category":"agentic","score":5.4,"unit":"percent","family":"gdpvalAaNormalized","index":37.89464943032868,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"apexAgentsAa","benchmarkName":"APEX-Agents-AA","category":"agentic","score":3.1,"unit":"percent","family":"apexAgentsAa","index":44.20615368710644,"aboveFit":false,"run":null,"source":"Artificial Analysis / Mercor","sourceUrl":"https://artificialanalysis.ai/evaluations/apex-agents-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":1.1,"unit":"percent","family":"critpt","index":38.97858766114168,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-mystery-game-puzzles:overall","benchmarkName":"Mystery Game Puzzles","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":0,"unit":"percent","family":"epoch-ai:epoch-mystery-game-puzzles:overall","index":38.6809982020702,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":0.15,"outputPerMillion":0.6,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":34,"timeToFirstToken":0.347,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":{"elo":1365.466613194138,"eloUpper":1369.820841280873,"eloLower":1361.112385107403,"votes":30133,"rank":197,"rankUpper":null,"rankLower":null,"arenaName":"gpt-oss-120b","organization":"openai","updatedAt":"2026-11-09"},"quality":{"coveragePercent":90,"benchmarkCount":25,"categoryCount":7,"confidence":"medium","state":"available"},"hosts":[{"host":"coreweave","name":"CoreWeave","kind":"host","url":"https://docs.wandb.ai/inference","inputPerMillion":0.03,"outputPerMillion":0.17,"cacheReadPerMillion":0.03,"contextTokens":131072,"maxOutputTokens":131072,"deprecated":false},{"host":"deepinfra","name":"Deep Infra","kind":"host","url":"https://deepinfra.com/models","inputPerMillion":0.037,"outputPerMillion":0.17,"cacheReadPerMillion":null,"contextTokens":131072,"maxOutputTokens":16384,"deprecated":false},{"host":"crusoe","name":"Crusoe","kind":"host","url":"https://docs.crusoecloud.com/managed-inference/overview","inputPerMillion":0.05,"outputPerMillion":0.25,"cacheReadPerMillion":0.05,"contextTokens":131072,"maxOutputTokens":32768,"deprecated":false},{"host":"novita-ai","name":"Novita AI","kind":"host","url":"https://novita.ai/docs/guides/introduction","inputPerMillion":0.05,"outputPerMillion":0.25,"cacheReadPerMillion":null,"contextTokens":131072,"maxOutputTokens":32768,"deprecated":false},{"host":"google-vertex","name":"Google Vertex AI","kind":"cloud","url":"https://cloud.google.com/vertex-ai/generative-ai/docs/models","inputPerMillion":0.09,"outputPerMillion":0.36,"cacheReadPerMillion":null,"contextTokens":131072,"maxOutputTokens":131072,"deprecated":false},{"host":"siliconflow","name":"SiliconFlow","kind":"host","url":"https://cloud.siliconflow.com/models","inputPerMillion":0.05,"outputPerMillion":0.45,"cacheReadPerMillion":null,"contextTokens":131000,"maxOutputTokens":8000,"deprecated":false},{"host":"ovhcloud","name":"OVHcloud","kind":"cloud","url":"https://www.ovhcloud.com/en/public-cloud/ai-endpoints/catalog//","inputPerMillion":0.09,"outputPerMillion":0.47,"cacheReadPerMillion":null,"contextTokens":131072,"maxOutputTokens":131072,"deprecated":false},{"host":"baseten","name":"Baseten","kind":"host","url":"https://docs.baseten.co/inference/model-apis/overview","inputPerMillion":0.1,"outputPerMillion":0.5,"cacheReadPerMillion":0.1,"contextTokens":128072,"maxOutputTokens":128072,"deprecated":false},{"host":"sambanova","name":"SambaNova","kind":"host","url":"https://docs.sambanova.ai/cloud/docs/get-started/supported-models","inputPerMillion":0.22,"outputPerMillion":0.59,"cacheReadPerMillion":null,"contextTokens":131072,"maxOutputTokens":131072,"deprecated":false},{"host":"amazon-bedrock","name":"Amazon Bedrock","kind":"cloud","url":"https://docs.aws.amazon.com/bedrock/latest/userguide/models-supported.html","inputPerMillion":0.15,"outputPerMillion":0.6,"cacheReadPerMillion":null,"contextTokens":131072,"maxOutputTokens":128000,"deprecated":false},{"host":"databricks","name":"Databricks","kind":"cloud","url":"https://docs.databricks.com/aws/en/machine-learning/foundation-models/","inputPerMillion":0.15,"outputPerMillion":0.6,"cacheReadPerMillion":null,"contextTokens":131072,"maxOutputTokens":25000,"deprecated":false},{"host":"groq","name":"Groq","kind":"host","url":"https://console.groq.com/docs/models","inputPerMillion":0.15,"outputPerMillion":0.6,"cacheReadPerMillion":0.075,"contextTokens":131072,"maxOutputTokens":65536,"deprecated":false},{"host":"nebius","name":"Nebius","kind":"host","url":"https://docs.tokenfactory.nebius.com/","inputPerMillion":0.15,"outputPerMillion":0.6,"cacheReadPerMillion":0.15,"contextTokens":131072,"maxOutputTokens":8192,"deprecated":false},{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":0.15,"outputPerMillion":0.6,"cacheReadPerMillion":0.075,"contextTokens":131072,"maxOutputTokens":65536,"deprecated":false},{"host":"oracle-cloud","name":"Oracle Cloud","kind":"cloud","url":"https://docs.oracle.com/en-us/iaas/Content/generative-ai/pretrained-models.htm","inputPerMillion":0.15,"outputPerMillion":0.6,"cacheReadPerMillion":null,"contextTokens":128000,"maxOutputTokens":16384,"deprecated":false},{"host":"tinfoil","name":"Tinfoil","kind":"host","url":"https://docs.tinfoil.sh","inputPerMillion":0.15,"outputPerMillion":0.6,"cacheReadPerMillion":null,"contextTokens":131072,"maxOutputTokens":32768,"deprecated":false},{"host":"together-ai","name":"Together AI","kind":"host","url":"https://docs.together.ai/docs/serverless-models","inputPerMillion":0.15,"outputPerMillion":0.6,"cacheReadPerMillion":null,"contextTokens":131072,"maxOutputTokens":131072,"deprecated":false},{"host":"ibm-watsonx","name":"IBM watsonx","kind":"cloud","url":"https://www.ibm.com/docs/en/watsonx/saas?topic=solutions-supported-foundation-models","inputPerMillion":0.159,"outputPerMillion":0.636,"cacheReadPerMillion":null,"contextTokens":131072,"maxOutputTokens":32768,"deprecated":false},{"host":"scaleway","name":"Scaleway","kind":"cloud","url":"https://www.scaleway.com/en/docs/generative-apis/","inputPerMillion":0.168225,"outputPerMillion":0.6729,"cacheReadPerMillion":null,"contextTokens":128000,"maxOutputTokens":32768,"deprecated":false},{"host":"digitalocean","name":"DigitalOcean","kind":"cloud","url":"https://docs.digitalocean.com/products/gradient-ai-platform/details/models/","inputPerMillion":0.1,"outputPerMillion":0.7,"cacheReadPerMillion":0.02,"contextTokens":128000,"maxOutputTokens":4096,"deprecated":false},{"host":"io-net","name":"IO.NET","kind":"host","url":"https://io.net/docs/guides/intelligence/io-intelligence","inputPerMillion":0.188,"outputPerMillion":0.7,"cacheReadPerMillion":0.094,"contextTokens":131072,"maxOutputTokens":4096,"deprecated":false},{"host":"cerebras","name":"Cerebras","kind":"host","url":"https://inference-docs.cerebras.ai/models/overview","inputPerMillion":0.35,"outputPerMillion":0.75,"cacheReadPerMillion":0.35,"contextTokens":131072,"maxOutputTokens":40960,"deprecated":false},{"host":"cloudflare","name":"Cloudflare Workers AI","kind":"cloud","url":"https://developers.cloudflare.com/workers-ai/models/","inputPerMillion":0.35,"outputPerMillion":0.75,"cacheReadPerMillion":null,"contextTokens":128000,"maxOutputTokens":16384,"deprecated":false},{"host":"privatemode","name":"Privatemode","kind":"host","url":"https://docs.privatemode.ai/api/overview","inputPerMillion":0.2311,"outputPerMillion":0.7511,"cacheReadPerMillion":0.0462,"contextTokens":128000,"maxOutputTokens":32768,"deprecated":false},{"host":"stackit","name":"STACKIT","kind":"cloud","url":"https://docs.stackit.cloud/products/data-and-ai/ai-model-serving/basics/available-shared-models","inputPerMillion":0.53,"outputPerMillion":0.76,"cacheReadPerMillion":null,"contextTokens":131000,"maxOutputTokens":8192,"deprecated":false},{"host":"regolo","name":"Regolo","kind":"host","url":"https://docs.regolo.ai/","inputPerMillion":1.1215,"outputPerMillion":4.7103,"cacheReadPerMillion":null,"contextTokens":128000,"maxOutputTokens":16384,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:swe-bench-team","name":"SWE-bench team","url":"https://www.swebench.com","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"kimik2instruct","providerId":"moonshot-ai","slug":"moonshot-ai-kimi-k2-instruct","name":"Kimi K2 Instruct","provider":"Moonshot AI","description":"Kimi K2 Instruct is a model from Moonshot AI. 9 benchmarks count toward its index, in 4 categories.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":43.234138650180064,"overallScoreRange":{"low":34.344138650180064,"high":52.124138650180065},"badges":[],"capabilityScores":{"agentic":44.63842501217318,"coding":41.2878756473728,"reasoning":null,"multimodal":null,"knowledge":43.63079699387506,"multilingual":null,"instruction":null,"math":43.29111920140476},"capabilityCounts":{"math":2,"knowledge":2,"coding":3,"agentic":2},"benchmarks":[{"benchmarkId":"vals-ai:math500:overall","benchmarkName":"MATH 500","category":"math","domain":null,"eligible":true,"ranked":true,"score":94.2,"unit":"percent","family":"math-500","index":43.673406867271986,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/math500","measuredAt":"2026-01-09","stderr":1.045,"costPerTest":0.002985,"latency":37.346},{"benchmarkId":"vals-ai:mgsm:overall","benchmarkName":"MGSM","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":90.946,"unit":"percent","family":"vals-ai:mgsm:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mgsm","measuredAt":"2026-01-09","stderr":1.766,"costPerTest":0.001001,"latency":11.83},{"benchmarkId":"vals-ai:medqa:overall","benchmarkName":"MedQA","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":83.975,"unit":"percent","family":"vals-ai:medqa:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medqa","measuredAt":"2026-04-16","stderr":0.335,"costPerTest":0.001028,"latency":10.984},{"benchmarkId":"vals-ai:legal_bench:overall","benchmarkName":"LegalBench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":81.454,"unit":"percent","family":"vals-ai:legal_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","measuredAt":"2026-10-13","stderr":0.444,"costPerTest":null,"latency":1.172},{"benchmarkId":"vals-ai:mmlu_pro:overall","benchmarkName":"MMLU Pro","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":79.394,"unit":"percent","family":"mmlu-pro","index":44.50766520943307,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","measuredAt":"2026-09-01","stderr":0.394,"costPerTest":0.002836,"latency":20.338},{"benchmarkId":"vals-ai:gpqa:overall","benchmarkName":"GPQA Diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":71.464,"unit":"percent","family":"gpqa-diamond","index":42.75392877831705,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","measuredAt":"2026-09-01","stderr":2.266,"costPerTest":0.004481,"latency":41.434},{"benchmarkId":"vals-ai:lcb:overall","benchmarkName":"LiveCodeBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":70.449,"unit":"percent","family":"livecodebench","index":47.798206345512746,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","measuredAt":"2026-09-01","stderr":1.159,"costPerTest":0.006889,"latency":66.651},{"benchmarkId":"vals-ai:tax_eval_v2:overall","benchmarkName":"TaxEval v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":70.196,"unit":"percent","family":"vals-ai:tax_eval_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","measuredAt":"2026-09-01","stderr":0.895,"costPerTest":0.001543,"latency":12.834},{"benchmarkId":"vals-ai:aime:overall","benchmarkName":"AIME","category":"math","domain":null,"eligible":true,"ranked":true,"score":62.708,"unit":"percent","family":"vals-ai:aime:overall","index":42.908831535537544,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/aime","measuredAt":"2026-04-16","stderr":1.217,"costPerTest":0.01278,"latency":124.668},{"benchmarkId":"vals-ai:corp_fin_v2:overall","benchmarkName":"CorpFin v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":50.388,"unit":"percent","family":"vals-ai:corp_fin_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","measuredAt":"2026-08-12","stderr":0.985,"costPerTest":0.059336,"latency":40.894},{"benchmarkId":"swebench-experiments:swe-bench-verified:overall","benchmarkName":"SWE-bench Verified","category":"coding","domain":null,"eligible":true,"ranked":true,"score":48.6,"unit":"percent","family":"swe-bench-verified","index":37.07322580975908,"aboveFit":false,"run":"CodeSweep - SWE-agent","source":"SWE-bench team","sourceUrl":"https://github.com/SWE-bench/experiments/tree/main/evaluation/verified","measuredAt":"2026-09-01","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"swe-bench:swe-bench-verified:overall","benchmarkName":"SWE-bench Verified","category":"coding","domain":null,"eligible":true,"ranked":true,"score":43.8,"unit":"percent","family":"swe-bench-verified","index":33.53294943639735,"aboveFit":false,"run":"mini-SWE-agent","source":"SWE-bench team","sourceUrl":"https://www.swebench.com/","measuredAt":"2026-02-26","stderr":null,"costPerTest":0.531593366,"latency":null},{"benchmarkId":"vals-ai:terminal-bench:overall","benchmarkName":"Terminal-Bench 1.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":40,"unit":"percent","family":"vals-ai:terminal-bench:overall","index":47.165224329822124,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench","measuredAt":"2026-01-12","stderr":5.477,"costPerTest":0.33518,"latency":133.903},{"benchmarkId":"vals-ai:terminal-bench-2:overall","benchmarkName":"Terminal-Bench 2.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":25.843,"unit":"percent","family":"vals-ai:terminal-bench-2:overall","index":42.111625694524236,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2","measuredAt":"2026-06-04","stderr":4.667,"costPerTest":1.107135,"latency":702.2},{"benchmarkId":"vals-ai:ioi-v1:overall","benchmarkName":"IOI v1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":1.25,"unit":"percent","family":"vals-ai:ioi-v1:overall","index":40.76233297352745,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/ioi-v1","measuredAt":"2026-08-09","stderr":0.632,"costPerTest":2.367573,"latency":1551.101}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":65,"benchmarkCount":9,"categoryCount":4,"confidence":"low","state":"available"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:swe-bench-team","name":"SWE-bench team","url":"https://github.com","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"claude-haiku-4-5","providerId":"anthropic","slug":"claude-haiku-4-5","name":"Claude Haiku 4.5","provider":"Anthropic","description":"Claude Haiku 4.5 is a standard model from Anthropic. 19 benchmarks count toward its index, in 7 categories.","logoUrl":null,"releaseDate":"2025-10-15","contextWindowTokens":200000,"modalities":["text","image","file"],"openWeights":false,"details":{"apiId":"anthropic/claude-haiku-4.5","maxOutputTokens":64000,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.09999999999999999,"weightsUrl":null},"reasoning":false,"overallScore":43.23081516277367,"overallScoreRange":{"low":39.02116274608669,"high":47.44046757946066},"badges":[],"capabilityScores":{"agentic":48.89154367649199,"coding":49.930880091431874,"reasoning":38.06239089754432,"multimodal":42.55887566170943,"knowledge":36.13355336531331,"multilingual":44.48085848031967,"instruction":null,"math":40.477969427346345},"capabilityCounts":{"math":4,"coding":3,"knowledge":2,"reasoning":4,"agentic":4,"multilingual":1,"multimodal":1},"benchmarks":[{"benchmarkId":"epoch-ai:epoch-math-level-5:overall","benchmarkName":"MATH level 5","category":"math","domain":null,"eligible":true,"ranked":true,"score":91.63776601105387,"unit":"percent","family":"epoch-ai:epoch-math-level-5:overall","index":43.51144877487864,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0.515811348923114,"costPerTest":null,"latency":null},{"benchmarkId":"vulcanBench","benchmarkName":"VulcanBench v3","category":"coding","score":76.2,"unit":"percent","family":"vulcanBench","index":53.96060514469284,"aboveFit":false,"run":null,"source":"VulcanBench contributors","sourceUrl":"https://github.com/morganlinton/VulcanBench/tree/main","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"sweVerified","benchmarkName":"Software Engineering Benchmark Verified","category":"coding","score":73.3,"unit":"percent","family":"swe-bench-verified","index":55.290897981016265,"aboveFit":false,"run":null,"source":"Carlos E. Jimenez, John Yang, Alexander Wettig, Shunyu Yao, Kexin Pei, Ofir Press, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2310.06770","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"swebench-experiments:swe-bench-verified:overall","benchmarkName":"SWE-bench Verified","category":"coding","domain":null,"eligible":true,"ranked":true,"score":66.6,"unit":"percent","family":"swe-bench-verified","index":50.34926220986553,"aboveFit":false,"run":"high effort · mini-SWE-agent","source":"SWE-bench team","sourceUrl":"https://github.com/SWE-bench/experiments/tree/main/evaluation/verified","measuredAt":"2026-09-01","stderr":null,"costPerTest":0.33092443610000005,"latency":null},{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":65.8459595959596,"unit":"percent","family":"gpqa-diamond","index":37.60121563313588,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":3.225883512300997,"costPerTest":null,"latency":null},{"benchmarkId":"swebench-experiments:swe-bench-multilingual:overall","benchmarkName":"SWE-bench Multilingual","category":"coding","domain":null,"eligible":true,"ranked":true,"score":64.7,"unit":"percent","family":"swe-bench-multilingual","index":null,"aboveFit":false,"run":"mini-SWE-agent","source":"SWE-bench team","sourceUrl":"https://github.com/SWE-bench/experiments/tree/main/evaluation/multilingual","measuredAt":"2026-09-02","stderr":null,"costPerTest":0.3794902823333333,"latency":null},{"benchmarkId":"livebench:livebench-reasoning:overall","benchmarkName":"LiveBench Reasoning","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":61.7,"unit":"percent","family":"livebench","index":41.54799889766194,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Reasoning","measuredAt":"2026-10-14","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:corp_fin_v2:overall","benchmarkName":"CorpFin v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":60.295,"unit":"percent","family":"vals-ai:corp_fin_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","measuredAt":"2026-08-12","stderr":0.964,"costPerTest":0.075779,"latency":7.951},{"benchmarkId":"livebench:livebench-design:overall","benchmarkName":"LiveBench Design","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":59.9,"unit":"percent","family":"livebench-2-agentic","index":62.39620416810558,"aboveFit":true,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Design","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.155804,"latency":null},{"benchmarkId":"livebench:livebench-data-analysis:overall","benchmarkName":"LiveBench Data Analysis","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":59.3,"unit":"percent","family":"livebench","index":38.4389573069102,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Data%20Analysis","measuredAt":"2026-10-14","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-french:overall","benchmarkName":"EuroEval French","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":57.58874999999999,"unit":"percent","family":"euroeval","index":50.35491841464485,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/french","measuredAt":null,"stderr":0.41991628630001954,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:public-benefits-bench:overall","benchmarkName":"Public Benefits Bench v1.1","category":null,"domain":"Public benefits","eligible":false,"ranked":true,"score":54.263,"unit":"percent","family":"vals-ai:public-benefits-bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/public-benefits-bench","measuredAt":"2026-10-12","stderr":1.296,"costPerTest":0.224512,"latency":177.657},{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":53.667500000000004,"unit":"percent","family":"euroeval","index":45.03468441856495,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":0.546415878704124,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-portuguese:overall","benchmarkName":"EuroEval Portuguese","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":53.39125000000001,"unit":"percent","family":"euroeval","index":44.65987673187905,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/portuguese","measuredAt":null,"stderr":0.5514497937255939,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-dutch:overall","benchmarkName":"EuroEval Dutch","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":53.3811111111111,"unit":"percent","family":"euroeval","index":44.646120592548684,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/dutch","measuredAt":null,"stderr":0.5477225575051662,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-italian:overall","benchmarkName":"EuroEval Italian","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":53.1375,"unit":"percent","family":"euroeval","index":44.315596368090645,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/italian","measuredAt":null,"stderr":0.45247582808366676,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":51.25,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":38.7360357203854,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":7.106690545187014,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-spanish:overall","benchmarkName":"EuroEval Spanish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":49.69,"unit":"percent","family":"euroeval","index":39.63813211524129,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/spanish","measuredAt":null,"stderr":0.44209904150540746,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:public-benefits-bench-v1:overall","benchmarkName":"Public Benefits Bench v1","category":null,"domain":"Public benefits","eligible":false,"ranked":true,"score":49.526,"unit":"percent","family":"vals-ai:public-benefits-bench-v1:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/public-benefits-bench-v1","measuredAt":"2026-06-09","stderr":1.301,"costPerTest":0.194641,"latency":190.393},{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":49.51714285714285,"unit":"percent","family":"euroeval","index":39.40360474567959,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.6177972720785962,"costPerTest":null,"latency":null},{"benchmarkId":"parseBenchLlamaIndex","benchmarkName":"ParseBench: document parsing for AI agents (LlamaIndex leaderboard)","category":"multimodal","score":49.34,"unit":"percent","family":"parseBenchLlamaIndex","index":42.55887566170943,"aboveFit":false,"run":null,"source":"Boyang Zhang, Sebastián G. Acosta, Preston Carlson, Sacha Bron, Pierre-Loïc Doulcet, Daniel B. Ospina and Simon Suo (LlamaIndex)","sourceUrl":"https://arxiv.org/abs/2604.08538","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-coding-2:overall","benchmarkName":"LiveBench Coding 2.0","category":"coding","domain":null,"eligible":true,"ranked":true,"score":48.1,"unit":"percent","family":"livebench:livebench-coding-2:overall","index":43.01195503416188,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Coding","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.4940241379310345,"latency":null},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":47.845,"unit":"percent","family":"euroeval","index":37.13489163764679,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.5749972826022746,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-research:overall","benchmarkName":"LiveBench Research","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":42.2,"unit":"percent","family":"livebench-2-agentic","index":49.451643788013484,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Research","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.43215909090909094,"latency":null},{"benchmarkId":"livebench:livebench-browser:overall","benchmarkName":"LiveBench Browser Use","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":40,"unit":"percent","family":"livebench-2-agentic","index":47.842715379188476,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Browser","measuredAt":"2026-10-14","stderr":null,"costPerTest":1.5018479999999998,"latency":null},{"benchmarkId":"arc-prize:v1-public-eval:overall","benchmarkName":"ARC-AGI-1 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":26.619999999999997,"unit":"percent","family":"arc-prize:v1-public-eval:overall","index":null,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.0223,"latency":null},{"benchmarkId":"jobBench","benchmarkName":"JobBench","category":"agentic","score":16,"unit":"percent","family":"jobBench","index":45.343175167119185,"aboveFit":false,"run":null,"source":"Yuetai Li, Yichen Feng, Zhangchen Xu, Zixian Ma, Kaiyuan Zheng, Fengqing Jiang, Xinghua Sun, Rulin Shao, Zichen Chen, Yue Huang, Xinyang Han, Brian Lee, Kayla Xu, Shenglai Zeng, Hang Hua, Xiangliang Zhang, Basel Alomair, Ranjay Krishna, Luke Zettlemoyer, Pang Wei Koh, Bhaskar Ramasubramanian, Luyao Niu, Xiang Yue, Radha Poovendran","sourceUrl":"https://arxiv.org/abs/2605.26329","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"osWorld21Offline","benchmarkName":"OSWorld 2.1 offline subset (82 tasks)","category":"agentic","score":15.7,"unit":"percent","family":"osWorld21Offline","index":39.59715260689998,"aboveFit":false,"run":null,"source":"Mengqi Yuan, Zilong Zhou, Xinzhuang Xiong, Weiming Wu, Jiayang Sun, Jiamin Song, Kaiqian Cui, Bowen Wang, Haoyuan Wu, Yitong Li, Dunjie Lu, Haikong Lu, Qi Zhen, Xinyuan Wang, Jiaqi Deng, Yuhao Yang, Cheng Chen, Boyuan Zheng, Alex Su, Xiao Yu, Hao Zou, Saaket Agashe, Xing Han Lu, Manpreet Kaur, Zhengyang Qi, Vincent Sunn Chen, Frederic Sala, Dayiheng Liu, Junyang Lin, Zhou Yu, Yu Su, Siva Reddy, Xin Eric Wang, Peng Qi, Tianbao Xie, Tao Yu","sourceUrl":"https://github.com/xlang-ai/OSWorld-V2/releases","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"arc-prize:v1-semi-private:overall","benchmarkName":"ARC-AGI-1 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":14.330000000000002,"unit":"percent","family":"arc-prize:v1-semi-private:overall","index":33.575377215813916,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.0263,"latency":null},{"benchmarkId":"epoch-ai:epoch-simpleqa-verified:overall","benchmarkName":"SimpleQA Verified","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":12.9,"unit":"percent","family":"epoch-ai:epoch-simpleqa-verified:overall","index":34.66589109749074,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.0499249222407965,"costPerTest":null,"latency":null},{"benchmarkId":"livebench:livebench-long-horizon:overall","benchmarkName":"LiveBench Long Agentic","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":11.1,"unit":"percent","family":"livebench-2-agentic","index":26.707246735987273,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Long%20Agentic","measuredAt":"2026-10-14","stderr":null,"costPerTest":2.5485166666666665,"latency":null},{"benchmarkId":"epoch-ai:epoch-chess-puzzles:overall","benchmarkName":"Chess Puzzles","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":6.5,"unit":"percent","family":"epoch-ai:epoch-chess-puzzles:overall","index":38.918274493614554,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.7265992434429074,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-2025-02-28-private:overall","benchmarkName":"FrontierMath-2025-02-28-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":5.020354406130268,"unit":"percent","family":"epoch-ai:epoch-frontiermath-2025-02-28-private:overall","index":34.346029903181886,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.391155477279583,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-private:overall","benchmarkName":"FrontierMath-Tier-4-2025-07-01-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":2.083333333333333,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-private:overall","index":45.31836331093945,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.083333333333333,"costPerTest":null,"latency":null},{"benchmarkId":"arc-prize:v2-semi-private:overall","benchmarkName":"ARC-AGI-2 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":1.25,"unit":"percent","family":"arc-prize:v2-semi-private:overall","index":39.76243377846274,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.0426,"latency":null},{"benchmarkId":"mazeBench","benchmarkName":"MazeBench: visual spatial reasoning in a 3D open world","category":"agentic","score":0,"unit":"percent","family":"mazeBench","index":61.97866734834783,"aboveFit":false,"run":null,"source":"Jonathan Pappas and David Pappas","sourceUrl":"https://mazebench.com/blog?post=maze-bench-results","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"arc-prize:v2-public-eval:overall","benchmarkName":"ARC-AGI-2 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":0,"unit":"percent","family":"arc-prize:v2-public-eval:overall","index":null,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.0432,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-2025-02-28-public:overall","benchmarkName":"FrontierMath-2025-02-28-Public","category":"math","domain":null,"eligible":false,"ranked":true,"score":0,"unit":"percent","family":"epoch-ai:epoch-frontiermath-2025-02-28-public:overall","index":null,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":1,"outputPerMillion":5,"batch":{"inputPerMillion":0.5,"outputPerMillion":2.5},"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":88,"timeToFirstToken":0.772,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":{"elo":1396.5157588084426,"eloUpper":1398.8947795017298,"eloLower":1394.1367381151554,"votes":154057,"rank":164,"rankUpper":null,"rankLower":null,"arenaName":"Claude Haiku 4.5-20251001","organization":"anthropic","updatedAt":"2026-11-09"},"quality":{"coveragePercent":95,"benchmarkCount":19,"categoryCount":7,"confidence":"medium","state":"available"},"hosts":[{"host":"anthropic","name":"Anthropic","kind":"maker","url":"https://docs.anthropic.com/en/docs/about-claude/models","inputPerMillion":1,"outputPerMillion":5,"cacheReadPerMillion":0.1,"contextTokens":200000,"maxOutputTokens":64000,"deprecated":false},{"host":"databricks","name":"Databricks","kind":"cloud","url":"https://docs.databricks.com/aws/en/machine-learning/foundation-models/","inputPerMillion":1,"outputPerMillion":5,"cacheReadPerMillion":0.1,"contextTokens":200000,"maxOutputTokens":64000,"deprecated":false},{"host":"digitalocean","name":"DigitalOcean","kind":"cloud","url":"https://docs.digitalocean.com/products/gradient-ai-platform/details/models/","inputPerMillion":1,"outputPerMillion":5,"cacheReadPerMillion":0.1,"contextTokens":200000,"maxOutputTokens":8192,"deprecated":false},{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":1,"outputPerMillion":5,"cacheReadPerMillion":0.1,"contextTokens":200000,"maxOutputTokens":64000,"deprecated":false},{"host":"sap-ai-core","name":"SAP AI Core","kind":"cloud","url":"https://help.sap.com/docs/sap-ai-core","inputPerMillion":1,"outputPerMillion":5,"cacheReadPerMillion":0.1,"contextTokens":200000,"maxOutputTokens":64000,"deprecated":false},{"host":"snowflake","name":"Snowflake Cortex","kind":"cloud","url":"https://docs.snowflake.com/en/user-guide/snowflake-cortex/cortex-rest-api","inputPerMillion":1,"outputPerMillion":5,"cacheReadPerMillion":0.1,"contextTokens":200000,"maxOutputTokens":64000,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:swe-bench-team","name":"SWE-bench team","url":"https://github.com","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:livebench","name":"LiveBench","url":"https://livebench.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:arc-prize-foundation","name":"ARC Prize Foundation","url":"https://arcprize.org","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"claudesonnet4","providerId":"anthropic","slug":"anthropic-claude-sonnet-4","name":"Claude Sonnet 4","provider":"Anthropic","description":"Claude Sonnet 4 is a model from Anthropic. 15 benchmarks count toward its index, in 6 categories.","logoUrl":null,"releaseDate":"2025-05-22T16:12:51.000Z","contextWindowTokens":200000,"modalities":["image","text","file"],"openWeights":null,"details":{"apiId":"anthropic/claude-sonnet-4","maxOutputTokens":64000,"knowledgeCutoff":"2025-01-31","retiresAt":null,"tools":true,"structuredOutput":false,"cacheReadPerMillion":0.3,"weightsUrl":null},"reasoning":null,"overallScore":43.03610581610247,"overallScoreRange":{"low":37.86102976889087,"high":48.21118186331407},"badges":[],"capabilityScores":{"agentic":49.405451257831686,"coding":46.20955234510489,"reasoning":38.85724333327647,"multimodal":49.35684325075156,"knowledge":44.46556168611488,"multilingual":null,"instruction":null,"math":38.857508250317224},"capabilityCounts":{"math":6,"agentic":1,"knowledge":2,"coding":3,"multimodal":1,"reasoning":2},"benchmarks":[{"benchmarkId":"vals-ai:mgsm:overall","benchmarkName":"MGSM","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":93.018,"unit":"percent","family":"vals-ai:mgsm:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mgsm","measuredAt":"2026-01-09","stderr":1.563,"costPerTest":0.00576,"latency":5.987},{"benchmarkId":"vals-ai:medqa:overall","benchmarkName":"MedQA","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":90.35,"unit":"percent","family":"vals-ai:medqa:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medqa","measuredAt":"2026-04-16","stderr":0.27,"costPerTest":0.007087,"latency":8.713},{"benchmarkId":"vals-ai:math500:overall","benchmarkName":"MATH 500","category":"math","domain":null,"eligible":true,"ranked":true,"score":90.323,"unit":"percent","family":"math-500","index":39.57635527768133,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/math500","measuredAt":"2026-01-09","stderr":1.288,"costPerTest":0.010201,"latency":10.028},{"benchmarkId":"epoch-ai:epoch-math-level-5:overall","benchmarkName":"MATH level 5","category":"math","domain":null,"eligible":true,"ranked":true,"score":84.3655589123867,"unit":"percent","family":"epoch-ai:epoch-math-level-5:overall","index":39.97893506534422,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0.9984903043396951,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:legal_bench:overall","benchmarkName":"LegalBench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":82.954,"unit":"percent","family":"vals-ai:legal_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","measuredAt":"2026-10-13","stderr":0.406,"costPerTest":null,"latency":4.849},{"benchmarkId":"tau2-bench:retail:overall","benchmarkName":"τ²-bench Retail","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":80.5,"unit":"percent","family":"tau2-bench","index":56.20707565502815,"aboveFit":false,"run":"Anthropic","source":"Sierra Research","sourceUrl":"https://github.com/sierra-research/tau2-bench","measuredAt":"2026-04-30","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:mmlu_pro:overall","benchmarkName":"MMLU Pro","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":79.432,"unit":"percent","family":"mmlu-pro","index":44.55554066094999,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","measuredAt":"2026-09-01","stderr":0.372,"costPerTest":0.010229,"latency":9.707},{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":77.02020202020202,"unit":"percent","family":"gpqa-diamond","index":47.849925447580446,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.937661648494561,"costPerTest":null,"latency":null},{"benchmarkId":"swebench-experiments:swe-bench-verified:overall","benchmarkName":"SWE-bench Verified","category":"coding","domain":null,"eligible":true,"ranked":true,"score":74.8,"unit":"percent","family":"swe-bench-verified","index":56.3972343476918,"aboveFit":false,"run":"Harness AI","source":"SWE-bench team","sourceUrl":"https://github.com/SWE-bench/experiments/tree/main/evaluation/verified","measuredAt":"2026-09-01","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:medscribe:overall","benchmarkName":"MedScribe","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":72.411,"unit":"percent","family":"vals-ai:medscribe:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medscribe","measuredAt":"2026-10-29","stderr":1.929,"costPerTest":0.038973,"latency":25.674},{"benchmarkId":"vals-ai:mmmu:overall","benchmarkName":"MMMU Pro","category":"multimodal","domain":null,"eligible":true,"ranked":true,"score":72.386,"unit":"percent","family":"mmmu-pro","index":49.35684325075156,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmmu","measuredAt":"2026-09-01","stderr":1.075,"costPerTest":0.012845,"latency":10.823},{"benchmarkId":"vals-ai:tax_eval_v2:overall","benchmarkName":"TaxEval v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":69.624,"unit":"percent","family":"vals-ai:tax_eval_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","measuredAt":"2026-09-01","stderr":0.899,"costPerTest":0.0076,"latency":11.847},{"benchmarkId":"vals-ai:gpqa:overall","benchmarkName":"GPQA Diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":69.444,"unit":"percent","family":"gpqa-diamond","index":40.901239974979084,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","measuredAt":"2026-09-01","stderr":2.315,"costPerTest":0.013552,"latency":12.818},{"benchmarkId":"vals-ai:mortgage_tax:overall","benchmarkName":"MortgageTax","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":62.468,"unit":"percent","family":"vals-ai:mortgage_tax:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mortgage_tax","measuredAt":"2026-09-01","stderr":0.957,"costPerTest":0.015974,"latency":8.422},{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":61.111111111111114,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":44.47317144914222,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":6.979205927323111,"costPerTest":null,"latency":null},{"benchmarkId":"tau2-bench:airline:overall","benchmarkName":"τ²-bench Airline","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":60,"unit":"percent","family":"tau2-bench","index":42.60382686063522,"aboveFit":false,"run":"Anthropic","source":"Sierra Research","sourceUrl":"https://github.com/sierra-research/tau2-bench","measuredAt":"2026-03-02","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:lcb:overall","benchmarkName":"LiveCodeBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":59.673,"unit":"percent","family":"livecodebench","index":38.85691486177923,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","measuredAt":"2026-09-01","stderr":1.168,"costPerTest":0.013812,"latency":14.058},{"benchmarkId":"vals-ai:corp_fin_v2:overall","benchmarkName":"CorpFin v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":54.701,"unit":"percent","family":"vals-ai:corp_fin_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","measuredAt":"2026-08-12","stderr":0.98,"costPerTest":0.227257,"latency":186.345},{"benchmarkId":"vals-ai:aime:overall","benchmarkName":"AIME","category":"math","domain":null,"eligible":true,"ranked":true,"score":38.542,"unit":"percent","family":"vals-ai:aime:overall","index":32.34718368887164,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/aime","measuredAt":"2026-04-16","stderr":0.968,"costPerTest":0.02164,"latency":23.398},{"benchmarkId":"swe-bench:swe-bench-multimodal:overall","benchmarkName":"SWE-bench Multimodal","category":"multimodal","domain":null,"eligible":true,"ranked":true,"score":34.43,"unit":"percent","family":"swe-bench-multimodal","index":null,"aboveFit":false,"run":"OpenHands-Versa","source":"SWE-bench team","sourceUrl":"https://www.swebench.com/","measuredAt":"2025-11-17","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:medcode:overall","benchmarkName":"MedCode","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":33.943,"unit":"percent","family":"vals-ai:medcode:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medcode","measuredAt":"2026-10-29","stderr":1.906,"costPerTest":0.03946,"latency":7.298},{"benchmarkId":"arc-prize:v1-public-eval:overall","benchmarkName":"ARC-AGI-1 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":33,"unit":"percent","family":"arc-prize:v1-public-eval:overall","index":null,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.0696,"latency":null},{"benchmarkId":"arc-prize:v1-semi-private:overall","benchmarkName":"ARC-AGI-1 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":23.830000000000002,"unit":"percent","family":"arc-prize:v1-semi-private:overall","index":37.94287019407068,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.0806,"latency":null},{"benchmarkId":"vals-ai:ioi-v1:overall","benchmarkName":"IOI v1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":6.5,"unit":"percent","family":"vals-ai:ioi-v1:overall","index":43.374507825843644,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/ioi-v1","measuredAt":"2026-08-09","stderr":2.096,"costPerTest":2.388928,"latency":3253.12},{"benchmarkId":"epoch-ai:epoch-frontiermath-2025-02-28-private:overall","benchmarkName":"FrontierMath-2025-02-28-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":4.137931034482759,"unit":"percent","family":"epoch-ai:epoch-frontiermath-2025-02-28-private:overall","index":33.567046293698304,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.1715642254112493,"costPerTest":null,"latency":null},{"benchmarkId":"arc-prize:v2-public-eval:overall","benchmarkName":"ARC-AGI-2 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":2.08,"unit":"percent","family":"arc-prize:v2-public-eval:overall","index":null,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.1311,"latency":null},{"benchmarkId":"arc-prize:v2-semi-private:overall","benchmarkName":"ARC-AGI-2 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":1.27,"unit":"percent","family":"arc-prize:v2-semi-private:overall","index":39.771616472482265,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.1272,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-public:overall","benchmarkName":"FrontierMath-Tier-4-2025-07-01-Public","category":"math","domain":null,"eligible":false,"ranked":true,"score":0,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-public:overall","index":null,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-private:overall","benchmarkName":"FrontierMath-Tier-4-2025-07-01-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":0,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-private:overall","index":43.20235772716562,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-2025-02-28-public:overall","benchmarkName":"FrontierMath-2025-02-28-Public","category":"math","domain":null,"eligible":false,"ranked":true,"score":0,"unit":"percent","family":"epoch-ai:epoch-frontiermath-2025-02-28-public:overall","index":null,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":3,"outputPerMillion":15,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":49,"timeToFirstToken":1.117,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":{"elo":1339.3014487566406,"eloUpper":1343.7332742192982,"eloLower":1334.8696232939833,"votes":39126,"rank":220,"rankUpper":null,"rankLower":null,"arenaName":"Claude Sonnet 4-20250514","organization":"anthropic","updatedAt":"2026-11-09"},"quality":{"coveragePercent":90,"benchmarkCount":15,"categoryCount":6,"confidence":"medium","state":"available"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:sierra-research","name":"Sierra Research","url":"https://github.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:swe-bench-team","name":"SWE-bench team","url":"https://github.com","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:arc-prize-foundation","name":"ARC Prize Foundation","url":"https://arcprize.org","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"deepseek-v3-1-reasoning","providerId":"deepseek","slug":"deepseek-v3-1-reasoning","name":"DeepSeek V3.1 (Reasoning)","provider":"DeepSeek","description":"DeepSeek V3.1 (Reasoning) is a reasoning model from DeepSeek in the DeepSeek V3.1 family. 11 benchmarks count toward its index, in 6 categories.","logoUrl":null,"releaseDate":"2025-08-21","contextWindowTokens":128000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":true,"overallScore":43.01334998628673,"overallScoreRange":{"low":36.63791154446429,"high":49.388788428109166},"badges":[],"capabilityScores":{"agentic":27.60707453120693,"coding":51.408410432903665,"reasoning":43.45984895302416,"multimodal":null,"knowledge":46.986760249363805,"multilingual":null,"instruction":27.805819907489273,"math":46.54851678222532},"capabilityCounts":{"math":2,"knowledge":4,"coding":1,"reasoning":2,"instruction":1,"agentic":1},"benchmarks":[{"benchmarkId":"aime2024","benchmarkName":"American Invitational Mathematics Examination 2024","category":"math","score":93.1,"unit":"percent","family":"aime2024","index":46.121689640605574,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaAime2025","benchmarkName":"Artificial Analysis AIME 2025","category":"math","score":89.7,"unit":"percent","family":"aaAime2025","index":null,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/aime-2025","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aime2025","benchmarkName":"American Invitational Mathematics Examination 2025","category":"math","score":88.4,"unit":"percent","family":"aime2025","index":46.97534392384507,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaMmluPro","benchmarkName":"Artificial Analysis MMLU-Pro","category":"knowledge","score":85.1,"unit":"percent","family":"mmlu-pro","index":51.69654221878969,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmlu-pro","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmluPro","benchmarkName":"Massive Multitask Language Understanding Professional","category":"knowledge","score":84.8,"unit":"percent","family":"mmlu-pro","index":51.31857812786663,"aboveFit":false,"run":null,"source":"Yubo Wang, Xueguang Ma, Ge Zhang, Yuansheng Ni, Abhranil Chandra, Shiguang Guo, Weiming Ren, Aaran Arulraj, Xuan He, Ziyan Jiang, Tianle Li, Max Ku, Kai Wang, Alex Zhuang, Rongqi Fan, Xiang Yue, Wenhu Chen","sourceUrl":"https://arxiv.org/abs/2406.01574","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gpqa","benchmarkName":"Graduate-Level Google-Proof Q&A","category":"knowledge","score":80.1,"unit":"percent","family":"gpqa-diamond","index":50.674631998924305,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":77.9,"unit":"percent","family":"gpqa-diamond","index":48.656852114100786,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"liveCodeBench","benchmarkName":"LiveCodeBench: Holistic and Contamination Free Evaluation of Large Language Models for Code","category":"coding","score":74.8,"unit":"percent","family":"livecodebench","index":51.408410432903665,"aboveFit":false,"run":null,"source":"Naman Jain, King Han, Alex Gu, Wen-Ding Li, Fanjia Yan, Tianjun Zhang, Sida Wang, Armando Solar-Lezama, Koushik Sen, Ion Stoica","sourceUrl":"https://arxiv.org/abs/2403.07974","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":56.7,"unit":"percent","family":"lcr","index":46.19881525987764,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaIfBench","benchmarkName":"Artificial Analysis IFBench","category":"instruction","score":41.5,"unit":"percent","family":"ifbench-family","index":27.805819907489273,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"tau2Bench","benchmarkName":"τ²-Bench Tool-Agent-User Evaluation","category":"agentic","score":37.4,"unit":"percent","family":"tau2-bench","index":27.60707453120693,"aboveFit":false,"run":null,"source":"Victor Barres, Honghua Dong, Soham Ray, Xujie Si, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2506.07982","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"omniscienceAccuracy","benchmarkName":"Artificial Analysis Omniscience Accuracy","category":"knowledge","score":29,"unit":"percent","family":"omniscienceAccuracy","index":47.23737400435114,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hle","benchmarkName":"Humanity's Last Exam","category":"knowledge","score":15.9,"unit":"percent","family":"hle","index":40.33899895928721,"aboveFit":false,"run":null,"source":"Center for AI Safety, Scale AI, and thousands of expert contributors","sourceUrl":"https://lastexam.ai/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":14.3,"unit":"percent","family":"hle","index":38.73373056723953,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":2,"unit":"percent","family":"critpt","index":40.72088264617068,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":85,"benchmarkCount":11,"categoryCount":6,"confidence":"low","state":"available"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"mistralmedium35","providerId":"mistral","slug":"mistral-mistral-medium-3-5","name":"Mistral Medium 3.5","provider":"Mistral","description":"Mistral Medium 3.5 is a model from Mistral. 9 benchmarks count toward its index, in 4 categories.","logoUrl":null,"releaseDate":"2026-04-30T17:33:59.000Z","contextWindowTokens":262144,"modalities":["text","image","file"],"openWeights":true,"details":{"apiId":"mistralai/mistral-medium-3-5","maxOutputTokens":209715,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":null,"weightsUrl":"https://huggingface.co/mistralai/Mistral-Medium-3.5-128B"},"reasoning":null,"overallScore":42.692164145865306,"overallScoreRange":{"low":33.802164145865305,"high":51.582164145865306},"badges":[],"capabilityScores":{"agentic":47.12135103980981,"coding":47.1353714880651,"reasoning":null,"multimodal":null,"knowledge":24.282272586349116,"multilingual":null,"instruction":null,"math":50.26382289960114},"capabilityCounts":{"knowledge":2,"coding":3,"agentic":3,"math":1},"benchmarks":[{"benchmarkId":"vals-ai:mmlu_pro:overall","benchmarkName":"MMLU Pro","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":75.335,"unit":"percent","family":"mmlu-pro","index":39.39381105924403,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","measuredAt":"2026-09-01","stderr":0.418,"costPerTest":0.041763,"latency":66.384},{"benchmarkId":"vals-ai:sage:overall","benchmarkName":"SAGE","category":null,"domain":"Education","eligible":false,"ranked":true,"score":71.965,"unit":"percent","family":"vals-ai:sage:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/sage","measuredAt":"2026-11-05","stderr":2.215,"costPerTest":0.073632,"latency":46.75},{"benchmarkId":"vals-ai:tax_eval_v2:overall","benchmarkName":"TaxEval v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":67.988,"unit":"percent","family":"vals-ai:tax_eval_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","measuredAt":"2026-09-01","stderr":0.916,"costPerTest":0.079101,"latency":102.392},{"benchmarkId":"vals-ai:medscribe:overall","benchmarkName":"MedScribe","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":67.728,"unit":"percent","family":"vals-ai:medscribe:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medscribe","measuredAt":"2026-10-29","stderr":2.011,"costPerTest":0.157641,"latency":104.053},{"benchmarkId":"vals-ai:swebench:overall","benchmarkName":"SWE-bench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":66.4,"unit":"percent","family":"swe-bench-verified","index":50.20175069430879,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/swebench","measuredAt":"2026-09-01","stderr":2.114,"costPerTest":4.004623,"latency":1143.68},{"benchmarkId":"vals-ai:corp_fin_v2:overall","benchmarkName":"CorpFin v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":58.78,"unit":"percent","family":"vals-ai:corp_fin_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","measuredAt":"2026-08-12","stderr":0.969,"costPerTest":0.14574,"latency":43.699},{"benchmarkId":"vals-ai:finance_agent:overall","benchmarkName":"Finance Agent v1.1","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":46.113,"unit":"percent","family":"vals-ai:finance_agent:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/finance_agent","measuredAt":"2026-06-04","stderr":2.883,"costPerTest":1.147222,"latency":562.191},{"benchmarkId":"vals-ai:case_law_v2:overall","benchmarkName":"CaseLaw v2","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":44.159,"unit":"percent","family":"vals-ai:case_law_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/case_law_v2","measuredAt":"2026-05-04","stderr":1.056,"costPerTest":0.084987,"latency":64.635},{"benchmarkId":"vals-ai:terminal-bench-2-1:overall","benchmarkName":"Terminal-Bench 2.1","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":38.951,"unit":"percent","family":"terminal-bench-2-1","index":43.92182250894197,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2-1","measuredAt":"2026-10-01","stderr":2.701,"costPerTest":1.190984,"latency":989.169},{"benchmarkId":"vals-ai:gpqa:overall","benchmarkName":"GPQA Diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":34.848,"unit":"percent","family":"gpqa-diamond","index":9.170734113454202,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","measuredAt":"2026-09-01","stderr":2.39,"costPerTest":0.095489,"latency":256.17},{"benchmarkId":"vals-ai:medcode:overall","benchmarkName":"MedCode","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":33.752,"unit":"percent","family":"vals-ai:medcode:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medcode","measuredAt":"2026-10-29","stderr":1.148,"costPerTest":0.05337,"latency":34.879},{"benchmarkId":"vals-ai:fabv2:overall","benchmarkName":"Finance Agent (v2)","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":32.103,"unit":"percent","family":"vals-ai:fabv2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/fabv2","measuredAt":"2026-10-29","stderr":0.967,"costPerTest":1.908513,"latency":768.405},{"benchmarkId":"vals-ai:terminal-bench-2:overall","benchmarkName":"Terminal-Bench 2.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":30.337,"unit":"percent","family":"vals-ai:terminal-bench-2:overall","index":44.817513677231766,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2","measuredAt":"2026-06-04","stderr":4.901,"costPerTest":4.710448,"latency":1253.133},{"benchmarkId":"vals-ai:mortgage_tax:overall","benchmarkName":"MortgageTax","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":28.895,"unit":"percent","family":"vals-ai:mortgage_tax:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mortgage_tax","measuredAt":"2026-09-01","stderr":0.838,"costPerTest":0.024032,"latency":68.43},{"benchmarkId":"vals-ai:emb:overall","benchmarkName":"EMB","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":11.185,"unit":"percent","family":"vals-ai:emb:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/emb","measuredAt":"2026-10-20","stderr":1.637,"costPerTest":10.603072,"latency":3082.124},{"benchmarkId":"vals-ai:legal_research:overall","benchmarkName":"Legal Research Bench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":9.135,"unit":"percent","family":"vals-ai:legal_research:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_research","measuredAt":"2026-11-09","stderr":2.002,"costPerTest":1.308079,"latency":613.248},{"benchmarkId":"vals-ai:proof_bench:overall","benchmarkName":"ProofBench v1.1","category":"math","domain":null,"eligible":true,"ranked":true,"score":9,"unit":"percent","family":"vals-ai:proof_bench:overall","index":50.26382289960114,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/proof_bench","measuredAt":"2026-11-09","stderr":2.876,"costPerTest":1.302032,"latency":479.048},{"benchmarkId":"vals-ai:code-migration:overall","benchmarkName":"Code Migration","category":"coding","domain":null,"eligible":true,"ranked":true,"score":5.127,"unit":"percent","family":"vals-ai:code-migration:overall","index":47.912805405042,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/code-migration","measuredAt":"2026-10-29","stderr":1.369,"costPerTest":18.623756,"latency":13055.328},{"benchmarkId":"vals-ai:tax_agent_bench:overall","benchmarkName":"Tax Agent Bench","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":4.482,"unit":"percent","family":"vals-ai:tax_agent_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench","measuredAt":"2026-11-09","stderr":1.47,"costPerTest":1.443655,"latency":430.673},{"benchmarkId":"vals-ai:vibe-code:overall","benchmarkName":"Vibe Code Bench v1.1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":2.888,"unit":"percent","family":"vals-ai:vibe-code:overall","index":43.2915583648445,"aboveFit":false,"run":"OpenHands","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","measuredAt":"2026-11-09","stderr":1.713,"costPerTest":16.735503,"latency":2470.533},{"benchmarkId":"lmarena:agent-task-outcome:overall","benchmarkName":"Agent Arena task outcome","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":-16.794319584035666,"unit":"index","family":"agent-arena","index":52.723237744780604,"aboveFit":false,"run":null,"source":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset","measuredAt":"2026-11-11","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lmarena:agent-steerability:overall","benchmarkName":"Agent Arena steerability","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":-16.914151731578286,"unit":"index","family":"agent-arena","index":52.6247169332557,"aboveFit":false,"run":null,"source":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset","measuredAt":"2026-11-11","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lmarena:agent-bash-recovery:overall","benchmarkName":"Agent Arena command recovery","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":-20.447688894985472,"unit":"index","family":"agent-arena","index":49.719595427622394,"aboveFit":false,"run":null,"source":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset","measuredAt":"2026-10-08","stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":1.5,"outputPerMillion":7.5,"batch":{"inputPerMillion":0.75,"outputPerMillion":3.75},"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":92,"timeToFirstToken":0.6465,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":{"elo":1420.9720341412935,"eloUpper":1427.3706100667546,"eloLower":1414.5734582158325,"votes":11656,"rank":114,"rankUpper":null,"rankLower":null,"arenaName":"Mistral Medium 3.5","organization":"mistral","updatedAt":"2026-11-09"},"quality":{"coveragePercent":65,"benchmarkCount":9,"categoryCount":4,"confidence":"low","state":"available"},"hosts":[{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":1.5,"outputPerMillion":7.5,"cacheReadPerMillion":null,"contextTokens":262144,"maxOutputTokens":209715,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:lmarena","name":"LMArena","url":"https://huggingface.co","license":"CC BY 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"k-exaone","providerId":"lg-ai-research","slug":"k-exaone","name":"K-Exaone","provider":"LG AI Research","description":"K-Exaone is a reasoning model from LG AI Research. 9 benchmarks count toward its index, in 5 categories.","logoUrl":null,"releaseDate":"2025-12-31","contextWindowTokens":256000,"modalities":["text"],"openWeights":false,"details":null,"reasoning":true,"overallScore":42.31789146270353,"overallScoreRange":{"low":34.42789146270353,"high":50.20789146270353},"badges":[],"capabilityScores":{"agentic":43.06334863915349,"coding":37.663271871986026,"reasoning":44.16668666935668,"multimodal":null,"knowledge":39.571427752832435,"multilingual":null,"instruction":49.48646617153375,"math":null},"capabilityCounts":{"knowledge":3,"agentic":2,"instruction":1,"reasoning":2,"coding":1},"benchmarks":[{"benchmarkId":"aaAime2025","benchmarkName":"Artificial Analysis AIME 2025","category":"math","score":90.3,"unit":"percent","family":"aaAime2025","index":null,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/aime-2025","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":78.3,"unit":"percent","family":"gpqa-diamond","index":49.0237211840687,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"tau2Bench","benchmarkName":"τ²-Bench Tool-Agent-User Evaluation","category":"agentic","score":74.3,"unit":"percent","family":"tau2-bench","index":52.092922361114184,"aboveFit":false,"run":null,"source":"Victor Barres, Honghua Dong, Soham Ray, Xujie Si, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2506.07982","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaIfBench","benchmarkName":"Artificial Analysis IFBench","category":"instruction","score":64.7,"unit":"percent","family":"ifbench-family","index":49.48646617153375,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":61.3,"unit":"percent","family":"lcr","index":49.35478567757167,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"sweVerified","benchmarkName":"Software Engineering Benchmark Verified","category":"coding","score":49.4,"unit":"percent","family":"swe-bench-verified","index":37.663271871986026,"aboveFit":false,"run":null,"source":"Carlos E. Jimenez, John Yang, Alexander Wettig, Shunyu Yao, Kexin Pei, Ofir Press, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2310.06770","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"omniscienceAccuracy","benchmarkName":"Artificial Analysis Omniscience Accuracy","category":"knowledge","score":16.4,"unit":"percent","family":"omniscienceAccuracy","index":31.358148605200977,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":13.9,"unit":"percent","family":"hle","index":38.33241346922762,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":1.1,"unit":"percent","family":"critpt","index":38.97858766114168,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gdpvalAaNormalized","benchmarkName":"GDPval-AA normalized","category":"agentic","score":0,"unit":"percent","family":"gdpvalAaNormalized","index":34.03377491719279,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":75,"benchmarkCount":9,"categoryCount":5,"confidence":"low","state":"available"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"qwen3-235b-2507","providerId":"alibaba","slug":"qwen3-235b-2507","name":"Qwen3 235B 2507","provider":"Alibaba","description":"Qwen3 235B 2507 is a standard model from Alibaba. 10 benchmarks count toward its index, in 6 categories.","logoUrl":null,"releaseDate":"2025-07-21","contextWindowTokens":262144,"modalities":["text"],"openWeights":true,"details":{"apiId":"qwen/qwen3-235b-a22b-2507","maxOutputTokens":117964,"knowledgeCutoff":"2025-06-30","retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.049999999999999996,"weightsUrl":"https://huggingface.co/Qwen/Qwen3-235B-A22B-Instruct-2507"},"reasoning":false,"overallScore":42.297562034489374,"overallScoreRange":{"low":34.18415905799711,"high":50.41096501098164},"badges":[],"capabilityScores":{"agentic":null,"coding":26.947831327163065,"reasoning":35.90344488568596,"multimodal":null,"knowledge":51.173418610755064,"multilingual":48.83174101403117,"instruction":41.81670978506855,"math":37.153816648101085},"capabilityCounts":{"instruction":1,"knowledge":4,"multilingual":1,"math":1,"coding":1,"reasoning":2},"benchmarks":[{"benchmarkId":"ifeval","benchmarkName":"Instruction-Following Eval","category":"instruction","score":88.7,"unit":"percent","family":"ifeval","index":41.81670978506855,"aboveFit":false,"run":null,"source":"Jeffrey Zhou, Tianjian Lu, Swaroop Mishra, Siddhartha Brahma, Sujoy Basu, Yi Luan, Denny Zhou, Le Hou","sourceUrl":"https://arxiv.org/abs/2311.07911","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmluPro","benchmarkName":"Massive Multitask Language Understanding Professional","category":"knowledge","score":83,"unit":"percent","family":"mmlu-pro","index":49.05079358232827,"aboveFit":false,"run":null,"source":"Yubo Wang, Xueguang Ma, Ge Zhang, Yuansheng Ni, Abhranil Chandra, Shiguang Guo, Weiming Ren, Aaran Arulraj, Xuan He, Ziyan Jiang, Tianle Li, Max Ku, Kai Wang, Alex Zhuang, Rongqi Fan, Xiang Yue, Wenhu Chen","sourceUrl":"https://arxiv.org/abs/2406.01574","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmluProX","benchmarkName":"MMLU-ProX","category":"multilingual","score":79.4,"unit":"percent","family":"mmluProX","index":48.83174101403117,"aboveFit":false,"run":null,"source":"MMLU-ProX authors","sourceUrl":"https://arxiv.org/abs/2503.10497","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gpqa","benchmarkName":"Graduate-Level Google-Proof Q&A","category":"knowledge","score":77.5,"unit":"percent","family":"gpqa-diamond","index":48.28998304413287,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aime2025","benchmarkName":"American Invitational Mathematics Examination 2025","category":"math","score":70.3,"unit":"percent","family":"aime2025","index":37.153816648101085,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"superGpqa","benchmarkName":"SuperGPQA: Scaling LLM Evaluation Across 285 Graduate Disciplines","category":"knowledge","score":62.6,"unit":"percent","family":"superGpqa","index":48.562078818548066,"aboveFit":false,"run":null,"source":"Xiaoxuan Du, Yao Yao, Kexin Ma, Bowen Wang, Tianyu Zheng, Kaiyan Zhu, Yiming Zhang, Yutao Zhu, Jiawei Zhou, Jingren Zhou","sourceUrl":"https://arxiv.org/abs/2502.14739","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"simpleQa","benchmarkName":"Measuring Short-Form Factuality in Large Language Models","category":"knowledge","score":54.3,"unit":"percent","family":"simpleQa","index":58.79081899801104,"aboveFit":false,"run":null,"source":"Jason Wei, Najoung Kim, Hyung Won Chung, Yu-An Chung, Siddhartha Papay, Yifeng Lu, Hannaneh Hajishirzi, Luke Zettlemoyer","sourceUrl":"https://arxiv.org/abs/2411.04368","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"liveCodeBenchV6","benchmarkName":"LiveCodeBench v6","category":"coding","score":51.8,"unit":"percent","family":"liveCodeBenchV6","index":26.947831327163065,"aboveFit":false,"run":null,"source":"LiveCodeBench maintainers","sourceUrl":"https://github.com/LiveCodeBench/LiveCodeBench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"arc-prize:v1-public-eval:overall","benchmarkName":"ARC-AGI-1 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":17,"unit":"percent","family":"arc-prize:v1-public-eval:overall","index":null,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.0021,"latency":null},{"benchmarkId":"arc-prize:v1-semi-private:overall","benchmarkName":"ARC-AGI-1 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":11,"unit":"percent","family":"arc-prize:v1-semi-private:overall","index":32.04445599290917,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.0025,"latency":null},{"benchmarkId":"arc-prize:v2-semi-private:overall","benchmarkName":"ARC-AGI-2 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":1.25,"unit":"percent","family":"arc-prize:v2-semi-private:overall","index":39.76243377846274,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.0044,"latency":null},{"benchmarkId":"arc-prize:v2-public-eval:overall","benchmarkName":"ARC-AGI-2 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":0.83,"unit":"percent","family":"arc-prize:v2-public-eval:overall","index":null,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.0045,"latency":null}],"price":{"inputPerMillion":0.14,"outputPerMillion":0.7999999999999999,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":48,"timeToFirstToken":0.392,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":{"elo":1419.1703362744033,"eloUpper":1421.7099955923068,"eloLower":1416.6306769565003,"votes":97642,"rank":123,"rankUpper":null,"rankLower":null,"arenaName":"Qwen3-235B-A22B-Instruct-2507","organization":"alibaba","updatedAt":"2026-11-09"},"quality":{"coveragePercent":70,"benchmarkCount":10,"categoryCount":6,"confidence":"low","state":"available"},"hosts":[{"host":"deepinfra","name":"Deep Infra","kind":"host","url":"https://deepinfra.com/models","inputPerMillion":0.09,"outputPerMillion":0.55,"cacheReadPerMillion":null,"contextTokens":262144,"maxOutputTokens":16384,"deprecated":false},{"host":"nebius","name":"Nebius","kind":"host","url":"https://docs.tokenfactory.nebius.com/","inputPerMillion":0.2,"outputPerMillion":0.6,"cacheReadPerMillion":0.2,"contextTokens":262144,"maxOutputTokens":8192,"deprecated":false},{"host":"together-ai","name":"Together AI","kind":"host","url":"https://docs.together.ai/docs/serverless-models","inputPerMillion":0.2,"outputPerMillion":0.6,"cacheReadPerMillion":null,"contextTokens":262144,"maxOutputTokens":262144,"deprecated":true},{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":0.14,"outputPerMillion":0.8,"cacheReadPerMillion":0.05,"contextTokens":262144,"maxOutputTokens":117964,"deprecated":false},{"host":"crusoe","name":"Crusoe","kind":"host","url":"https://docs.crusoecloud.com/managed-inference/overview","inputPerMillion":0.22,"outputPerMillion":0.8,"cacheReadPerMillion":0.11,"contextTokens":262144,"maxOutputTokens":16384,"deprecated":true},{"host":"amazon-bedrock","name":"Amazon Bedrock","kind":"cloud","url":"https://docs.aws.amazon.com/bedrock/latest/userguide/models-supported.html","inputPerMillion":0.22,"outputPerMillion":0.88,"cacheReadPerMillion":null,"contextTokens":262144,"maxOutputTokens":131072,"deprecated":false},{"host":"scaleway","name":"Scaleway","kind":"cloud","url":"https://www.scaleway.com/en/docs/generative-apis/","inputPerMillion":0.841125,"outputPerMillion":2.523375,"cacheReadPerMillion":null,"contextTokens":260000,"maxOutputTokens":16384,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:arc-prize-foundation","name":"ARC Prize Foundation","url":"https://arcprize.org","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"trinity-large-preview","providerId":"arcee-ai","slug":"trinity-large-preview","name":"Trinity-Large-Preview","provider":"Arcee AI","description":"Trinity-Large-Preview is a standard model from Arcee AI in the Trinity Large family. 10 benchmarks count toward its index, in 5 categories.","logoUrl":null,"releaseDate":"2026-01-27","contextWindowTokens":512000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":42.17715257722539,"overallScoreRange":{"low":34.563749600733125,"high":49.79055555371766},"badges":[],"capabilityScores":{"agentic":48.30557622333417,"coding":48.76965898694944,"reasoning":35.98026021461385,"multimodal":null,"knowledge":38.9585314473571,"multilingual":null,"instruction":41.636577006965915,"math":null},"capabilityCounts":{"agentic":2,"knowledge":4,"instruction":1,"coding":1,"reasoning":2},"benchmarks":[{"benchmarkId":"tau2Bench","benchmarkName":"τ²-Bench Tool-Agent-User Evaluation","category":"agentic","score":90.1,"unit":"percent","family":"tau2-bench","index":62.577377529475555,"aboveFit":true,"run":null,"source":"Victor Barres, Honghua Dong, Soham Ray, Xujie Si, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2506.07982","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmlu","benchmarkName":"Massive Multitask Language Understanding","category":"knowledge","score":87.2,"unit":"percent","family":"mmlu","index":35.64842041027089,"aboveFit":false,"run":null,"source":"Dan Hendrycks, Collin Burns, Steven Basart, Andy Zou, Mantas Mazeika, Dawn Song, Jacob Steinhardt","sourceUrl":"https://arxiv.org/abs/2009.03300","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmluProArcee","benchmarkName":"MMLU-Pro first-party comparison snapshot","category":"knowledge","score":75.2,"unit":"percent","family":"mmlu-pro","index":39.22372721832867,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":75.2,"unit":"percent","family":"gpqa-diamond","index":46.18048589181737,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gpqaDiamond","benchmarkName":"GPQA Diamond","category":"knowledge","score":63.3,"unit":"percent","family":"gpqa-diamond","index":35.26613106027192,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaIfBench","benchmarkName":"Artificial Analysis IFBench","category":"instruction","score":56.3,"unit":"percent","family":"ifbench-family","index":41.636577006965915,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaSciCode","benchmarkName":"Artificial Analysis SciCode","category":"coding","score":40.6,"unit":"percent","family":"scicode-family","index":48.76965898694944,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/scicode","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":38,"unit":"percent","family":"lcr","index":33.36910943142579,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aime2025Arcee","benchmarkName":"AIME25 first-party comparison snapshot","category":"math","score":24,"unit":"percent","family":"aime2025Arcee","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":15.8,"unit":"percent","family":"hle","index":40.23866968478423,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":0.9,"unit":"percent","family":"critpt","index":38.5914109978019,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gdpvalAaNormalized","benchmarkName":"GDPval-AA normalized","category":"agentic","score":0,"unit":"percent","family":"gdpvalAaNormalized","index":34.03377491719279,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":0.25,"outputPerMillion":1,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":{"elo":1338.897110469858,"eloUpper":1343.1434248406963,"eloLower":1334.6507960990195,"votes":31031,"rank":221,"rankUpper":null,"rankLower":null,"arenaName":"Trinity Large Preview","organization":"","updatedAt":"2026-11-09"},"quality":{"coveragePercent":75,"benchmarkCount":10,"categoryCount":5,"confidence":"low","state":"available"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gpt-5-nano","providerId":"openai","slug":"gpt-5-nano","name":"GPT-5 nano","provider":"OpenAI","description":"GPT-5 nano is a reasoning model from OpenAI in the GPT-5 family. 19 benchmarks count toward its index, in 6 categories.","logoUrl":null,"releaseDate":"2025-08-07","contextWindowTokens":400000,"modalities":["text","image","file"],"openWeights":false,"details":{"apiId":"openai/gpt-5-nano","maxOutputTokens":128000,"knowledgeCutoff":"2024-05-31","retiresAt":"2026-12-11","tools":true,"structuredOutput":true,"cacheReadPerMillion":0.005,"weightsUrl":null},"reasoning":true,"overallScore":41.911013456756606,"overallScoreRange":{"low":35.70136104006962,"high":48.12066587344359},"badges":[],"capabilityScores":{"agentic":null,"coding":37.249903937749615,"reasoning":42.64933601567923,"multimodal":47.47760393991948,"knowledge":37.368213063573734,"multilingual":44.46327969561767,"instruction":null,"math":45.84119920373904},"capabilityCounts":{"math":8,"knowledge":3,"multimodal":1,"coding":2,"multilingual":1,"reasoning":4},"benchmarks":[{"benchmarkId":"epoch-ai:epoch-math-level-5:overall","benchmarkName":"MATH level 5","category":"math","domain":null,"eligible":true,"ranked":true,"score":94.90181268882175,"unit":"percent","family":"epoch-ai:epoch-math-level-5:overall","index":45.09697700328795,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0.48488515751137634,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:math500:overall","benchmarkName":"MATH 500","category":"math","domain":null,"eligible":true,"ranked":true,"score":93.8,"unit":"percent","family":"math-500","index":43.25070358230003,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/math500","measuredAt":"2026-01-09","stderr":1.078,"costPerTest":0.001664,"latency":22.396},{"benchmarkId":"vals-ai:medqa:overall","benchmarkName":"MedQA","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":93.258,"unit":"percent","family":"vals-ai:medqa:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medqa","measuredAt":"2026-04-16","stderr":0.229,"costPerTest":0.001254,"latency":20.462},{"benchmarkId":"vals-ai:mgsm:overall","benchmarkName":"MGSM","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":89.309,"unit":"percent","family":"vals-ai:mgsm:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mgsm","measuredAt":"2026-01-09","stderr":1.892,"costPerTest":0.001888,"latency":22.775},{"benchmarkId":"vals-ai:aime:overall","benchmarkName":"AIME","category":"math","domain":null,"eligible":true,"ranked":true,"score":81.181,"unit":"percent","family":"vals-ai:aime:overall","index":50.982377868048474,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/aime","measuredAt":"2026-04-16","stderr":1.29,"costPerTest":0.008862,"latency":193.682},{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":81.11111111111111,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":56.10905236380394,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":4.626556218905063,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:mmlu_pro:overall","benchmarkName":"MMLU Pro","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":76.067,"unit":"percent","family":"mmlu-pro","index":40.316043441096305,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","measuredAt":"2026-09-01","stderr":0.412,"costPerTest":0.001887,"latency":26.107},{"benchmarkId":"vals-ai:medscribe:overall","benchmarkName":"MedScribe","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":72.865,"unit":"percent","family":"vals-ai:medscribe:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medscribe","measuredAt":"2026-10-29","stderr":1.891,"costPerTest":0.006961,"latency":112.909},{"benchmarkId":"vals-ai:mmmu:overall","benchmarkName":"MMMU Pro","category":"multimodal","domain":null,"eligible":true,"ranked":true,"score":70.942,"unit":"percent","family":"mmmu-pro","index":47.47760393991948,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmmu","measuredAt":"2026-09-01","stderr":1.091,"costPerTest":0.002619,"latency":30.776},{"benchmarkId":"vals-ai:lcb:overall","benchmarkName":"LiveCodeBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":70.216,"unit":"percent","family":"livecodebench","index":47.6048766391551,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","measuredAt":"2026-09-01","stderr":0.927,"costPerTest":0.005053,"latency":136.359},{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":69.44444444444444,"unit":"percent","family":"gpqa-diamond","index":40.901647607279045,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.7594508998251044,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:tax_eval_v2:overall","benchmarkName":"TaxEval v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":67.376,"unit":"percent","family":"vals-ai:tax_eval_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","measuredAt":"2026-09-01","stderr":0.913,"costPerTest":0.004904,"latency":69.944},{"benchmarkId":"vals-ai:gpqa:overall","benchmarkName":"GPQA Diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":63.384,"unit":"percent","family":"gpqa-diamond","index":35.343173564965184,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","measuredAt":"2026-09-01","stderr":2.455,"costPerTest":0.004578,"latency":49.716},{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":62.331250000000004,"unit":"percent","family":"euroeval","index":56.789399696483315,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":0.5445783919694207,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-french:overall","benchmarkName":"EuroEval French","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":53.87,"unit":"percent","family":"euroeval","index":45.309430324642385,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/french","measuredAt":null,"stderr":0.4791398543223053,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-italian:overall","benchmarkName":"EuroEval Italian","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":53.77374999999999,"unit":"percent","family":"euroeval","index":45.17884122113644,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/italian","measuredAt":null,"stderr":0.45388772014673406,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-dutch:overall","benchmarkName":"EuroEval Dutch","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":53.72555555555555,"unit":"percent","family":"euroeval","index":45.113452449251064,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/dutch","measuredAt":null,"stderr":0.5147851043918265,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:mortgage_tax:overall","benchmarkName":"MortgageTax","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":53.617,"unit":"percent","family":"vals-ai:mortgage_tax:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mortgage_tax","measuredAt":"2026-09-01","stderr":0.969,"costPerTest":0.002295,"latency":33.364},{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":52.76714285714286,"unit":"percent","family":"euroeval","index":43.81310694198427,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.5073198886183697,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:case_law_v2:overall","benchmarkName":"CaseLaw v2","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":52.626,"unit":"percent","family":"vals-ai:case_law_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/case_law_v2","measuredAt":"2026-05-04","stderr":1.022,"costPerTest":0.004165,"latency":56.48},{"benchmarkId":"vals-ai:legal_bench:overall","benchmarkName":"LegalBench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":50.129,"unit":"percent","family":"vals-ai:legal_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","measuredAt":"2026-10-13","stderr":0.472,"costPerTest":null,"latency":22.496},{"benchmarkId":"euroeval:euroeval-portuguese:overall","benchmarkName":"EuroEval Portuguese","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":49.03,"unit":"percent","family":"euroeval","index":38.742663976914805,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/portuguese","measuredAt":null,"stderr":0.6188585763322667,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-spanish:overall","benchmarkName":"EuroEval Spanish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":49.005,"unit":"percent","family":"euroeval","index":38.708744729250924,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/spanish","measuredAt":null,"stderr":0.48342204645630305,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":47.02375,"unit":"percent","family":"euroeval","index":36.02064435188826,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.6465557787229188,"costPerTest":null,"latency":null},{"benchmarkId":"swe-bench:swe-bench-verified:overall","benchmarkName":"SWE-bench Verified","category":"coding","domain":null,"eligible":true,"ranked":true,"score":34.8,"unit":"percent","family":"swe-bench-verified","index":26.894931236344128,"aboveFit":false,"run":"medium effort · mini-SWE-agent","source":"SWE-bench team","sourceUrl":"https://www.swebench.com/","measuredAt":"2026-02-26","stderr":null,"costPerTest":0.03807518078,"latency":null},{"benchmarkId":"swebench-experiments:swe-bench-verified:overall","benchmarkName":"SWE-bench Verified","category":"coding","domain":null,"eligible":true,"ranked":true,"score":34.8,"unit":"percent","family":"swe-bench-verified","index":26.894931236344128,"aboveFit":false,"run":"medium effort · mini-SWE-agent","source":"SWE-bench team","sourceUrl":"https://github.com/SWE-bench/experiments/tree/main/evaluation/verified","measuredAt":"2026-09-01","stderr":null,"costPerTest":0.03807518078,"latency":null},{"benchmarkId":"vals-ai:medcode:overall","benchmarkName":"MedCode","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":30.441,"unit":"percent","family":"vals-ai:medcode:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medcode","measuredAt":"2026-10-29","stderr":1.948,"costPerTest":0.001729,"latency":29.741},{"benchmarkId":"arc-prize:v1-public-eval:overall","benchmarkName":"ARC-AGI-1 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":29.67,"unit":"percent","family":"arc-prize:v1-public-eval:overall","index":null,"aboveFit":false,"run":"high effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.0245,"latency":null},{"benchmarkId":"epoch-ai:epoch-chess-puzzles:overall","benchmarkName":"Chess Puzzles","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":22,"unit":"percent","family":"epoch-ai:epoch-chess-puzzles:overall","index":50.683010018441045,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":4.163331998932265,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tiers-1-3-v2-private:overall","benchmarkName":"FrontierMath-Tiers-1-3-v2-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":20,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tiers-1-3-v2-private:overall","index":41.421730531297136,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.3735633163877066,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-2025-02-28-public:overall","benchmarkName":"FrontierMath-2025-02-28-Public","category":"math","domain":null,"eligible":false,"ranked":true,"score":20,"unit":"percent","family":"epoch-ai:epoch-frontiermath-2025-02-28-public:overall","index":null,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":13.333333333333334,"costPerTest":null,"latency":null},{"benchmarkId":"arc-prize:v1-semi-private:overall","benchmarkName":"ARC-AGI-1 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":16.669999999999998,"unit":"percent","family":"arc-prize:v1-semi-private:overall","index":34.651159696774,"aboveFit":false,"run":"high effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.0292,"latency":null},{"benchmarkId":"epoch-ai:epoch-simpleqa-verified:overall","benchmarkName":"SimpleQA Verified","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":11.700000000000001,"unit":"percent","family":"epoch-ai:epoch-simpleqa-verified:overall","index":33.66618516350279,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.0169287802713343,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-2025-02-28-private:overall","benchmarkName":"FrontierMath-2025-02-28-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":8.275862068965518,"unit":"percent","family":"epoch-ai:epoch-frontiermath-2025-02-28-private:overall","index":37.219919230435444,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.620688385836769,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-mystery-game-puzzles:overall","benchmarkName":"Mystery Game Puzzles","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":8,"unit":"percent","family":"epoch-ai:epoch-mystery-game-puzzles:overall","index":44.876317375711565,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.7265992434429074,"costPerTest":null,"latency":null},{"benchmarkId":"arc-prize:v2-semi-private:overall","benchmarkName":"ARC-AGI-2 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":2.6100000000000003,"unit":"percent","family":"arc-prize:v2-semi-private:overall","index":40.38685697179031,"aboveFit":false,"run":"high effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.0295,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tier-4-v2-private:overall","benchmarkName":"FrontierMath-Tier-4-v2-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":2.4390243902439024,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tier-4-v2-private:overall","index":49.446475323573715,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.4390243902439024,"costPerTest":null,"latency":null},{"benchmarkId":"arc-prize:v2-public-eval:overall","benchmarkName":"ARC-AGI-2 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":0.3,"unit":"percent","family":"arc-prize:v2-public-eval:overall","index":null,"aboveFit":false,"run":"high effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.0321,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-public:overall","benchmarkName":"FrontierMath-Tier-4-2025-07-01-Public","category":"math","domain":null,"eligible":false,"ranked":true,"score":0,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-public:overall","index":null,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-private:overall","benchmarkName":"FrontierMath-Tier-4-2025-07-01-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":0,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-private:overall","index":43.20235772716562,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":0.049999999999999996,"outputPerMillion":0.39999999999999997,"batch":{"inputPerMillion":0.024999999999999998,"outputPerMillion":0.19999999999999998},"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":81,"timeToFirstToken":2.5265,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":{"elo":1319.9942324670371,"eloUpper":1326.8461384644893,"eloLower":1313.142326469585,"votes":8126,"rank":237,"rankUpper":null,"rankLower":null,"arenaName":"GPT-5 nano-high","organization":"openai","updatedAt":"2026-11-09"},"quality":{"coveragePercent":75,"benchmarkCount":19,"categoryCount":6,"confidence":"low","state":"available"},"hosts":[{"host":"databricks","name":"Databricks","kind":"cloud","url":"https://docs.databricks.com/aws/en/machine-learning/foundation-models/","inputPerMillion":0.05,"outputPerMillion":0.4,"cacheReadPerMillion":0.005,"contextTokens":400000,"maxOutputTokens":128000,"deprecated":false},{"host":"digitalocean","name":"DigitalOcean","kind":"cloud","url":"https://docs.digitalocean.com/products/gradient-ai-platform/details/models/","inputPerMillion":0.05,"outputPerMillion":0.4,"cacheReadPerMillion":0.005,"contextTokens":400000,"maxOutputTokens":128000,"deprecated":false},{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":0.05,"outputPerMillion":0.4,"cacheReadPerMillion":0.005,"contextTokens":400000,"maxOutputTokens":128000,"deprecated":false},{"host":"sap-ai-core","name":"SAP AI Core","kind":"cloud","url":"https://help.sap.com/docs/sap-ai-core","inputPerMillion":0.05,"outputPerMillion":0.4,"cacheReadPerMillion":0.005,"contextTokens":400000,"maxOutputTokens":128000,"deprecated":false},{"host":"snowflake","name":"Snowflake Cortex","kind":"cloud","url":"https://docs.snowflake.com/en/user-guide/snowflake-cortex/cortex-rest-api","inputPerMillion":0.06,"outputPerMillion":0.44,"cacheReadPerMillion":0.01,"contextTokens":272000,"maxOutputTokens":8192,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:swe-bench-team","name":"SWE-bench team","url":"https://www.swebench.com","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:arc-prize-foundation","name":"ARC Prize Foundation","url":"https://arcprize.org","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"glm-4-6","providerId":"z-ai","slug":"glm-4-6","name":"GLM-4.6","provider":"Z.AI","description":"GLM-4.6 is a reasoning model from Z.AI. 17 benchmarks count toward its index, in 6 categories.","logoUrl":null,"releaseDate":"2025-09-30","contextWindowTokens":204800,"modalities":["text"],"openWeights":true,"details":{"apiId":"z-ai/glm-4.6","maxOutputTokens":131072,"knowledgeCutoff":"2025-03-31","retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.09999999999999999,"weightsUrl":"https://huggingface.co/zai-org/GLM-4.6"},"reasoning":true,"overallScore":41.789524564152515,"overallScoreRange":{"low":36.36772350731501,"high":47.21132562099002},"badges":[],"capabilityScores":{"agentic":48.752404872822304,"coding":46.673180144602114,"reasoning":31.095541647423232,"multimodal":null,"knowledge":38.991249597910354,"multilingual":null,"instruction":23.320168956307658,"math":44.889827504209876},"capabilityCounts":{"math":3,"knowledge":4,"coding":4,"agentic":3,"instruction":1,"reasoning":2},"benchmarks":[{"benchmarkId":"vals-ai:aime:overall","benchmarkName":"AIME","category":"math","domain":null,"eligible":true,"ranked":true,"score":92.708,"unit":"percent","family":"vals-ai:aime:overall","index":56.02020434857971,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/aime","measuredAt":"2026-04-16","stderr":0.605,"costPerTest":0.116286,"latency":699.868},{"benchmarkId":"vals-ai:medqa:overall","benchmarkName":"MedQA","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":92.225,"unit":"percent","family":"vals-ai:medqa:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medqa","measuredAt":"2026-04-16","stderr":0.244,"costPerTest":0.004252,"latency":28.734},{"benchmarkId":"vals-ai:mgsm:overall","benchmarkName":"MGSM","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":89.746,"unit":"percent","family":"vals-ai:mgsm:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mgsm","measuredAt":"2026-01-09","stderr":1.803,"costPerTest":0.004302,"latency":34.307},{"benchmarkId":"vals-ai:mmlu_pro:overall","benchmarkName":"MMLU Pro","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":82.203,"unit":"percent","family":"mmlu-pro","index":48.046668980776005,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","measuredAt":"2026-09-01","stderr":0.375,"costPerTest":0.006572,"latency":46.996},{"benchmarkId":"vals-ai:lcb:overall","benchmarkName":"LiveCodeBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":81.036,"unit":"percent","family":"livecodebench","index":56.58267673696475,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","measuredAt":"2026-09-01","stderr":1.082,"costPerTest":0.026909,"latency":236.176},{"benchmarkId":"vals-ai:legal_bench:overall","benchmarkName":"LegalBench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":79.608,"unit":"percent","family":"vals-ai:legal_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","measuredAt":"2026-10-13","stderr":0.522,"costPerTest":null,"latency":29.819},{"benchmarkId":"tau2Bench","benchmarkName":"τ²-Bench Tool-Agent-User Evaluation","category":"agentic","score":76.9,"unit":"percent","family":"tau2-bench","index":53.818212452110366,"aboveFit":false,"run":null,"source":"Victor Barres, Honghua Dong, Soham Ray, Xujie Si, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2506.07982","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:gpqa:overall","benchmarkName":"GPQA Diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":74.495,"unit":"percent","family":"gpqa-diamond","index":45.53387915599892,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","measuredAt":"2026-09-01","stderr":2.197,"costPerTest":0.01541,"latency":474.834},{"benchmarkId":"vals-ai:tax_eval_v2:overall","benchmarkName":"TaxEval v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":66.235,"unit":"percent","family":"vals-ai:tax_eval_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","measuredAt":"2026-09-01","stderr":0.808,"costPerTest":0.00928,"latency":64.149},{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":63.2,"unit":"percent","family":"gpqa-diamond","index":35.174413792779944,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"swebench-experiments:swe-bench-verified:overall","benchmarkName":"SWE-bench Verified","category":"coding","domain":null,"eligible":true,"ranked":true,"score":61.8,"unit":"percent","family":"swe-bench-verified","index":46.80898583650381,"aboveFit":false,"run":"mini-SWE-agent","source":"SWE-bench team","sourceUrl":"https://github.com/SWE-bench/experiments/tree/main/evaluation/verified","measuredAt":"2026-09-01","stderr":null,"costPerTest":0.09661007519999999,"latency":null},{"benchmarkId":"vals-ai:corp_fin_v2:overall","benchmarkName":"CorpFin v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":56.838,"unit":"percent","family":"vals-ai:corp_fin_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","measuredAt":"2026-08-12","stderr":0.97,"costPerTest":0.04461,"latency":78.061},{"benchmarkId":"swe-bench:swe-bench-verified:overall","benchmarkName":"SWE-bench Verified","category":"coding","domain":null,"eligible":true,"ranked":true,"score":55.4,"unit":"percent","family":"swe-bench-verified","index":42.08861733868818,"aboveFit":false,"run":"mini-SWE-agent","source":"SWE-bench team","sourceUrl":"https://www.swebench.com/","measuredAt":"2026-02-26","stderr":null,"costPerTest":0.09661007519999999,"latency":null},{"benchmarkId":"vals-ai:terminal-bench:overall","benchmarkName":"Terminal-Bench 1.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":42.5,"unit":"percent","family":"vals-ai:terminal-bench:overall","index":48.97443248047854,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench","measuredAt":"2026-01-12","stderr":5.527,"costPerTest":0.285842,"latency":135.723},{"benchmarkId":"aaIfBench","benchmarkName":"Artificial Analysis IFBench","category":"instruction","score":36.7,"unit":"percent","family":"ifbench-family","index":23.320168956307658,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:finance_agent:overall","benchmarkName":"Finance Agent v1.1","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":36.48,"unit":"percent","family":"vals-ai:finance_agent:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/finance_agent","measuredAt":"2026-06-04","stderr":2.792,"costPerTest":0.187068,"latency":736.029},{"benchmarkId":"vals-ai:terminal-bench-2:overall","benchmarkName":"Terminal-Bench 2.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":28.09,"unit":"percent","family":"vals-ai:terminal-bench-2:overall","index":43.464569685878004,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2","measuredAt":"2026-06-04","stderr":4.791,"costPerTest":0.14355,"latency":576.085},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":26.3,"unit":"percent","family":"lcr","index":25.341967282073565,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"omniscienceAccuracy","benchmarkName":"Artificial Analysis Omniscience Accuracy","category":"knowledge","score":21.4,"unit":"percent","family":"omniscienceAccuracy","index":37.659428525498654,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":5.5,"unit":"percent","family":"hle","index":29.904754410977343,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:ioi-v1:overall","benchmarkName":"IOI v1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":4.334,"unit":"percent","family":"vals-ai:ioi-v1:overall","index":42.296799115345195,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/ioi-v1","measuredAt":"2026-08-09","stderr":2.485,"costPerTest":1.516537,"latency":7105.854},{"benchmarkId":"epoch-ai:epoch-frontiermath-2025-02-28-private:overall","benchmarkName":"FrontierMath-2025-02-28-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":3.8194444444444446,"unit":"percent","family":"epoch-ai:epoch-frontiermath-2025-02-28-private:overall","index":33.28589345771101,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.1313651347924143,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:vibe-code:overall","benchmarkName":"Vibe Code Bench v1.1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":3.09,"unit":"percent","family":"vals-ai:vibe-code:overall","index":43.36444313850252,"aboveFit":false,"run":"OpenHands","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","measuredAt":"2026-11-09","stderr":1.162,"costPerTest":10.845475,"latency":10002.446},{"benchmarkId":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-private:overall","benchmarkName":"FrontierMath-Tier-4-2025-07-01-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":2.127659574468085,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-private:overall","index":45.3633847063389,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.127659574468085,"costPerTest":null,"latency":null},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":0,"unit":"percent","family":"critpt","index":36.8491160127729,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-2025-02-28-public:overall","benchmarkName":"FrontierMath-2025-02-28-Public","category":"math","domain":null,"eligible":false,"ranked":true,"score":0,"unit":"percent","family":"epoch-ai:epoch-frontiermath-2025-02-28-public:overall","index":null,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-public:overall","benchmarkName":"FrontierMath-Tier-4-2025-07-01-Public","category":"math","domain":null,"eligible":false,"ranked":true,"score":0,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-public:overall","index":null,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":0.5,"outputPerMillion":2,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":30,"timeToFirstToken":0.989,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":{"elo":1440.2502905106298,"eloUpper":1444.143025529284,"eloLower":1436.3575554919757,"votes":35652,"rank":76,"rankUpper":null,"rankLower":null,"arenaName":"GLM-4.6","organization":"zai","updatedAt":"2026-11-09"},"quality":{"coveragePercent":85,"benchmarkCount":17,"categoryCount":6,"confidence":"medium","state":"available"},"hosts":[{"host":"deepinfra","name":"Deep Infra","kind":"host","url":"https://deepinfra.com/models","inputPerMillion":0.5,"outputPerMillion":2,"cacheReadPerMillion":0.1,"contextTokens":202752,"maxOutputTokens":131072,"deprecated":false},{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":0.5,"outputPerMillion":2,"cacheReadPerMillion":0.1,"contextTokens":204800,"maxOutputTokens":131072,"deprecated":false},{"host":"io-net","name":"IO.NET","kind":"host","url":"https://io.net/docs/guides/intelligence/io-intelligence","inputPerMillion":0.52,"outputPerMillion":2.0375,"cacheReadPerMillion":0.26,"contextTokens":200000,"maxOutputTokens":4096,"deprecated":false},{"host":"novita-ai","name":"Novita AI","kind":"host","url":"https://novita.ai/docs/guides/introduction","inputPerMillion":0.55,"outputPerMillion":2.2,"cacheReadPerMillion":0.11,"contextTokens":204800,"maxOutputTokens":131072,"deprecated":false},{"host":"z-ai","name":"Z.AI","kind":"maker","url":"https://docs.z.ai/guides/overview/pricing","inputPerMillion":0.6,"outputPerMillion":2.2,"cacheReadPerMillion":0.11,"contextTokens":204800,"maxOutputTokens":131072,"deprecated":false},{"host":"zhipu-ai","name":"Zhipu AI","kind":"maker","url":"https://docs.bigmodel.cn/cn/guide/start/pricing","inputPerMillion":0.6,"outputPerMillion":2.2,"cacheReadPerMillion":0.11,"contextTokens":204800,"maxOutputTokens":131072,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:swe-bench-team","name":"SWE-bench team","url":"https://github.com","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gpt-oss-20b","providerId":"openai","slug":"gpt-oss-20b","name":"GPT-OSS 20B","provider":"OpenAI","description":"GPT-OSS 20B is a standard model from OpenAI in the GPT-OSS family. 20 benchmarks count toward its index, in 7 categories.","logoUrl":null,"releaseDate":"2025-08-05","contextWindowTokens":131072,"modalities":["text"],"openWeights":true,"details":{"apiId":"openai/gpt-oss-20b","maxOutputTokens":117964,"knowledgeCutoff":"2024-06-30","retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.03,"weightsUrl":"https://huggingface.co/openai/gpt-oss-20b"},"reasoning":false,"overallScore":41.71567995406868,"overallScoreRange":{"low":37.099958034451525,"high":46.33140187368584},"badges":[],"capabilityScores":{"agentic":39.77572153254509,"coding":48.91092214293241,"reasoning":35.332198176572234,"multimodal":null,"knowledge":35.96520974051903,"multilingual":37.026350045122356,"instruction":49.86027041746554,"math":45.93946109066302},"capabilityCounts":{"math":5,"knowledge":4,"coding":4,"instruction":1,"agentic":3,"multilingual":1,"reasoning":2},"benchmarks":[{"benchmarkId":"vals-ai:math500:overall","benchmarkName":"MATH 500","category":"math","domain":null,"eligible":true,"ranked":true,"score":94.2,"unit":"percent","family":"math-500","index":43.673406867271986,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/math500","measuredAt":"2026-01-09","stderr":1.045,"costPerTest":0.000955,"latency":31.472},{"benchmarkId":"aime2024","benchmarkName":"American Invitational Mathematics Examination 2024","category":"math","score":92.1,"unit":"percent","family":"aime2024","index":45.65741062409288,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aime2025","benchmarkName":"American Invitational Mathematics Examination 2025","category":"math","score":91.7,"unit":"percent","family":"aime2025","index":48.76600911776524,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:mgsm:overall","benchmarkName":"MGSM","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":89.018,"unit":"percent","family":"vals-ai:mgsm:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mgsm","measuredAt":"2026-01-09","stderr":1.89,"costPerTest":0.000453,"latency":5.791},{"benchmarkId":"vals-ai:aime:overall","benchmarkName":"AIME","category":"math","domain":null,"eligible":true,"ranked":true,"score":86.042,"unit":"percent","family":"vals-ai:aime:overall","index":53.10685730952174,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/aime","measuredAt":"2026-04-16","stderr":1.133,"costPerTest":0.011043,"latency":244.712},{"benchmarkId":"mmlu","benchmarkName":"Massive Multitask Language Understanding","category":"knowledge","score":85.3,"unit":"percent","family":"mmlu","index":33.37984241294858,"aboveFit":false,"run":null,"source":"Dan Hendrycks, Collin Burns, Steven Basart, Andy Zou, Mantas Mazeika, Dawn Song, Jacob Steinhardt","sourceUrl":"https://arxiv.org/abs/2009.03300","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:medqa:overall","benchmarkName":"MedQA","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":82.875,"unit":"percent","family":"vals-ai:medqa:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medqa","measuredAt":"2026-04-16","stderr":0.344,"costPerTest":0.000335,"latency":25.351},{"benchmarkId":"vals-ai:lcb:overall","benchmarkName":"LiveCodeBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":80.387,"unit":"percent","family":"livecodebench","index":56.04417467934197,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","measuredAt":"2026-09-01","stderr":1.078,"costPerTest":0.006706,"latency":109.453},{"benchmarkId":"vals-ai:mmlu_pro:overall","benchmarkName":"MMLU Pro","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":71.636,"unit":"percent","family":"mmlu-pro","index":34.733513818162685,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","measuredAt":"2026-09-01","stderr":0.447,"costPerTest":0.00062,"latency":30.207},{"benchmarkId":"gpqa","benchmarkName":"Graduate-Level Google-Proof Q&A","category":"knowledge","score":71.5,"unit":"percent","family":"gpqa-diamond","index":42.78694699461416,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"reactNativeEvals","benchmarkName":"React Native Evals","category":"coding","score":71,"unit":"percent","family":"reactNativeEvals","index":47.04845735340152,"aboveFit":false,"run":null,"source":"Callstack","sourceUrl":"https://rn-evals.vercel.app/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:legal_bench:overall","benchmarkName":"LegalBench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":70.849,"unit":"percent","family":"vals-ai:legal_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","measuredAt":"2026-10-13","stderr":0.61,"costPerTest":null,"latency":7.187},{"benchmarkId":"vals-ai:gpqa:overall","benchmarkName":"GPQA Diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":68.94,"unit":"percent","family":"gpqa-diamond","index":40.43898494681951,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","measuredAt":"2026-09-01","stderr":2.465,"costPerTest":0.006738,"latency":145.092},{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":68.8,"unit":"percent","family":"gpqa-diamond","index":40.31058077233074,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaIfBench","benchmarkName":"Artificial Analysis IFBench","category":"instruction","score":65.1,"unit":"percent","family":"ifbench-family","index":49.86027041746554,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:tax_eval_v2:overall","benchmarkName":"TaxEval v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":63.696,"unit":"percent","family":"vals-ai:tax_eval_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","measuredAt":"2026-09-01","stderr":0.936,"costPerTest":0.002254,"latency":49.004},{"benchmarkId":"sweVerified","benchmarkName":"Software Engineering Benchmark Verified","category":"coding","score":60.7,"unit":"percent","family":"swe-bench-verified","index":45.997672500941746,"aboveFit":false,"run":null,"source":"Carlos E. Jimenez, John Yang, Alexander Wettig, Shunyu Yao, Kexin Pei, Ofir Press, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2310.06770","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"tau2Bench","benchmarkName":"τ²-Bench Tool-Agent-User Evaluation","category":"agentic","score":60.2,"unit":"percent","family":"tau2-bench","index":42.73654148301955,"aboveFit":false,"run":null,"source":"Victor Barres, Honghua Dong, Soham Ray, Xujie Si, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2506.07982","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":55.527499999999996,"unit":"percent","family":"euroeval","index":47.55827644475777,"aboveFit":false,"run":"medium effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":0.19258115172570756,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:corp_fin_v2:overall","benchmarkName":"CorpFin v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":53.147,"unit":"percent","family":"vals-ai:corp_fin_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","measuredAt":"2026-08-12","stderr":0.979,"costPerTest":0.004549,"latency":33.968},{"benchmarkId":"euroeval:euroeval-french:overall","benchmarkName":"EuroEval French","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":52.86625000000001,"unit":"percent","family":"euroeval","index":43.947572530937535,"aboveFit":false,"run":"medium effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/french","measuredAt":null,"stderr":0.27225045913643564,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":52.0425,"unit":"percent","family":"euroeval","index":42.8299333204126,"aboveFit":false,"run":"low effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":0.3515768052645112,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-french:overall","benchmarkName":"EuroEval French","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":51.6875,"unit":"percent","family":"euroeval","index":42.34828000358547,"aboveFit":false,"run":"low effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/french","measuredAt":null,"stderr":0.29780971273616985,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-italian:overall","benchmarkName":"EuroEval Italian","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":50.997499999999995,"unit":"percent","family":"euroeval","index":41.412108768062325,"aboveFit":false,"run":"medium effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/italian","measuredAt":null,"stderr":0.29775986549567085,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":50.83333333333333,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":38.493621534663276,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":6.711496995394688,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-portuguese:overall","benchmarkName":"EuroEval Portuguese","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":50.60999999999999,"unit":"percent","family":"euroeval","index":40.886360429272145,"aboveFit":false,"run":"medium effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/portuguese","measuredAt":null,"stderr":0.3748145374715341,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":50.17,"unit":"percent","family":"euroeval","index":40.28938167038783,"aboveFit":false,"run":"medium effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.2546225698189862,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-dutch:overall","benchmarkName":"EuroEval Dutch","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":49.864444444444445,"unit":"percent","family":"euroeval","index":39.87481308782927,"aboveFit":false,"run":"low effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/dutch","measuredAt":null,"stderr":0.29458696760291603,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-portuguese:overall","benchmarkName":"EuroEval Portuguese","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":49.69499999999999,"unit":"percent","family":"euroeval","index":39.64491596477406,"aboveFit":false,"run":"low effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/portuguese","measuredAt":null,"stderr":0.34168699126539775,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":48.567142857142855,"unit":"percent","family":"euroeval","index":38.114673334452064,"aboveFit":false,"run":"low effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.30096104568599596,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":48.356249999999996,"unit":"percent","family":"euroeval","index":37.82854025237317,"aboveFit":false,"run":"high effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":0.43461189583351256,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-italian:overall","benchmarkName":"EuroEval Italian","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":47.76499999999999,"unit":"percent","family":"euroeval","index":37.026350045122356,"aboveFit":false,"run":"low effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/italian","measuredAt":null,"stderr":0.33922475956215226,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":45.95959595959596,"unit":"percent","family":"gpqa-diamond","index":19.3619863023447,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":3.5507024651313412,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-spanish:overall","benchmarkName":"EuroEval Spanish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":45.80250000000001,"unit":"percent","family":"euroeval","index":34.363689103507625,"aboveFit":false,"run":"medium effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/spanish","measuredAt":null,"stderr":0.331982680271125,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":45.75125,"unit":"percent","family":"euroeval","index":34.29415464579665,"aboveFit":false,"run":"medium effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.4270062938646221,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-french:overall","benchmarkName":"EuroEval French","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":44.795,"unit":"percent","family":"euroeval","index":32.99674342265317,"aboveFit":false,"run":"high effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/french","measuredAt":null,"stderr":0.6103393011268404,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-spanish:overall","benchmarkName":"EuroEval Spanish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":44.1975,"unit":"percent","family":"euroeval","index":32.186073403486375,"aboveFit":false,"run":"low effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/spanish","measuredAt":null,"stderr":0.400764893672088,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:case_law_v2:overall","benchmarkName":"CaseLaw v2","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":43.837,"unit":"percent","family":"vals-ai:case_law_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/case_law_v2","measuredAt":"2026-05-04","stderr":0.935,"costPerTest":0.002158,"latency":85.696},{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":43.49,"unit":"percent","family":"euroeval","index":31.226158694598517,"aboveFit":false,"run":"high effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.33340849492746266,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-italian:overall","benchmarkName":"EuroEval Italian","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":43.28125,"unit":"percent","family":"euroeval","index":30.9429329766051,"aboveFit":false,"run":"high effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/italian","measuredAt":null,"stderr":0.45061416422478334,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":42.46625,"unit":"percent","family":"euroeval","index":29.837165502762545,"aboveFit":false,"run":"low effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.6144891170720601,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-portuguese:overall","benchmarkName":"EuroEval Portuguese","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":40.70625,"unit":"percent","family":"euroeval","index":27.449250467225234,"aboveFit":false,"run":"high effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/portuguese","measuredAt":null,"stderr":0.44022189007363094,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-dutch:overall","benchmarkName":"EuroEval Dutch","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":40.546666666666674,"unit":"percent","family":"euroeval","index":27.2327326029708,"aboveFit":false,"run":"high effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/dutch","measuredAt":null,"stderr":0.6448198140508654,"costPerTest":null,"latency":null},{"benchmarkId":"aaSciCode","benchmarkName":"Artificial Analysis SciCode","category":"coding","score":38.9,"unit":"percent","family":"scicode-family","index":46.553384038044385,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/scicode","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":38.29875,"unit":"percent","family":"euroeval","index":24.18282691719338,"aboveFit":false,"run":"high effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.5638899493695556,"costPerTest":null,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":34.7,"unit":"percent","family":"lcr","index":31.105043696993118,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-spanish:overall","benchmarkName":"EuroEval Spanish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":32.94125,"unit":"percent","family":"euroeval","index":16.913932142823427,"aboveFit":false,"run":"high effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/spanish","measuredAt":null,"stderr":0.6391559375457604,"costPerTest":null,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":11,"unit":"percent","family":"hle","index":35.42286450864121,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hle","benchmarkName":"Humanity's Last Exam","category":"knowledge","score":10.9,"unit":"percent","family":"hle","index":35.322535234138236,"aboveFit":false,"run":null,"source":"Center for AI Safety, Scale AI, and thousands of expert contributors","sourceUrl":"https://lastexam.ai/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":1.4,"unit":"percent","family":"critpt","index":39.55935265615135,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"apexAgentsAa","benchmarkName":"APEX-Agents-AA","category":"agentic","score":0.7,"unit":"percent","family":"apexAgentsAa","index":42.55684819742294,"aboveFit":false,"run":null,"source":"Artificial Analysis / Mercor","sourceUrl":"https://artificialanalysis.ai/evaluations/apex-agents-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gdpvalAaNormalized","benchmarkName":"GDPval-AA normalized","category":"agentic","score":0,"unit":"percent","family":"gdpvalAaNormalized","index":34.03377491719279,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":0.03,"outputPerMillion":0.13,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":310,"timeToFirstToken":0.403,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":{"elo":1287.5999005724748,"eloUpper":1293.9940981551777,"eloLower":1281.205702989772,"votes":10400,"rank":264,"rankUpper":null,"rankLower":null,"arenaName":"gpt-oss-20b","organization":"openai","updatedAt":"2026-11-09"},"quality":{"coveragePercent":90,"benchmarkCount":20,"categoryCount":7,"confidence":"medium","state":"available"},"hosts":[{"host":"coreweave","name":"CoreWeave","kind":"host","url":"https://docs.wandb.ai/inference","inputPerMillion":0.03,"outputPerMillion":0.13,"cacheReadPerMillion":0.03,"contextTokens":131072,"maxOutputTokens":131072,"deprecated":false},{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":0.03,"outputPerMillion":0.13,"cacheReadPerMillion":0.03,"contextTokens":131072,"maxOutputTokens":117964,"deprecated":false},{"host":"deepinfra","name":"Deep Infra","kind":"host","url":"https://deepinfra.com/models","inputPerMillion":0.03,"outputPerMillion":0.14,"cacheReadPerMillion":null,"contextTokens":131072,"maxOutputTokens":16384,"deprecated":false},{"host":"siliconflow","name":"SiliconFlow","kind":"host","url":"https://cloud.siliconflow.com/models","inputPerMillion":0.04,"outputPerMillion":0.18,"cacheReadPerMillion":null,"contextTokens":131000,"maxOutputTokens":8000,"deprecated":false},{"host":"ovhcloud","name":"OVHcloud","kind":"cloud","url":"https://www.ovhcloud.com/en/public-cloud/ai-endpoints/catalog//","inputPerMillion":0.05,"outputPerMillion":0.18,"cacheReadPerMillion":null,"contextTokens":131072,"maxOutputTokens":131072,"deprecated":false},{"host":"io-net","name":"IO.NET","kind":"host","url":"https://io.net/docs/guides/intelligence/io-intelligence","inputPerMillion":0.055,"outputPerMillion":0.194,"cacheReadPerMillion":0.0275,"contextTokens":64000,"maxOutputTokens":4096,"deprecated":false},{"host":"stackit","name":"STACKIT","kind":"cloud","url":"https://docs.stackit.cloud/products/data-and-ai/ai-model-serving/basics/available-shared-models","inputPerMillion":0.18,"outputPerMillion":0.29,"cacheReadPerMillion":null,"contextTokens":131072,"maxOutputTokens":8192,"deprecated":false},{"host":"amazon-bedrock","name":"Amazon Bedrock","kind":"cloud","url":"https://docs.aws.amazon.com/bedrock/latest/userguide/models-supported.html","inputPerMillion":0.07,"outputPerMillion":0.3,"cacheReadPerMillion":null,"contextTokens":131072,"maxOutputTokens":131072,"deprecated":false},{"host":"databricks","name":"Databricks","kind":"cloud","url":"https://docs.databricks.com/aws/en/machine-learning/foundation-models/","inputPerMillion":0.07,"outputPerMillion":0.3,"cacheReadPerMillion":null,"contextTokens":131072,"maxOutputTokens":25000,"deprecated":false},{"host":"oracle-cloud","name":"Oracle Cloud","kind":"cloud","url":"https://docs.oracle.com/en-us/iaas/Content/generative-ai/pretrained-models.htm","inputPerMillion":0.07,"outputPerMillion":0.3,"cacheReadPerMillion":null,"contextTokens":128000,"maxOutputTokens":16384,"deprecated":false},{"host":"groq","name":"Groq","kind":"host","url":"https://console.groq.com/docs/models","inputPerMillion":0.075,"outputPerMillion":0.3,"cacheReadPerMillion":0.0375,"contextTokens":131072,"maxOutputTokens":65536,"deprecated":false},{"host":"cloudflare","name":"Cloudflare Workers AI","kind":"cloud","url":"https://developers.cloudflare.com/workers-ai/models/","inputPerMillion":0.2,"outputPerMillion":0.3,"cacheReadPerMillion":null,"contextTokens":128000,"maxOutputTokens":16384,"deprecated":false},{"host":"digitalocean","name":"DigitalOcean","kind":"cloud","url":"https://docs.digitalocean.com/products/gradient-ai-platform/details/models/","inputPerMillion":0.05,"outputPerMillion":0.45,"cacheReadPerMillion":0.01,"contextTokens":128000,"maxOutputTokens":4096,"deprecated":false},{"host":"regolo","name":"Regolo","kind":"host","url":"https://docs.regolo.ai/","inputPerMillion":0.11215,"outputPerMillion":0.47103,"cacheReadPerMillion":null,"contextTokens":128000,"maxOutputTokens":16384,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"granite-4-2-30b","providerId":"ibm","slug":"granite-4-2-30b","name":"Granite 4.2 30B","provider":"IBM","description":"Granite 4.2 30B is a reasoning model from IBM in the Granite 4.2 family. 17 benchmarks count toward its index, in 6 categories.","logoUrl":null,"releaseDate":"2026-08-25","contextWindowTokens":128000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":true,"overallScore":41.62665107491319,"overallScoreRange":{"low":36.20485001807569,"high":47.04845213175069},"badges":[],"capabilityScores":{"agentic":41.96753217151226,"coding":43.20796402002801,"reasoning":39.17293811032531,"multimodal":null,"knowledge":34.96814000810744,"multilingual":null,"instruction":61.13981353845765,"math":47.50366516942194},"capabilityCounts":{"math":2,"knowledge":4,"instruction":1,"coding":4,"agentic":4,"reasoning":2},"benchmarks":[{"benchmarkId":"aime2025","benchmarkName":"American Invitational Mathematics Examination 2025","category":"math","score":89.17,"unit":"percent","family":"aime2025","index":47.39316580242644,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hmmtFeb2025","benchmarkName":"Harvard-MIT Mathematics Tournament February 2025","category":"math","score":89.17,"unit":"percent","family":"hmmtFeb2025","index":47.61416453641745,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmluPro","benchmarkName":"Massive Multitask Language Understanding Professional","category":"knowledge","score":77.6,"unit":"percent","family":"mmlu-pro","index":42.24743994571315,"aboveFit":false,"run":null,"source":"Yubo Wang, Xueguang Ma, Ge Zhang, Yuansheng Ni, Abhranil Chandra, Shiguang Guo, Weiming Ren, Aaran Arulraj, Xuan He, Ziyan Jiang, Tianle Li, Max Ku, Kai Wang, Alex Zhuang, Rongqi Fan, Xiang Yue, Wenhu Chen","sourceUrl":"https://arxiv.org/abs/2406.01574","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"ifBench","benchmarkName":"Instruction Following Benchmark","category":"instruction","score":77.17,"unit":"percent","family":"ifbench-family","index":61.13981353845765,"aboveFit":true,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"liveCodeBenchV6","benchmarkName":"LiveCodeBench v6","category":"coding","score":75.77,"unit":"percent","family":"liveCodeBenchV6","index":46.628571142041196,"aboveFit":false,"run":null,"source":"LiveCodeBench maintainers","sourceUrl":"https://github.com/LiveCodeBench/LiveCodeBench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaOpennessIndex","benchmarkName":"Artificial Analysis Openness Index","category":"knowledge","score":72.2,"unit":"percent","family":"aaOpennessIndex","index":null,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-openness-index","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gpqa","benchmarkName":"Graduate-Level Google-Proof Q&A","category":"knowledge","score":66.41,"unit":"percent","family":"gpqa-diamond","index":38.118538079272454,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":64.4,"unit":"percent","family":"gpqa-diamond","index":36.275021002683694,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"tau3Bench","benchmarkName":"τ³-Bench Tool-Agent-User Evaluation","category":"agentic","score":62,"unit":"percent","family":"tau3Bench","index":50.791366808819355,"aboveFit":false,"run":null,"source":"Sierra Research","sourceUrl":"https://github.com/sierra-research/tau2-bench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"bfclV4","benchmarkName":"Berkeley Function Calling Leaderboard v4","category":"agentic","score":61.39,"unit":"percent","family":"bfclV4","index":42.15728087723223,"aboveFit":false,"run":null,"source":"Shishir G. Patil et al.","sourceUrl":"https://gorilla.cs.berkeley.edu/leaderboard.html","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"sweVerified","benchmarkName":"Software Engineering Benchmark Verified","category":"coding","score":57,"unit":"percent","family":"swe-bench-verified","index":43.26870946314209,"aboveFit":false,"run":null,"source":"Carlos E. Jimenez, John Yang, Alexander Wettig, Shunyu Yao, Kexin Pei, Ofir Press, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2310.06770","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":49,"unit":"percent","family":"lcr","index":40.915995212868054,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"sciCode","benchmarkName":"Scientific Code Benchmark","category":"coding","score":38.76,"unit":"percent","family":"scicode-family","index":46.370867277546324,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaSciCode","benchmarkName":"Artificial Analysis SciCode","category":"coding","score":37.8,"unit":"percent","family":"scicode-family","index":45.11932377698818,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/scicode","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"swePro","benchmarkName":"SWE-bench Pro","category":"coding","score":33.29,"unit":"percent","family":"swePro","index":37.18947994766148,"aboveFit":false,"run":null,"source":"Xiang Deng, Jeff Da, Edwin Pan, Yannis Yiming He, Charles Ide, Kanak Garg, Niklas Lauffer, Andrew Park, Nitin Pasari, Chetan Rane, Karmini Sampath, Maya Krishnan, Srivatsa Kundurthy, Sean Hendryx, Zifan Wang, Chen Bo Calvin Zhang, Noah Jacobson, Bing Liu, Brad Kenstler","sourceUrl":"https://arxiv.org/abs/2509.16941","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"terminalBench21","benchmarkName":"Terminal-Bench 2.1","category":"agentic","score":29.24,"unit":"percent","family":"terminal-bench-2-1","index":38.24229206454489,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":11.2,"unit":"percent","family":"hle","index":35.623523057647176,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"omniscienceAccuracy","benchmarkName":"Artificial Analysis Omniscience Accuracy","category":"knowledge","score":11.2,"unit":"percent","family":"omniscienceAccuracy","index":24.804817488091388,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gdpvalAaNormalized","benchmarkName":"GDPval-AA normalized","category":"agentic","score":3.7,"unit":"percent","family":"gdpvalAaNormalized","index":36.679188935452565,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":0.3,"unit":"percent","family":"critpt","index":37.42988100778257,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":{"elo":1361.1838309729992,"eloUpper":1371.7460197275764,"eloLower":1350.621642218422,"votes":3314,"rank":201,"rankUpper":null,"rankLower":null,"arenaName":"Granite 4.2 30B","organization":"ibm","updatedAt":"2026-11-09"},"quality":{"coveragePercent":85,"benchmarkCount":17,"categoryCount":6,"confidence":"medium","state":"available"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"lfm3-1-7b","providerId":"liquidai","slug":"lfm3-1-7b","name":"LFM3-1.7B","provider":"LiquidAI","description":"LFM3-1.7B is a reasoning model from LiquidAI in the LFM3 family. 16 benchmarks count toward its index, in 5 categories.","logoUrl":null,"releaseDate":"2026-10-27","contextWindowTokens":262144,"modalities":["text"],"openWeights":true,"details":{"apiId":"liquid/lfm-3-1.7b","maxOutputTokens":8192,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.02,"weightsUrl":"https://huggingface.co/LiquidAI/LFM3-1.7B"},"reasoning":true,"overallScore":41.4626143181411,"overallScoreRange":{"low":34.9201143181411,"high":48.005114318141096},"badges":[],"capabilityScores":{"agentic":46.44798177503828,"coding":36.97524697285655,"reasoning":43.543683625861135,"multimodal":null,"knowledge":28.836848905539636,"multilingual":null,"instruction":57.55914223961932,"math":null},"capabilityCounts":{"instruction":2,"coding":3,"agentic":7,"knowledge":3,"reasoning":1},"benchmarks":[{"benchmarkId":"ifeval","benchmarkName":"Instruction-Following Eval","category":"instruction","score":96.66,"unit":"percent","family":"ifeval","index":57.84734488617514,"aboveFit":false,"run":null,"source":"Jeffrey Zhou, Tianjian Lu, Swaroop Mishra, Siddhartha Brahma, Sujoy Basu, Yi Luan, Denny Zhou, Le Hou","sourceUrl":"https://arxiv.org/abs/2311.07911","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"ifStruct","benchmarkName":"IFStruct","category":"instruction","score":93.72,"unit":"percent","family":"ifStruct","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"multiIf","benchmarkName":"Multi-IF","category":"instruction","score":89.1,"unit":"percent","family":"multiIf","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"toolSandbox","benchmarkName":"ToolSandbox","category":"agentic","score":80.36,"unit":"percent","family":"toolSandbox","index":null,"aboveFit":false,"run":null,"source":"Apple","sourceUrl":"https://github.com/apple-aiml-research/ToolSandbox","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"liveCodeBenchV6","benchmarkName":"LiveCodeBench v6","category":"coding","score":73.46,"unit":"percent","family":"liveCodeBenchV6","index":44.731929132347055,"aboveFit":false,"run":null,"source":"LiveCodeBench maintainers","sourceUrl":"https://github.com/LiveCodeBench/LiveCodeBench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"ifBench","benchmarkName":"Instruction Following Benchmark","category":"instruction","score":73.03,"unit":"percent","family":"ifbench-family","index":57.27093959306351,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"bfclV4","benchmarkName":"Berkeley Function Calling Leaderboard v4","category":"agentic","score":71.39,"unit":"percent","family":"bfclV4","index":51.44884099658853,"aboveFit":false,"run":null,"source":"Shishir G. Patil et al.","sourceUrl":"https://gorilla.cs.berkeley.edu/leaderboard.html","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"clawEval","benchmarkName":"Claw-Eval","category":"agentic","score":68.64,"unit":"percent","family":"clawEval","index":64.38126481568959,"aboveFit":true,"run":null,"source":"Bowen Ye, Rang Li, Qibin Yang, Yuanxin Liu, Linli Yao, Hanglong Lv, Zhihui Xie, Chenxin An, Lei Li, Lingpeng Kong, Qi Liu, Zhifang Sui, Tong Yang","sourceUrl":"https://arxiv.org/abs/2604.06132","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"tau3Bench","benchmarkName":"τ³-Bench Tool-Agent-User Evaluation","category":"agentic","score":68.01,"unit":"percent","family":"tau3Bench","index":57.70455924905784,"aboveFit":false,"run":null,"source":"Sierra Research","sourceUrl":"https://github.com/sierra-research/tau2-bench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"ruler128k","benchmarkName":"RULER at 128K context","category":"reasoning","score":65.56,"unit":"percent","family":"ruler128k","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gpqa","benchmarkName":"Graduate-Level Google-Proof Q&A","category":"knowledge","score":63.37,"unit":"percent","family":"gpqa-diamond","index":35.330333147516306,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gpqaDiamond","benchmarkName":"GPQA Diamond","category":"knowledge","score":63.37,"unit":"percent","family":"gpqa-diamond","index":35.330333147516306,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":52.83,"unit":"percent","family":"lcr","index":43.543683625861135,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"liveClaw","benchmarkName":"LiveClaw","category":"agentic","score":45.6,"unit":"percent","family":"liveClaw","index":null,"aboveFit":false,"run":null,"source":"Liquid AI (reporting)","sourceUrl":"https://www.liquid.ai/blog/lfm3","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"sweVerified","benchmarkName":"Software Engineering Benchmark Verified","category":"coding","score":43.4,"unit":"percent","family":"swe-bench-verified","index":33.23792640528387,"aboveFit":false,"run":null,"source":"Carlos E. Jimenez, John Yang, Alexander Wettig, Shunyu Yao, Kexin Pei, Ofir Press, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2310.06770","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"automationBench","benchmarkName":"AutomationBench","category":"agentic","score":41.29,"unit":"percent","family":"automationBench","index":60.627872971650746,"aboveFit":true,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"webArenaVerified","benchmarkName":"WebArena-Verified Browser Agent Benchmark","category":"agentic","score":38.42,"unit":"percent","family":"webArenaVerified","index":null,"aboveFit":false,"run":null,"source":"Amine El Hattami, Megh Thakkar, Nicolas Chapados, Christopher Pal","sourceUrl":"https://openreview.net/forum?id=94tlGxmqkN","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mcpAtlas","benchmarkName":"MCP Atlas","category":"agentic","score":32.57,"unit":"percent","family":"mcpAtlas","index":32.99477787566692,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"liveCodeBenchPro","benchmarkName":"LiveCodeBench Pro","category":"coding","score":31.74,"unit":"percent","family":"liveCodeBenchPro","index":null,"aboveFit":false,"run":null,"source":"LiveCodeBench Pro authors","sourceUrl":"https://arxiv.org/abs/2506.11928","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"sciCode","benchmarkName":"Scientific Code Benchmark","category":"coding","score":28.47,"unit":"percent","family":"scicode-family","index":32.955885380938724,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"enterpriseOpsGym","benchmarkName":"EnterpriseOps-Gym (provider runs)","category":"agentic","score":28.2,"unit":"percent","family":"enterpriseOpsGym","index":null,"aboveFit":false,"run":null,"source":"ServiceNow","sourceUrl":"https://arxiv.org/abs/2603.13594","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hleWithTools","benchmarkName":"Humanity's Last Exam with tools","category":"agentic","score":13.9,"unit":"percent","family":"hleWithTools","index":29.28110400026842,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"terminalBench21","benchmarkName":"Terminal-Bench 2.1","category":"agentic","score":12.92,"unit":"percent","family":"terminal-bench-2-1","index":28.6974525163459,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hle","benchmarkName":"Humanity's Last Exam","category":"knowledge","score":7.17,"unit":"percent","family":"hle","index":31.5802532951771,"aboveFit":false,"run":null,"source":"Center for AI Safety, Scale AI, and thousands of expert contributors","sourceUrl":"https://lastexam.ai/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"omniscienceAccuracy","benchmarkName":"Artificial Analysis Omniscience Accuracy","category":"knowledge","score":7.07,"unit":"percent","family":"omniscienceAccuracy","index":19.5999602739255,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":0.08,"outputPerMillion":0.19999999999999998,"batch":null,"freeTier":true,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":218,"timeToFirstToken":0.2955,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":75,"benchmarkCount":16,"categoryCount":5,"confidence":"low","state":"available"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"claude-3-7-sonnet-thinking","providerId":"anthropic","slug":"claude-3-7-sonnet-thinking","name":"Claude 3.7 Sonnet Thinking","provider":"Anthropic","description":"Claude 3.7 Sonnet Thinking is a reasoning model from Anthropic in the Claude 3.7 Sonnet family. 15 benchmarks count toward its index, in 7 categories.","logoUrl":null,"releaseDate":"2025-02-24","contextWindowTokens":200000,"modalities":["text"],"openWeights":false,"details":null,"reasoning":true,"overallScore":41.41996666259482,"overallScoreRange":{"low":36.74489061538322,"high":46.095042709806414},"badges":[],"capabilityScores":{"agentic":39.086889367450716,"coding":39.490007419508316,"reasoning":38.5914109978019,"multimodal":48.05692092304999,"knowledge":44.05386756407974,"multilingual":null,"instruction":42.519873421308496,"math":36.56712448223435},"capabilityCounts":{"math":3,"instruction":2,"knowledge":5,"multimodal":2,"coding":1,"agentic":1,"reasoning":1},"benchmarks":[{"benchmarkId":"math500","benchmarkName":"MATH-500 Problem Set","category":"math","score":96.2,"unit":"percent","family":"math-500","index":45.786923292131746,"aboveFit":false,"run":null,"source":"Dan Hendrycks, Collin Burns, Saurav Kadavath, Akul Arora, Steven Basart, Eric Tang, Dawn Song, Jacob Steinhardt","sourceUrl":"https://arxiv.org/abs/2103.03874","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"ifeval","benchmarkName":"Instruction-Following Eval","category":"instruction","score":93.2,"unit":"percent","family":"ifeval","index":50.8792547542871,"aboveFit":false,"run":null,"source":"Jeffrey Zhou, Tianjian Lu, Swaroop Mishra, Siddhartha Brahma, Sujoy Basu, Yi Luan, Denny Zhou, Le Hou","sourceUrl":"https://arxiv.org/abs/2311.07911","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:mgsm:overall","benchmarkName":"MGSM","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":92.982,"unit":"percent","family":"vals-ai:mgsm:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mgsm","measuredAt":"2026-01-09","stderr":1.568,"costPerTest":0.025944,"latency":23.512},{"benchmarkId":"vals-ai:math500:overall","benchmarkName":"MATH 500","category":"math","domain":null,"eligible":true,"ranked":true,"score":91.6,"unit":"percent","family":"math-500","index":40.92583551495429,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/math500","measuredAt":"2026-01-09","stderr":1.24,"costPerTest":0.125312,"latency":94.242},{"benchmarkId":"vals-ai:medqa:overall","benchmarkName":"MedQA","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":90.217,"unit":"percent","family":"vals-ai:medqa:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medqa","measuredAt":"2026-04-16","stderr":0.532,"costPerTest":0.016693,"latency":15.739},{"benchmarkId":"mmmlu","benchmarkName":"MMMLU","category":"knowledge","score":86.1,"unit":"percent","family":"mmmlu","index":43.45069847845633,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:mmlu_pro:overall","benchmarkName":"MMLU Pro","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":82.728,"unit":"percent","family":"mmlu-pro","index":48.70810613989135,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","measuredAt":"2026-09-01","stderr":0.37,"costPerTest":0.040182,"latency":31.744},{"benchmarkId":"vals-ai:legal_bench:overall","benchmarkName":"LegalBench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":82.519,"unit":"percent","family":"vals-ai:legal_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","measuredAt":"2026-10-13","stderr":0.53,"costPerTest":null,"latency":6.536},{"benchmarkId":"gpqa","benchmarkName":"Graduate-Level Google-Proof Q&A","category":"knowledge","score":78.2,"unit":"percent","family":"gpqa-diamond","index":48.932003916576726,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":77.2,"unit":"percent","family":"gpqa-diamond","index":48.014831241656935,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:gpqa:overall","benchmarkName":"GPQA Diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":75.253,"unit":"percent","family":"gpqa-diamond","index":46.229096043588115,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","measuredAt":"2026-09-01","stderr":2.169,"costPerTest":0.18875,"latency":155.02},{"benchmarkId":"vals-ai:mmmu:overall","benchmarkName":"MMMU Pro","category":"multimodal","domain":null,"eligible":true,"ranked":true,"score":75.101,"unit":"percent","family":"mmmu-pro","index":52.8901775505501,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmmu","measuredAt":"2026-09-01","stderr":1.039,"costPerTest":0.051028,"latency":69.581},{"benchmarkId":"mmmu","benchmarkName":"Massive Multi-discipline Multimodal Understanding","category":"multimodal","score":75,"unit":"percent","family":"mmmu","index":43.223664295549874,"aboveFit":false,"run":null,"source":"MMMU authors","sourceUrl":"https://arxiv.org/abs/2401.05508","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:tax_eval_v2:overall","benchmarkName":"TaxEval v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":74.039,"unit":"percent","family":"vals-ai:tax_eval_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","measuredAt":"2026-09-01","stderr":0.859,"costPerTest":0.050764,"latency":43.314},{"benchmarkId":"vals-ai:mortgage_tax:overall","benchmarkName":"MortgageTax","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":66.852,"unit":"percent","family":"vals-ai:mortgage_tax:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mortgage_tax","measuredAt":"2026-09-01","stderr":1.804,"costPerTest":0.02365,"latency":13.585},{"benchmarkId":"aime2024","benchmarkName":"American Invitational Mathematics Examination 2024","category":"math","score":61.3,"unit":"percent","family":"aime2024","index":31.357616915502124,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:lcb:overall","benchmarkName":"LiveCodeBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":60.436,"unit":"percent","family":"livecodebench","index":39.490007419508316,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","measuredAt":"2026-09-01","stderr":1.138,"costPerTest":0.124735,"latency":96.471},{"benchmarkId":"vals-ai:corp_fin_v2:overall","benchmarkName":"CorpFin v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":60.412,"unit":"percent","family":"vals-ai:corp_fin_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","measuredAt":"2026-08-12","stderr":0.964,"costPerTest":0.232197,"latency":22.163},{"benchmarkId":"tau2Bench","benchmarkName":"τ²-Bench Tool-Agent-User Evaluation","category":"agentic","score":54.7,"unit":"percent","family":"tau2-bench","index":39.086889367450716,"aboveFit":false,"run":null,"source":"Victor Barres, Honghua Dong, Soham Ray, Xujie Si, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2506.07982","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaIfBench","benchmarkName":"Artificial Analysis IFBench","category":"instruction","score":48.3,"unit":"percent","family":"ifbench-family","index":34.16049208832989,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:aime:overall","benchmarkName":"AIME","category":"math","domain":null,"eligible":true,"ranked":true,"score":44.583,"unit":"percent","family":"vals-ai:aime:overall","index":34.9873771276579,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/aime","measuredAt":"2026-04-16","stderr":1.747,"costPerTest":0.395347,"latency":303.709},{"benchmarkId":"omniscienceAccuracy","benchmarkName":"Artificial Analysis Omniscience Accuracy","category":"knowledge","score":28,"unit":"percent","family":"omniscienceAccuracy","index":45.9771180202916,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":9.7,"unit":"percent","family":"hle","index":34.11858394010248,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":0.9,"unit":"percent","family":"critpt","index":38.5914109978019,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":95,"benchmarkCount":15,"categoryCount":7,"confidence":"medium","state":"available"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"grok-code-fast-1","providerId":"xai","slug":"grok-code-fast-1","name":"Grok Code Fast 1","provider":"xAI","description":"Grok Code Fast 1 is a standard model from xAI. 11 benchmarks count toward its index, in 5 categories.","logoUrl":null,"releaseDate":"2025-08-28","contextWindowTokens":256000,"modalities":["text"],"openWeights":false,"details":null,"reasoning":false,"overallScore":41.15227830786511,"overallScoreRange":{"low":33.77683986604267,"high":48.52771674968755},"badges":[],"capabilityScores":{"agentic":42.85674192118763,"coding":45.501768772785674,"reasoning":40.2547166639918,"multimodal":null,"knowledge":38.868835523364474,"multilingual":null,"instruction":27.712368846006317,"math":null},"capabilityCounts":{"agentic":2,"knowledge":3,"coding":3,"reasoning":2,"instruction":1},"benchmarks":[{"benchmarkId":"tau2Bench","benchmarkName":"τ²-Bench Tool-Agent-User Evaluation","category":"agentic","score":75.7,"unit":"percent","family":"tau2-bench","index":53.02192471780444,"aboveFit":false,"run":null,"source":"Victor Barres, Honghua Dong, Soham Ray, Xujie Si, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2506.07982","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":72.7,"unit":"percent","family":"gpqa-diamond","index":43.887554204517905,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"sweVerified","benchmarkName":"Software Engineering Benchmark Verified","category":"coding","score":70.8,"unit":"percent","family":"swe-bench-verified","index":53.44700403655703,"aboveFit":false,"run":null,"source":"Carlos E. Jimenez, John Yang, Alexander Wettig, Shunyu Yao, Kexin Pei, Ofir Press, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2310.06770","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:lcb:overall","benchmarkName":"LiveCodeBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":61.969,"unit":"percent","family":"livecodebench","index":40.76200072356952,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","measuredAt":"2026-09-01","stderr":1.203,"costPerTest":0.008528,"latency":25.838},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":53,"unit":"percent","family":"lcr","index":43.6603173152107,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaIfBench","benchmarkName":"Artificial Analysis IFBench","category":"instruction","score":41.4,"unit":"percent","family":"ifbench-family","index":27.712368846006317,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"omniscienceAccuracy","benchmarkName":"Artificial Analysis Omniscience Accuracy","category":"knowledge","score":23.5,"unit":"percent","family":"omniscienceAccuracy","index":40.30596609202369,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:terminal-bench:overall","benchmarkName":"Terminal-Bench 1.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":20,"unit":"percent","family":"vals-ai:terminal-bench:overall","index":32.69155912457081,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench","measuredAt":"2026-01-12","stderr":4.472,"costPerTest":0.237938,"latency":85.132},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":8,"unit":"percent","family":"hle","index":32.41298627355183,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:ioi-v1:overall","benchmarkName":"IOI v1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":4.333,"unit":"percent","family":"vals-ai:ioi-v1:overall","index":42.29630155823047,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/ioi-v1","measuredAt":"2026-08-09","stderr":1.157,"costPerTest":1.61801,"latency":1083.862},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":0,"unit":"percent","family":"critpt","index":36.8491160127729,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":0.2,"outputPerMillion":1.5,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":172,"timeToFirstToken":2.81,"source":"BenchLM","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":75,"benchmarkCount":11,"categoryCount":5,"confidence":"low","state":"available"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"command-a-plus","providerId":"cohere","slug":"command-a-plus","name":"Command A+","provider":"Cohere","description":"Command A+ is a reasoning model from Cohere in the Command A family. 13 benchmarks count toward its index, in 6 categories.","logoUrl":null,"releaseDate":"2026-05-20","contextWindowTokens":192000,"modalities":["text","image"],"openWeights":true,"details":{"apiId":"cohere/command-a-plus","maxOutputTokens":64000,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.15,"weightsUrl":"https://huggingface.co/CohereLabs/command-a-plus-05-2026-bf16"},"reasoning":true,"overallScore":40.24425156984971,"overallScoreRange":{"low":34.25950048336873,"high":46.229002656330685},"badges":[],"capabilityScores":{"agentic":41.72124783240844,"coding":46.0319075794785,"reasoning":40.44218708265878,"multimodal":34.92916282253943,"knowledge":35.4908525544414,"multilingual":null,"instruction":58.083963827965185,"math":null},"capabilityCounts":{"agentic":3,"knowledge":3,"multimodal":3,"instruction":1,"reasoning":2,"coding":1},"benchmarks":[{"benchmarkId":"tau2Bench","benchmarkName":"τ²-Bench Tool-Agent-User Evaluation","category":"agentic","score":85,"unit":"percent","family":"tau2-bench","index":59.19315465867537,"aboveFit":true,"run":null,"source":"Victor Barres, Honghua Dong, Soham Ray, Xujie Si, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2506.07982","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":76.1,"unit":"percent","family":"gpqa-diamond","index":47.005941299245166,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmmu","benchmarkName":"Massive Multi-discipline Multimodal Understanding","category":"multimodal","score":75.1,"unit":"percent","family":"mmmu","index":43.33249145200493,"aboveFit":false,"run":null,"source":"MMMU authors","sourceUrl":"https://arxiv.org/abs/2401.05508","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaIfBench","benchmarkName":"Artificial Analysis IFBench","category":"instruction","score":73.9,"unit":"percent","family":"ifbench-family","index":58.083963827965185,"aboveFit":true,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaMmmuPro","benchmarkName":"Artificial Analysis MMMU-Pro","category":"multimodal","score":63.2,"unit":"percent","family":"mmmu-pro","index":37.402070183366924,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmmu-pro","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmmuPro","benchmarkName":"Massive Multi-discipline Multimodal Understanding Pro","category":"multimodal","score":63,"unit":"percent","family":"mmmu-pro","index":37.14178773034308,"aboveFit":false,"run":null,"source":"MMMU-Pro authors","sourceUrl":"https://arxiv.org/abs/2409.02813","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:legal_bench:overall","benchmarkName":"LegalBench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":61.422,"unit":"percent","family":"vals-ai:legal_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","measuredAt":"2026-10-13","stderr":0.627,"costPerTest":0.033012,"latency":22.648},{"benchmarkId":"vals-ai:medscribe:overall","benchmarkName":"MedScribe","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":55.682,"unit":"percent","family":"vals-ai:medscribe:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medscribe","measuredAt":"2026-10-29","stderr":3.646,"costPerTest":0.140316,"latency":203.051},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":52.7,"unit":"percent","family":"lcr","index":43.454493157535,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"charxiv","benchmarkName":"CharXiv Reasoning","category":"multimodal","score":52.7,"unit":"percent","family":"charxiv","index":24.183068058758362,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaSciCode","benchmarkName":"Artificial Analysis SciCode","category":"coding","score":38.5,"unit":"percent","family":"scicode-family","index":46.0319075794785,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/scicode","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:medcode:overall","benchmarkName":"MedCode","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":19.718,"unit":"percent","family":"vals-ai:medcode:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medcode","measuredAt":"2026-10-29","stderr":1.835,"costPerTest":0.055695,"latency":103.751},{"benchmarkId":"vals-ai:terminal-bench-2-1:overall","benchmarkName":"Terminal-Bench 2.1","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":17.603,"unit":"percent","family":"terminal-bench-2-1","index":31.436330188172857,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2-1","measuredAt":"2026-10-01","stderr":0.375,"costPerTest":null,"latency":0},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":12,"unit":"percent","family":"hle","index":36.42615725367101,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"omniscienceAccuracy","benchmarkName":"Artificial Analysis Omniscience Accuracy","category":"knowledge","score":9.8,"unit":"percent","family":"omniscienceAccuracy","index":23.04045911040804,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:fabv2:overall","benchmarkName":"Finance Agent (v2)","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":9.044,"unit":"percent","family":"vals-ai:fabv2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/fabv2","measuredAt":"2026-10-29","stderr":0.409,"costPerTest":1.63003,"latency":566.745},{"benchmarkId":"gdpvalAaNormalized","benchmarkName":"GDPval-AA normalized","category":"agentic","score":0.7,"unit":"percent","family":"gdpvalAaNormalized","index":34.53425865037707,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":0.3,"unit":"percent","family":"critpt","index":37.42988100778257,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":0.3,"outputPerMillion":1.5,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":161,"timeToFirstToken":0.335,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":85,"benchmarkCount":13,"categoryCount":6,"confidence":"medium","state":"available"},"hosts":[{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":0.3,"outputPerMillion":1.5,"cacheReadPerMillion":0.15,"contextTokens":192000,"maxOutputTokens":64000,"deprecated":false},{"host":"cohere","name":"Cohere","kind":"maker","url":"https://docs.cohere.com/docs/models","inputPerMillion":2.5,"outputPerMillion":10,"cacheReadPerMillion":null,"contextTokens":128000,"maxOutputTokens":64000,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"o3-mini","providerId":"openai","slug":"o3-mini","name":"o3-mini","provider":"OpenAI","description":"o3-mini is a reasoning model from OpenAI in the o3 family. 23 benchmarks count toward its index, in 6 categories.","logoUrl":null,"releaseDate":"2025-01-31","contextWindowTokens":200000,"modalities":["text","file"],"openWeights":false,"details":{"apiId":"openai/o3-mini","maxOutputTokens":100000,"knowledgeCutoff":"2023-10-31","retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.55,"weightsUrl":null},"reasoning":true,"overallScore":40.02303832838254,"overallScoreRange":{"low":35.15136025307964,"high":44.89471640368544},"badges":[],"capabilityScores":{"agentic":21.833988457488964,"coding":37.126237016714214,"reasoning":43.22034717559138,"multimodal":null,"knowledge":39.008541589456286,"multilingual":null,"instruction":52.28898397172111,"math":46.09839212562813},"capabilityCounts":{"math":9,"instruction":1,"knowledge":5,"coding":3,"reasoning":4,"agentic":1},"benchmarks":[{"benchmarkId":"epoch-ai:epoch-math-level-5:overall","benchmarkName":"MATH level 5","category":"math","domain":null,"eligible":true,"ranked":true,"score":96.48791540785498,"unit":"percent","family":"epoch-ai:epoch-math-level-5:overall","index":45.86743492943001,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0.4265263461984669,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:medqa:overall","benchmarkName":"MedQA","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":94.833,"unit":"percent","family":"vals-ai:medqa:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medqa","measuredAt":"2026-04-16","stderr":0.397,"costPerTest":0.005327,"latency":8.387},{"benchmarkId":"ifeval","benchmarkName":"Instruction-Following Eval","category":"instruction","score":93.9,"unit":"percent","family":"ifeval","index":52.28898397172111,"aboveFit":false,"run":null,"source":"Jeffrey Zhou, Tianjian Lu, Swaroop Mishra, Siddhartha Brahma, Sujoy Basu, Yi Luan, Denny Zhou, Le Hou","sourceUrl":"https://arxiv.org/abs/2311.07911","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:math500:overall","benchmarkName":"MATH 500","category":"math","domain":null,"eligible":true,"ranked":true,"score":91.8,"unit":"percent","family":"math-500","index":41.137187157440266,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/math500","measuredAt":"2026-01-09","stderr":2.417,"costPerTest":0.007661,"latency":14.365},{"benchmarkId":"vals-ai:mgsm:overall","benchmarkName":"MGSM","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":91.346,"unit":"percent","family":"vals-ai:mgsm:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mgsm","measuredAt":"2026-01-09","stderr":1.718,"costPerTest":0.008194,"latency":15.976},{"benchmarkId":"aime2024","benchmarkName":"American Invitational Mathematics Examination 2024","category":"math","score":87.3,"unit":"percent","family":"aime2024","index":43.42887134483199,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmlu","benchmarkName":"Massive Multitask Language Understanding","category":"knowledge","score":86.9,"unit":"percent","family":"mmlu","index":35.2902238843779,"aboveFit":false,"run":null,"source":"Dan Hendrycks, Collin Burns, Steven Basart, Andy Zou, Mantas Mazeika, Dawn Song, Jacob Steinhardt","sourceUrl":"https://arxiv.org/abs/2009.03300","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:aime:overall","benchmarkName":"AIME","category":"math","domain":null,"eligible":true,"ranked":true,"score":86.458,"unit":"percent","family":"vals-ai:aime:overall","index":53.28866834586259,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/aime","measuredAt":"2026-04-16","stderr":1.054,"costPerTest":0.050817,"latency":154.65},{"benchmarkId":"vals-ai:mmlu_pro:overall","benchmarkName":"MMLU Pro","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":78.689,"unit":"percent","family":"mmlu-pro","index":43.61944959576386,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","measuredAt":"2026-09-01","stderr":0.394,"costPerTest":0.012991,"latency":22.375},{"benchmarkId":"gpqa","benchmarkName":"Graduate-Level Google-Proof Q&A","category":"knowledge","score":77.2,"unit":"percent","family":"gpqa-diamond","index":48.014831241656935,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":77.02020202020202,"unit":"percent","family":"gpqa-diamond","index":47.849925447580446,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.5855332698878484,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":76.94444444444444,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":53.684910506582746,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":5.239762076828802,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:gpqa:overall","benchmarkName":"GPQA Diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":75.505,"unit":"percent","family":"gpqa-diamond","index":46.460223557667895,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","measuredAt":"2026-09-01","stderr":2.176,"costPerTest":0.035857,"latency":99.984},{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":74.8,"unit":"percent","family":"gpqa-diamond","index":45.81361682184945,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:legal_bench:overall","benchmarkName":"LegalBench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":71.539,"unit":"percent","family":"vals-ai:legal_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","measuredAt":"2026-10-13","stderr":0.72,"costPerTest":null,"latency":12.011},{"benchmarkId":"vals-ai:lcb:overall","benchmarkName":"LiveCodeBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":71.484,"unit":"percent","family":"livecodebench","index":48.65698851753058,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","measuredAt":"2026-09-01","stderr":1.154,"costPerTest":0.033121,"latency":53.8},{"benchmarkId":"vals-ai:tax_eval_v2:overall","benchmarkName":"TaxEval v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":69.42,"unit":"percent","family":"vals-ai:tax_eval_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","measuredAt":"2026-09-01","stderr":0.909,"costPerTest":0.018142,"latency":74.312},{"benchmarkId":"sweVerified","benchmarkName":"Software Engineering Benchmark Verified","category":"coding","score":49.3,"unit":"percent","family":"swe-bench-verified","index":37.589516114207655,"aboveFit":false,"run":null,"source":"Carlos E. Jimenez, John Yang, Alexander Wettig, Shunyu Yao, Kexin Pei, Ofir Press, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2310.06770","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"arc-prize:v1-public-eval:overall","benchmarkName":"ARC-AGI-1 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":46.58,"unit":"percent","family":"arc-prize:v1-public-eval:overall","index":null,"aboveFit":false,"run":"high effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.3048,"latency":null},{"benchmarkId":"vals-ai:corp_fin_v2:overall","benchmarkName":"CorpFin v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":45.299,"unit":"percent","family":"vals-ai:corp_fin_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","measuredAt":"2026-08-12","stderr":0.964,"costPerTest":0.078121,"latency":31.047},{"benchmarkId":"swe-bench:swe-bench-verified:overall","benchmarkName":"SWE-bench Verified","category":"coding","domain":null,"eligible":true,"ranked":true,"score":42.4,"unit":"percent","family":"swe-bench-verified","index":32.50036882750018,"aboveFit":false,"run":"Agentless Lite","source":"SWE-bench team","sourceUrl":"https://www.swebench.com/","measuredAt":"2026-02-26","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"swebench-experiments:swe-bench-verified:overall","benchmarkName":"SWE-bench Verified","category":"coding","domain":null,"eligible":true,"ranked":true,"score":42.4,"unit":"percent","family":"swe-bench-verified","index":32.50036882750018,"aboveFit":false,"run":"Agentless Lite","source":"SWE-bench team","sourceUrl":"https://github.com/SWE-bench/experiments/tree/main/evaluation/verified","measuredAt":"2026-09-01","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-2025-02-28-public:overall","benchmarkName":"FrontierMath-2025-02-28-Public","category":"math","domain":null,"eligible":false,"ranked":true,"score":40,"unit":"percent","family":"epoch-ai:epoch-frontiermath-2025-02-28-public:overall","index":null,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":16.32993161855452,"costPerTest":null,"latency":null},{"benchmarkId":"arc-prize:v1-semi-private:overall","benchmarkName":"ARC-AGI-1 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":34.5,"unit":"percent","family":"arc-prize:v1-semi-private:overall","index":42.848254412807485,"aboveFit":false,"run":"high effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.3989,"latency":null},{"benchmarkId":"swe-bench:swe-bench-lite:overall","benchmarkName":"SWE-bench Lite","category":"coding","domain":null,"eligible":true,"ranked":true,"score":32.33,"unit":"percent","family":"swe-bench-lite","index":30.62661128181836,"aboveFit":false,"run":"Agentless Lite","source":"SWE-bench team","sourceUrl":"https://www.swebench.com/","measuredAt":"2025-09-11","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"swebench-experiments:swe-bench-lite:overall","benchmarkName":"SWE-bench Lite","category":"coding","domain":null,"eligible":true,"ranked":true,"score":31.333333333333332,"unit":"percent","family":"swe-bench-lite","index":29.816096128405416,"aboveFit":false,"run":"Aegis","source":"SWE-bench team","sourceUrl":"https://github.com/SWE-bench/experiments/tree/main/evaluation/lite","measuredAt":"2025-09-11","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"tau2Bench","benchmarkName":"τ²-Bench Tool-Agent-User Evaluation","category":"agentic","score":28.7,"unit":"percent","family":"tau2-bench","index":21.833988457488964,"aboveFit":false,"run":null,"source":"Victor Barres, Honghua Dong, Soham Ray, Xujie Si, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2506.07982","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tiers-1-3-v2-private:overall","benchmarkName":"FrontierMath-Tiers-1-3-v2-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":18.596491228070175,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tiers-1-3-v2-private:overall","index":40.682991287369106,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.3087552563757594,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-simpleqa-verified:overall","benchmarkName":"SimpleQA Verified","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":15.299999999999999,"unit":"percent","family":"epoch-ai:epoch-simpleqa-verified:overall","index":36.665302965466665,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.1389500459665505,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-chess-puzzles:overall","benchmarkName":"Chess Puzzles","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":15,"unit":"percent","family":"epoch-ai:epoch-chess-puzzles:overall","index":45.36990365239037,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":3.588702812826367,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-2025-02-28-private:overall","benchmarkName":"FrontierMath-2025-02-28-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":12.413793103448276,"unit":"percent","family":"epoch-ai:epoch-frontiermath-2025-02-28-private:overall","index":40.872792167172584,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.9396402576684655,"costPerTest":null,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":7.9,"unit":"percent","family":"hle","index":32.312656999048855,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-mystery-game-puzzles:overall","benchmarkName":"Mystery Game Puzzles","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":7.000000000000001,"unit":"percent","family":"epoch-ai:epoch-mystery-game-puzzles:overall","index":44.1019024790064,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.564323999762428,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-private:overall","benchmarkName":"FrontierMath-Tier-4-2025-07-01-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":4.166666666666666,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-private:overall","index":47.43436889471329,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.9147663515556026,"costPerTest":null,"latency":null},{"benchmarkId":"arc-prize:v2-semi-private:overall","benchmarkName":"ARC-AGI-2 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":2.9899999999999998,"unit":"percent","family":"arc-prize:v2-semi-private:overall","index":40.56132815816125,"aboveFit":false,"run":"high effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.5472,"latency":null},{"benchmarkId":"arc-prize:v2-public-eval:overall","benchmarkName":"ARC-AGI-2 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":0,"unit":"percent","family":"arc-prize:v2-public-eval:overall","index":null,"aboveFit":false,"run":"high effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.6072,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tier-4-v2-private:overall","benchmarkName":"FrontierMath-Tier-4-v2-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":0,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tier-4-v2-private:overall","index":48.48830449725058,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-public:overall","benchmarkName":"FrontierMath-Tier-4-2025-07-01-Public","category":"math","domain":null,"eligible":false,"ranked":true,"score":0,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-public:overall","index":null,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":1.1,"outputPerMillion":4.4,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":173,"timeToFirstToken":2.314,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":{"elo":1319.513646931629,"eloUpper":1323.0665299426548,"eloLower":1315.960763920603,"votes":56692,"rank":240,"rankUpper":null,"rankLower":null,"arenaName":"o3-mini","organization":"openai","updatedAt":"2026-11-09"},"quality":{"coveragePercent":85,"benchmarkCount":23,"categoryCount":6,"confidence":"medium","state":"available"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:swe-bench-team","name":"SWE-bench team","url":"https://www.swebench.com","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:arc-prize-foundation","name":"ARC Prize Foundation","url":"https://arcprize.org","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"lfm2-5-24b-a2b","providerId":"liquidai","slug":"lfm2-5-24b-a2b","name":"LFM2.5-24B-A2B","provider":"LiquidAI","description":"LFM2.5-24B-A2B is a reasoning model from LiquidAI. 11 benchmarks count toward its index, in 5 categories.","logoUrl":null,"releaseDate":"2026-10-13","contextWindowTokens":128000,"modalities":["text"],"openWeights":true,"details":{"apiId":"liquid/lfm-2.5-24b-a2b","maxOutputTokens":8192,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.024999999999999998,"weightsUrl":"https://huggingface.co/LiquidAI/LFM2.5-24B-A2B"},"reasoning":true,"overallScore":39.76449704616255,"overallScoreRange":{"low":31.889058604340114,"high":47.63993548798499},"badges":[],"capabilityScores":{"agentic":32.00901127070384,"coding":38.66344991908856,"reasoning":null,"multimodal":null,"knowledge":38.63805091595694,"multilingual":null,"instruction":58.42541149962,"math":45.61784197814297},"capabilityCounts":{"instruction":2,"math":1,"agentic":4,"knowledge":2,"coding":2},"benchmarks":[{"benchmarkId":"ifeval","benchmarkName":"Instruction-Following Eval","category":"instruction","score":96.75,"unit":"percent","family":"ifeval","index":58.02859578555952,"aboveFit":true,"run":null,"source":"Jeffrey Zhou, Tianjian Lu, Swaroop Mishra, Siddhartha Brahma, Sujoy Basu, Yi Luan, Denny Zhou, Le Hou","sourceUrl":"https://arxiv.org/abs/2311.07911","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"math500","benchmarkName":"MATH-500 Problem Set","category":"math","score":96.04,"unit":"percent","family":"math-500","index":45.61784197814297,"aboveFit":false,"run":null,"source":"Dan Hendrycks, Collin Burns, Saurav Kadavath, Akul Arora, Steven Basart, Eric Tang, Dawn Song, Jacob Steinhardt","sourceUrl":"https://arxiv.org/abs/2103.03874","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"pinchBench","benchmarkName":"PinchBench","category":"agentic","score":83.18,"unit":"percent","family":"pinchBench","index":42.755688522891695,"aboveFit":false,"run":null,"source":"Kilo Code","sourceUrl":"https://pinchbench.com/about","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"ifBench","benchmarkName":"Instruction Following Benchmark","category":"instruction","score":74.69,"unit":"percent","family":"ifbench-family","index":58.822227213680485,"aboveFit":true,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gpqa","benchmarkName":"Graduate-Level Google-Proof Q&A","category":"knowledge","score":74.42,"unit":"percent","family":"gpqa-diamond","index":45.465091205379935,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gpqaDiamond","benchmarkName":"GPQA Diamond","category":"knowledge","score":74.42,"unit":"percent","family":"gpqa-diamond","index":45.465091205379935,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"bfclV4","benchmarkName":"Berkeley Function Calling Leaderboard v4","category":"agentic","score":65.01,"unit":"percent","family":"bfclV4","index":45.52082564043922,"aboveFit":false,"run":null,"source":"Shishir G. Patil et al.","sourceUrl":"https://gorilla.cs.berkeley.edu/leaderboard.html","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"sweVerified","benchmarkName":"Software Engineering Benchmark Verified","category":"coding","score":51,"unit":"percent","family":"swe-bench-verified","index":38.84336399643993,"aboveFit":false,"run":null,"source":"Carlos E. Jimenez, John Yang, Alexander Wettig, Shunyu Yao, Kexin Pei, Ofir Press, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2310.06770","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"sciCode","benchmarkName":"Scientific Code Benchmark","category":"coding","score":32.71,"unit":"percent","family":"scicode-family","index":38.48353584173719,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"automationBench","benchmarkName":"AutomationBench","category":"agentic","score":27.69,"unit":"percent","family":"automationBench","index":34.75953091948443,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"webArenaVerified","benchmarkName":"WebArena-Verified Browser Agent Benchmark","category":"agentic","score":26.75,"unit":"percent","family":"webArenaVerified","index":null,"aboveFit":false,"run":null,"source":"Amine El Hattami, Megh Thakkar, Nicolas Chapados, Christopher Pal","sourceUrl":"https://openreview.net/forum?id=94tlGxmqkN","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"tau3Bench","benchmarkName":"τ³-Bench Tool-Agent-User Evaluation","category":"agentic","score":8.04,"unit":"percent","family":"tau3Bench","index":5,"aboveFit":false,"run":null,"source":"Sierra Research","sourceUrl":"https://github.com/sierra-research/tau2-bench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hle","benchmarkName":"Humanity's Last Exam","category":"knowledge","score":7.4,"unit":"percent","family":"hle","index":31.811010626533953,"aboveFit":false,"run":null,"source":"Center for AI Safety, Scale AI, and thousands of expert contributors","sourceUrl":"https://lastexam.ai/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":0.049999999999999996,"outputPerMillion":0.7,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":280,"timeToFirstToken":0.295,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":70,"benchmarkCount":11,"categoryCount":5,"confidence":"low","state":"available"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"kimi-k2","providerId":"moonshot-ai","slug":"kimi-k2","name":"Kimi K2","provider":"Moonshot AI","description":"Kimi K2 is a standard model from Moonshot AI. 17 benchmarks count toward its index, in 6 categories.","logoUrl":null,"releaseDate":"2025-07-11","contextWindowTokens":131072,"modalities":["text"],"openWeights":true,"details":{"apiId":"moonshotai/kimi-k2","maxOutputTokens":98304,"knowledgeCutoff":"2024-12-31","retiresAt":null,"tools":true,"structuredOutput":false,"cacheReadPerMillion":null,"weightsUrl":"https://huggingface.co/moonshotai/Kimi-K2-Instruct"},"reasoning":false,"overallScore":39.580366198415035,"overallScoreRange":{"low":34.15856514157753,"high":45.00216725525254},"badges":[],"capabilityScores":{"agentic":44.89315409676477,"coding":38.98601651154514,"reasoning":40.2547166639918,"multimodal":null,"knowledge":41.51305808248766,"multilingual":null,"instruction":35.918909231405614,"math":35.991617278660385},"capabilityCounts":{"math":3,"instruction":2,"knowledge":7,"agentic":1,"coding":2,"reasoning":2},"benchmarks":[{"benchmarkId":"math500","benchmarkName":"MATH-500 Problem Set","category":"math","score":97.4,"unit":"percent","family":"math-500","index":47.0550331470476,"aboveFit":false,"run":null,"source":"Dan Hendrycks, Collin Burns, Saurav Kadavath, Akul Arora, Steven Basart, Eric Tang, Dawn Song, Jacob Steinhardt","sourceUrl":"https://arxiv.org/abs/2103.03874","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaMath500","benchmarkName":"Artificial Analysis MATH-500","category":"math","score":97.1,"unit":"percent","family":"math-500","index":46.738005683318626,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/math-500","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"ifeval","benchmarkName":"Instruction-Following Eval","category":"instruction","score":89.8,"unit":"percent","family":"ifeval","index":44.03199855532196,"aboveFit":false,"run":null,"source":"Jeffrey Zhou, Tianjian Lu, Swaroop Mishra, Siddhartha Brahma, Sujoy Basu, Yi Luan, Denny Zhou, Le Hou","sourceUrl":"https://arxiv.org/abs/2311.07911","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmlu","benchmarkName":"Massive Multitask Language Understanding","category":"knowledge","score":89.5,"unit":"percent","family":"mmlu","index":38.39459377545052,"aboveFit":false,"run":null,"source":"Dan Hendrycks, Collin Burns, Steven Basart, Andy Zou, Mantas Mazeika, Dawn Song, Jacob Steinhardt","sourceUrl":"https://arxiv.org/abs/2009.03300","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmluPro","benchmarkName":"Massive Multitask Language Understanding Professional","category":"knowledge","score":81.1,"unit":"percent","family":"mmlu-pro","index":46.6570210064822,"aboveFit":false,"run":null,"source":"Yubo Wang, Xueguang Ma, Ge Zhang, Yuansheng Ni, Abhranil Chandra, Shiguang Guo, Weiming Ren, Aaran Arulraj, Xuan He, Ziyan Jiang, Tianle Li, Max Ku, Kai Wang, Alex Zhuang, Rongqi Fan, Xiang Yue, Wenhu Chen","sourceUrl":"https://arxiv.org/abs/2406.01574","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":76.6,"unit":"percent","family":"gpqa-diamond","index":47.46452763670506,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gpqa","benchmarkName":"Graduate-Level Google-Proof Q&A","category":"knowledge","score":75.1,"unit":"percent","family":"gpqa-diamond","index":46.08876862432538,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"tau2-bench:retail:overall","benchmarkName":"τ²-bench Retail","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":70.6,"unit":"percent","family":"tau2-bench","index":49.637701847004244,"aboveFit":false,"run":"Moonshot AI","source":"Sierra Research","sourceUrl":"https://github.com/sierra-research/tau2-bench","measuredAt":"2026-04-30","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aime2024","benchmarkName":"American Invitational Mathematics Examination 2024","category":"math","score":69.6,"unit":"percent","family":"aime2024","index":35.211132752557425,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"sweVerified","benchmarkName":"Software Engineering Benchmark Verified","category":"coding","score":65.8,"unit":"percent","family":"swe-bench-verified","index":49.759216147638575,"aboveFit":false,"run":null,"source":"Carlos E. Jimenez, John Yang, Alexander Wettig, Shunyu Yao, Kexin Pei, Ofir Press, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2310.06770","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"tau2-bench:telecom:overall","benchmarkName":"τ²-bench Telecom","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":65.8,"unit":"percent","family":"tau2-bench","index":46.452550909780534,"aboveFit":false,"run":"Moonshot AI","source":"Sierra Research","sourceUrl":"https://github.com/sierra-research/tau2-bench","measuredAt":"2026-03-02","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"swe-bench:swe-bench-verified:overall","benchmarkName":"SWE-bench Verified","category":"coding","domain":null,"eligible":true,"ranked":true,"score":65.4,"unit":"percent","family":"swe-bench-verified","index":49.4641931165251,"aboveFit":false,"run":"OpenHands","source":"SWE-bench team","sourceUrl":"https://www.swebench.com/","measuredAt":"2026-02-26","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"swebench-experiments:swe-bench-verified:overall","benchmarkName":"SWE-bench Verified","category":"coding","domain":null,"eligible":true,"ranked":true,"score":65.4,"unit":"percent","family":"swe-bench-verified","index":49.4641931165251,"aboveFit":false,"run":"OpenHands","source":"SWE-bench team","sourceUrl":"https://github.com/SWE-bench/experiments/tree/main/evaluation/verified","measuredAt":"2026-09-01","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"tau2Bench","benchmarkName":"τ²-Bench Tool-Agent-User Evaluation","category":"agentic","score":61.1,"unit":"percent","family":"tau2-bench","index":43.333757283748994,"aboveFit":false,"run":null,"source":"Victor Barres, Honghua Dong, Soham Ray, Xujie Si, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2506.07982","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"superGpqa","benchmarkName":"SuperGPQA: Scaling LLM Evaluation Across 285 Graduate Disciplines","category":"knowledge","score":57.2,"unit":"percent","family":"superGpqa","index":44.215221119879985,"aboveFit":false,"run":null,"source":"Xiaoxuan Du, Yao Yao, Kexin Ma, Bowen Wang, Tianyu Zheng, Kaiyan Zhu, Yiming Zhang, Yutao Zhu, Jiawei Zhou, Jingren Zhou","sourceUrl":"https://arxiv.org/abs/2502.14739","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"tau2-bench:airline:overall","benchmarkName":"τ²-bench Airline","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":56.5,"unit":"percent","family":"tau2-bench","index":40.281320968909604,"aboveFit":false,"run":"Moonshot AI","source":"Sierra Research","sourceUrl":"https://github.com/sierra-research/tau2-bench","measuredAt":"2026-03-02","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"liveCodeBenchV6","benchmarkName":"LiveCodeBench v6","category":"coding","score":53.7,"unit":"percent","family":"liveCodeBenchV6","index":28.507839906565174,"aboveFit":false,"run":null,"source":"LiveCodeBench maintainers","sourceUrl":"https://github.com/LiveCodeBench/LiveCodeBench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":53,"unit":"percent","family":"lcr","index":43.6603173152107,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aime2025","benchmarkName":"American Invitational Mathematics Examination 2025","category":"math","score":49.5,"unit":"percent","family":"aime2025","index":25.867199668240612,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaIfBench","benchmarkName":"Artificial Analysis IFBench","category":"instruction","score":41.5,"unit":"percent","family":"ifbench-family","index":27.805819907489273,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"simpleQa","benchmarkName":"Measuring Short-Form Factuality in Large Language Models","category":"knowledge","score":31,"unit":"percent","family":"simpleQa","index":38.87039269448607,"aboveFit":false,"run":null,"source":"Jason Wei, Najoung Kim, Hyung Won Chung, Yu-An Chung, Siddhartha Papay, Yifeng Lu, Hannaneh Hajishirzi, Luke Zettlemoyer","sourceUrl":"https://arxiv.org/abs/2411.04368","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"omniscienceAccuracy","benchmarkName":"Artificial Analysis Omniscience Accuracy","category":"knowledge","score":27.4,"unit":"percent","family":"omniscienceAccuracy","index":45.220964429855876,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":7.4,"unit":"percent","family":"hle","index":31.811010626533953,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hle","benchmarkName":"Humanity's Last Exam","category":"knowledge","score":4.7,"unit":"percent","family":"hle","index":29.10212021495351,"aboveFit":false,"run":null,"source":"Center for AI Safety, Scale AI, and thousands of expert contributors","sourceUrl":"https://lastexam.ai/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":0,"unit":"percent","family":"critpt","index":36.8491160127729,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":0.5700000000000001,"outputPerMillion":2.3,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":37,"timeToFirstToken":1.2615,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":85,"benchmarkCount":17,"categoryCount":6,"confidence":"medium","state":"available"},"hosts":[{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":0.57,"outputPerMillion":2.3,"cacheReadPerMillion":null,"contextTokens":131072,"maxOutputTokens":98304,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:sierra-research","name":"Sierra Research","url":"https://github.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:swe-bench-team","name":"SWE-bench team","url":"https://www.swebench.com","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"claude-4-sonnet","providerId":"anthropic","slug":"claude-4-sonnet","name":"Claude 4 Sonnet","provider":"Anthropic","description":"Claude 4 Sonnet is a standard model from Anthropic. 13 benchmarks count toward its index, in 7 categories.","logoUrl":null,"releaseDate":"2025-05-22","contextWindowTokens":200000,"modalities":["text"],"openWeights":false,"details":null,"reasoning":false,"overallScore":39.517298262926516,"overallScoreRange":{"low":34.53254717644554,"high":44.50204934940749},"badges":[],"capabilityScores":{"agentic":43.94168398596025,"coding":52.10251131607936,"reasoning":38.23209012304071,"multimodal":36.36094037127158,"knowledge":35.9501862855295,"multilingual":null,"instruction":31.45041130532433,"math":16.968136280273697},"capabilityCounts":{"coding":2,"knowledge":3,"multimodal":1,"agentic":3,"instruction":1,"reasoning":2,"math":1},"benchmarks":[{"benchmarkId":"sweVerified","benchmarkName":"Software Engineering Benchmark Verified","category":"coding","score":72.7,"unit":"percent","family":"swe-bench-verified","index":54.848363434346055,"aboveFit":false,"run":null,"source":"Carlos E. Jimenez, John Yang, Alexander Wettig, Shunyu Yao, Kexin Pei, Ofir Press, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2310.06770","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"swebench-experiments:swe-bench-verified:overall","benchmarkName":"SWE-bench Verified","category":"coding","domain":null,"eligible":true,"ranked":true,"score":70.8,"unit":"percent","family":"swe-bench-verified","index":53.44700403655703,"aboveFit":false,"run":"Moatless Tools","source":"SWE-bench team","sourceUrl":"https://github.com/SWE-bench/experiments/tree/main/evaluation/verified","measuredAt":"2026-09-01","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"swe-bench:swe-bench-verified:overall","benchmarkName":"SWE-bench Verified","category":"coding","domain":null,"eligible":true,"ranked":true,"score":68.5,"unit":"percent","family":"swe-bench-verified","index":51.750621607654544,"aboveFit":false,"run":"OpenHands","source":"SWE-bench team","sourceUrl":"https://www.swebench.com/","measuredAt":"2026-02-26","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":68.3,"unit":"percent","family":"gpqa-diamond","index":39.851994434870846,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaMmmuPro","benchmarkName":"Artificial Analysis MMMU-Pro","category":"multimodal","score":62.4,"unit":"percent","family":"mmmu-pro","index":36.36094037127158,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmmu-pro","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"swebench-experiments:swe-bench-lite:overall","benchmarkName":"SWE-bench Lite","category":"coding","domain":null,"eligible":true,"ranked":true,"score":57.5,"unit":"percent","family":"swe-bench-lite","index":51.09550734677195,"aboveFit":false,"run":"SWE-agent","source":"SWE-bench team","sourceUrl":"https://github.com/SWE-bench/experiments/tree/main/evaluation/lite","measuredAt":"2025-09-11","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"swe-bench:swe-bench-lite:overall","benchmarkName":"SWE-bench Lite","category":"coding","domain":null,"eligible":true,"ranked":true,"score":56.67,"unit":"percent","family":"swe-bench-lite","index":50.420529844431414,"aboveFit":false,"run":"SWE-agent","source":"SWE-bench team","sourceUrl":"https://www.swebench.com/","measuredAt":"2025-09-11","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"tau2Bench","benchmarkName":"τ²-Bench Tool-Agent-User Evaluation","category":"agentic","score":52.3,"unit":"percent","family":"tau2-bench","index":37.49431389883886,"aboveFit":false,"run":null,"source":"Victor Barres, Honghua Dong, Soham Ray, Xujie Si, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2506.07982","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaIfBench","benchmarkName":"Artificial Analysis IFBench","category":"instruction","score":45.4,"unit":"percent","family":"ifbench-family","index":31.45041130532433,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":44,"unit":"percent","family":"lcr","index":37.48559258493975,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gertLabs","benchmarkName":"Gert Labs Composite Game Benchmark","category":"agentic","score":39.66,"unit":"percent","family":"gertLabs","index":47.66012464292313,"aboveFit":false,"run":null,"source":"Gert Labs","sourceUrl":"https://gertlabs.com/rankings","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"swebench-experiments:swe-bench-multimodal:overall","benchmarkName":"SWE-bench Multimodal","category":"multimodal","domain":null,"eligible":true,"ranked":true,"score":35.5899419729207,"unit":"percent","family":"swe-bench-multimodal","index":null,"aboveFit":false,"run":"Refact.ai Agent","source":"SWE-bench team","sourceUrl":"https://github.com/SWE-bench/experiments/tree/main/evaluation/multimodal","measuredAt":"2025-11-17","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aime2025","benchmarkName":"American Invitational Mathematics Examination 2025","category":"math","score":33.1,"unit":"percent","family":"aime2025","index":16.968136280273697,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"omniscienceAccuracy","benchmarkName":"Artificial Analysis Omniscience Accuracy","category":"knowledge","score":22.7,"unit":"percent","family":"omniscienceAccuracy","index":39.29776130477606,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"jobBench","benchmarkName":"JobBench","category":"agentic","score":18.4,"unit":"percent","family":"jobBench","index":46.67061341611876,"aboveFit":false,"run":null,"source":"Yuetai Li, Yichen Feng, Zhangchen Xu, Zixian Ma, Kaiyuan Zheng, Fengqing Jiang, Xinghua Sun, Rulin Shao, Zichen Chen, Yue Huang, Xinyang Han, Brian Lee, Kayla Xu, Shenglai Zeng, Hang Hua, Xiangliang Zhang, Basel Alomair, Ranjay Krishna, Luke Zettlemoyer, Pang Wei Koh, Bhaskar Ramasubramanian, Luyao Niu, Xiang Yue, Radha Poovendran","sourceUrl":"https://arxiv.org/abs/2605.26329","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":4.3,"unit":"percent","family":"hle","index":28.700803116941593,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":1.1,"unit":"percent","family":"critpt","index":38.97858766114168,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":3,"outputPerMillion":15,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":40,"timeToFirstToken":1.33,"source":"BenchLM","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":95,"benchmarkCount":13,"categoryCount":7,"confidence":"medium","state":"available"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:swe-bench-team","name":"SWE-bench team","url":"https://github.com","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"longcat-flash-lite-sparse","providerId":"meituan","slug":"longcat-flash-lite-sparse","name":"LongCat-Flash-Lite-Sparse","provider":"Meituan","description":"LongCat-Flash-Lite-Sparse is a reasoning model from Meituan in the LongCat-Flash-Lite family. 14 benchmarks count toward its index, in 4 categories.","logoUrl":null,"releaseDate":"2026-07-31","contextWindowTokens":1000000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":true,"overallScore":39.2928618062931,"overallScoreRange":{"low":31.471247524569193,"high":47.114476088017},"badges":[],"capabilityScores":{"agentic":39.606178047437496,"coding":47.484825771950355,"reasoning":null,"multimodal":null,"knowledge":39.690706502882946,"multilingual":null,"instruction":null,"math":32.54162730728861},"capabilityCounts":{"math":4,"knowledge":4,"coding":2,"agentic":4},"benchmarks":[{"benchmarkId":"math500","benchmarkName":"MATH-500 Problem Set","category":"math","score":95.8,"unit":"percent","family":"math-500","index":45.36422000715979,"aboveFit":false,"run":null,"source":"Dan Hendrycks, Collin Burns, Saurav Kadavath, Akul Arora, Steven Basart, Eric Tang, Dawn Song, Jacob Steinhardt","sourceUrl":"https://arxiv.org/abs/2103.03874","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"cEval","benchmarkName":"C-Eval","category":"knowledge","score":85.76,"unit":"percent","family":"cEval","index":40.113970153602175,"aboveFit":false,"run":null,"source":"C-Eval authors","sourceUrl":"https://arxiv.org/abs/2305.08322","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmlu","benchmarkName":"Massive Multitask Language Understanding","category":"knowledge","score":85.31,"unit":"percent","family":"mmlu","index":33.39178229714502,"aboveFit":false,"run":null,"source":"Dan Hendrycks, Collin Burns, Steven Basart, Andy Zou, Mantas Mazeika, Dawn Song, Jacob Steinhardt","sourceUrl":"https://arxiv.org/abs/2009.03300","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"cmmlu","benchmarkName":"Chinese Massive Multitask Language Understanding","category":"knowledge","score":84.25,"unit":"percent","family":"cmmlu","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmluPro","benchmarkName":"Massive Multitask Language Understanding Professional","category":"knowledge","score":79.24,"unit":"percent","family":"mmlu-pro","index":44.31364364275922,"aboveFit":false,"run":null,"source":"Yubo Wang, Xueguang Ma, Ge Zhang, Yuansheng Ni, Abhranil Chandra, Shiguang Guo, Weiming Ren, Aaran Arulraj, Xuan He, Ziyan Jiang, Tianle Li, Max Ku, Kai Wang, Alex Zhuang, Rongqi Fan, Xiang Yue, Wenhu Chen","sourceUrl":"https://arxiv.org/abs/2406.01574","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gpqaDiamond","benchmarkName":"GPQA Diamond","category":"knowledge","score":69.49,"unit":"percent","family":"gpqa-diamond","index":40.94342991802539,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"sweVerified","benchmarkName":"Software Engineering Benchmark Verified","category":"coding","score":68.2,"unit":"percent","family":"swe-bench-verified","index":51.529354334319436,"aboveFit":false,"run":null,"source":"Carlos E. Jimenez, John Yang, Alexander Wettig, Shunyu Yao, Kexin Pei, Ofir Press, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2310.06770","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aime2026","benchmarkName":"AIME 2026","category":"math","score":65.73,"unit":"percent","family":"aime2026","index":32.61266697990236,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"imoAnswerBench","benchmarkName":"IMOAnswerBench","category":"math","score":49.38,"unit":"percent","family":"imoAnswerBench","index":30.425297075695276,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"browseComp","benchmarkName":"BrowseComp","category":"agentic","score":48.62,"unit":"percent","family":"browseComp","index":38.31975701828445,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mcpAtlas","benchmarkName":"MCP Atlas","category":"agentic","score":45.6,"unit":"percent","family":"mcpAtlas","index":41.76912249248224,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"swePro","benchmarkName":"SWE-bench Pro","category":"coding","score":40.63,"unit":"percent","family":"swePro","index":43.440297209581274,"aboveFit":false,"run":null,"source":"Xiang Deng, Jeff Da, Edwin Pan, Yannis Yiming He, Charles Ide, Kanak Garg, Niklas Lauffer, Andrew Park, Nitin Pasari, Chetan Rane, Karmini Sampath, Maya Krishnan, Srivatsa Kundurthy, Sean Hendryx, Zifan Wang, Chen Bo Calvin Zhang, Noah Jacobson, Bing Liu, Brad Kenstler","sourceUrl":"https://arxiv.org/abs/2509.16941","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hmmtFeb2026","benchmarkName":"Harvard-MIT Mathematics Tournament February 2026","category":"math","score":40.53,"unit":"percent","family":"hmmtFeb2026","index":21.764325166397015,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"terminalBench2","benchmarkName":"Terminal-Bench 2.0","category":"agentic","score":33.7,"unit":"percent","family":"terminalBench2","index":37.869981645890434,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vitaBench","benchmarkName":"VITA-Bench","category":"agentic","score":21.67,"unit":"percent","family":"vitaBench","index":40.46585103309287,"aboveFit":false,"run":null,"source":"Meituan LongCat Team","sourceUrl":"https://vitabench.github.io/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":65,"benchmarkCount":14,"categoryCount":4,"confidence":"low","state":"available"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"grok-3-beta","providerId":"xai","slug":"grok-3-beta","name":"Grok 3 [Beta]","provider":"xAI","description":"Grok 3 [Beta] is a standard model from xAI in the Grok 3 family. 13 benchmarks count toward its index, in 4 categories.","logoUrl":null,"releaseDate":"2025-02-19","contextWindowTokens":128000,"modalities":["text"],"openWeights":false,"details":null,"reasoning":false,"overallScore":39.25733734181028,"overallScoreRange":{"low":30.7725862553293,"high":47.742088428291254},"badges":[],"capabilityScores":{"agentic":null,"coding":34.938461907813796,"reasoning":34.35221141439627,"multimodal":null,"knowledge":47.06092140198038,"multilingual":null,"instruction":null,"math":38.16402853470577},"capabilityCounts":{"math":7,"knowledge":3,"coding":1,"reasoning":2},"benchmarks":[{"benchmarkId":"vals-ai:mgsm:overall","benchmarkName":"MGSM","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":91.346,"unit":"percent","family":"vals-ai:mgsm:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mgsm","measuredAt":"2026-01-09","stderr":1.72,"costPerTest":0.00567,"latency":5.831},{"benchmarkId":"vals-ai:math500:overall","benchmarkName":"MATH 500","category":"math","domain":null,"eligible":true,"ranked":true,"score":89.8,"unit":"percent","family":"math-500","index":39.0236707325805,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/math500","measuredAt":"2026-01-09","stderr":1.354,"costPerTest":0.012781,"latency":9.085},{"benchmarkId":"epoch-ai:epoch-math-level-5:overall","benchmarkName":"MATH level 5","category":"math","domain":null,"eligible":true,"ranked":true,"score":88.74622356495469,"unit":"percent","family":"epoch-ai:epoch-math-level-5:overall","index":42.10686648040326,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0.8688487093324447,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:medqa:overall","benchmarkName":"MedQA","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":83.85,"unit":"percent","family":"vals-ai:medqa:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medqa","measuredAt":"2026-04-16","stderr":0.336,"costPerTest":0.006196,"latency":7.536},{"benchmarkId":"vals-ai:legal_bench:overall","benchmarkName":"LegalBench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":82.595,"unit":"percent","family":"vals-ai:legal_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","measuredAt":"2026-10-13","stderr":0.394,"costPerTest":null,"latency":0.741},{"benchmarkId":"vals-ai:mmlu_pro:overall","benchmarkName":"MMLU Pro","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":79.949,"unit":"percent","family":"mmlu-pro","index":45.20689877764073,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","measuredAt":"2026-09-01","stderr":0.39,"costPerTest":0.016622,"latency":12.096},{"benchmarkId":"mmluPro","benchmarkName":"Massive Multitask Language Understanding Professional","category":"knowledge","score":79.9,"unit":"percent","family":"mmlu-pro","index":45.14516464278997,"aboveFit":false,"run":null,"source":"Yubo Wang, Xueguang Ma, Ge Zhang, Yuansheng Ni, Abhranil Chandra, Shiguang Guo, Weiming Ren, Aaran Arulraj, Xuan He, Ziyan Jiang, Tianle Li, Max Ku, Kai Wang, Alex Zhuang, Rongqi Fan, Xiang Yue, Wenhu Chen","sourceUrl":"https://arxiv.org/abs/2406.01574","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:tax_eval_v2:overall","benchmarkName":"TaxEval v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":75.879,"unit":"percent","family":"vals-ai:tax_eval_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","measuredAt":"2026-09-01","stderr":0.83,"costPerTest":0.017475,"latency":13.058},{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":75.75757575757575,"unit":"percent","family":"gpqa-diamond","index":46.69187914086354,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.686307874634035,"costPerTest":null,"latency":null},{"benchmarkId":"gpqa","benchmarkName":"Graduate-Level Google-Proof Q&A","category":"knowledge","score":75.4,"unit":"percent","family":"gpqa-diamond","index":46.36392042680133,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:gpqa:overall","benchmarkName":"GPQA Diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":74.242,"unit":"percent","family":"gpqa-diamond","index":45.30183446924421,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","measuredAt":"2026-09-01","stderr":2.209,"costPerTest":0.039242,"latency":29.912},{"benchmarkId":"vals-ai:corp_fin_v2:overall","benchmarkName":"CorpFin v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":59.713,"unit":"percent","family":"vals-ai:corp_fin_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","measuredAt":"2026-08-12","stderr":0.965,"costPerTest":0.169809,"latency":24.354},{"benchmarkId":"vals-ai:aime:overall","benchmarkName":"AIME","category":"math","domain":null,"eligible":true,"ranked":true,"score":58.75,"unit":"percent","family":"vals-ai:aime:overall","index":41.17900441573685,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/aime","measuredAt":"2026-04-16","stderr":1.439,"costPerTest":0.090668,"latency":63.989},{"benchmarkId":"liveCodeBench","benchmarkName":"LiveCodeBench: Holistic and Contamination Free Evaluation of Large Language Models for Code","category":"coding","score":57,"unit":"percent","family":"livecodebench","index":36.63901655665491,"aboveFit":false,"run":null,"source":"Naman Jain, King Han, Alex Gu, Wen-Ding Li, Fanjia Yan, Tianjun Zhang, Sida Wang, Armando Solar-Lezama, Koushik Sen, Ion Stoica","sourceUrl":"https://arxiv.org/abs/2403.07974","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":55.55555555555556,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":41.240982306180626,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":7.4911095829249135,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:lcb:overall","benchmarkName":"LiveCodeBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":52.901,"unit":"percent","family":"livecodebench","index":33.23790725897268,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","measuredAt":"2026-09-01","stderr":1.134,"costPerTest":0.009356,"latency":5.517},{"benchmarkId":"aime2024","benchmarkName":"American Invitational Mathematics Examination 2024","category":"math","score":52.2,"unit":"percent","family":"aime2024","index":27.132677865236676,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"simpleQa","benchmarkName":"Measuring Short-Form Factuality in Large Language Models","category":"knowledge","score":43.6,"unit":"percent","family":"simpleQa","index":49.64281206892447,"aboveFit":false,"run":null,"source":"Jason Wei, Najoung Kim, Hyung Won Chung, Yu-An Chung, Siddhartha Papay, Yifeng Lu, Hannaneh Hajishirzi, Luke Zettlemoyer","sourceUrl":"https://arxiv.org/abs/2411.04368","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"arc-prize:v1-public-eval:overall","benchmarkName":"ARC-AGI-1 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":8.38,"unit":"percent","family":"arc-prize:v1-public-eval:overall","index":null,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.0735,"latency":null},{"benchmarkId":"arc-prize:v1-semi-private:overall","benchmarkName":"ARC-AGI-1 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":5.5,"unit":"percent","family":"arc-prize:v1-semi-private:overall","index":29.515907426549994,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.0931,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-2025-02-28-private:overall","benchmarkName":"FrontierMath-2025-02-28-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":3.793103448275862,"unit":"percent","family":"epoch-ai:epoch-frontiermath-2025-02-28-private:overall","index":33.26264021563687,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.1237029601999629,"costPerTest":null,"latency":null},{"benchmarkId":"arc-prize:v2-public-eval:overall","benchmarkName":"ARC-AGI-2 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":0,"unit":"percent","family":"arc-prize:v2-public-eval:overall","index":null,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.1402,"latency":null},{"benchmarkId":"arc-prize:v2-semi-private:overall","benchmarkName":"ARC-AGI-2 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":0,"unit":"percent","family":"arc-prize:v2-semi-private:overall","index":39.188515402242544,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.1421,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-private:overall","benchmarkName":"FrontierMath-Tier-4-2025-07-01-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":0,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-private:overall","index":43.20235772716562,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-public:overall","benchmarkName":"FrontierMath-Tier-4-2025-07-01-Public","category":"math","domain":null,"eligible":false,"ranked":true,"score":0,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-public:overall","index":null,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-2025-02-28-public:overall","benchmarkName":"FrontierMath-2025-02-28-Public","category":"math","domain":null,"eligible":false,"ranked":true,"score":0,"unit":"percent","family":"epoch-ai:epoch-frontiermath-2025-02-28-public:overall","index":null,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":60,"benchmarkCount":13,"categoryCount":4,"confidence":"low","state":"available"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:arc-prize-foundation","name":"ARC Prize Foundation","url":"https://arcprize.org","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"solar-pro-3","providerId":"upstage","slug":"solar-pro-3","name":"Solar Pro 3","provider":"Upstage","description":"Solar Pro 3 is a reasoning model from Upstage in the Solar family. 9 benchmarks count toward its index, in 5 categories.","logoUrl":null,"releaseDate":"2026-01-26","contextWindowTokens":131072,"modalities":["text"],"openWeights":false,"details":{"apiId":"upstage/solar-pro-3","maxOutputTokens":117964,"knowledgeCutoff":null,"retiresAt":"2026-10-30","tools":true,"structuredOutput":true,"cacheReadPerMillion":0.015,"weightsUrl":null},"reasoning":true,"overallScore":39.15012939860838,"overallScoreRange":{"low":31.260129398608377,"high":47.04012939860838},"badges":[],"capabilityScores":{"agentic":47.04478731068312,"coding":29.08392267608697,"reasoning":33.153783224180216,"multimodal":null,"knowledge":37.44588272029611,"multilingual":null,"instruction":48.35570832759005,"math":null},"capabilityCounts":{"agentic":2,"knowledge":3,"instruction":1,"reasoning":2,"coding":1},"benchmarks":[{"benchmarkId":"tau2Bench","benchmarkName":"τ²-Bench Tool-Agent-User Evaluation","category":"agentic","score":86.3,"unit":"percent","family":"tau2-bench","index":60.05579970417346,"aboveFit":true,"run":null,"source":"Victor Barres, Honghua Dong, Soham Ray, Xujie Si, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2506.07982","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":72.4,"unit":"percent","family":"gpqa-diamond","index":43.61240240204197,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaIfBench","benchmarkName":"Artificial Analysis IFBench","category":"instruction","score":71.2,"unit":"percent","family":"ifbench-family","index":55.56078516792552,"aboveFit":true,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"ifBench","benchmarkName":"Instruction Following Benchmark","category":"instruction","score":55.78,"unit":"percent","family":"ifbench-family","index":41.15063148725458,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":32.3,"unit":"percent","family":"lcr","index":29.458450435587526,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaSciCode","benchmarkName":"Artificial Analysis SciCode","category":"coding","score":25.5,"unit":"percent","family":"scicode-family","index":29.08392267608697,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/scicode","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"omniscienceAccuracy","benchmarkName":"Artificial Analysis Omniscience Accuracy","category":"knowledge","score":18.5,"unit":"percent","family":"omniscienceAccuracy","index":34.00468617172601,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":10.3,"unit":"percent","family":"hle","index":34.72055958712036,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gdpvalAaNormalized","benchmarkName":"GDPval-AA normalized","category":"agentic","score":0,"unit":"percent","family":"gdpvalAaNormalized","index":34.03377491719279,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":0,"unit":"percent","family":"critpt","index":36.8491160127729,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":0.15,"outputPerMillion":0.6,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":114,"timeToFirstToken":0.2085,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":75,"benchmarkCount":9,"categoryCount":5,"confidence":"low","state":"available"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gemma-4-12b","providerId":"google","slug":"gemma-4-12b","name":"Gemma 4 12B","provider":"Google","description":"Gemma 4 12B is a reasoning model from Google in the Gemma 4 family. 16 benchmarks count toward its index, in 8 categories.","logoUrl":null,"releaseDate":"2026-06-03","contextWindowTokens":256000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":true,"overallScore":39.024354486946265,"overallScoreRange":{"low":34.98185448694626,"high":43.06685448694627},"badges":[],"capabilityScores":{"agentic":30.455459512642978,"coding":43.5331856976486,"reasoning":41.53484388354889,"multimodal":36.18314491222481,"knowledge":41.50821266552627,"multilingual":29.73879968453728,"instruction":57.71015958203338,"math":41.08636685146925},"capabilityCounts":{"knowledge":4,"multimodal":3,"math":1,"instruction":1,"coding":1,"reasoning":3,"multilingual":1,"agentic":2},"benchmarks":[{"benchmarkId":"mmmlu","benchmarkName":"MMMLU","category":"knowledge","score":83.4,"unit":"percent","family":"mmmlu","index":39.93600181721925,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mathVision","benchmarkName":"MathVision","category":"multimodal","score":79.7,"unit":"percent","family":"mathVision","index":51.07701534519599,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gpqa","benchmarkName":"Graduate-Level Google-Proof Q&A","category":"knowledge","score":78.8,"unit":"percent","family":"gpqa-diamond","index":49.48230752152859,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gpqaDiamond","benchmarkName":"GPQA Diamond","category":"knowledge","score":78.8,"unit":"percent","family":"gpqa-diamond","index":49.48230752152859,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aime2026","benchmarkName":"AIME 2026","category":"math","score":77.5,"unit":"percent","family":"aime2026","index":41.08636685146925,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmluPro","benchmarkName":"Massive Multitask Language Understanding Professional","category":"knowledge","score":77.2,"unit":"percent","family":"mmlu-pro","index":41.743487824482415,"aboveFit":false,"run":null,"source":"Yubo Wang, Xueguang Ma, Ge Zhang, Yuansheng Ni, Abhranil Chandra, Shiguang Guo, Weiming Ren, Aaran Arulraj, Xuan He, Ziyan Jiang, Tianle Li, Max Ku, Kai Wang, Alex Zhuang, Rongqi Fan, Xiang Yue, Wenhu Chen","sourceUrl":"https://arxiv.org/abs/2406.01574","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":75.3,"unit":"percent","family":"gpqa-diamond","index":46.27220315930934,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaIfBench","benchmarkName":"Artificial Analysis IFBench","category":"instruction","score":73.5,"unit":"percent","family":"ifbench-family","index":57.71015958203338,"aboveFit":true,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"liveCodeBenchV6","benchmarkName":"LiveCodeBench v6","category":"coding","score":72,"unit":"percent","family":"liveCodeBenchV6","index":43.5331856976486,"aboveFit":false,"run":null,"source":"LiveCodeBench maintainers","sourceUrl":"https://github.com/LiveCodeBench/LiveCodeBench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaMmmuPro","benchmarkName":"Artificial Analysis MMMU-Pro","category":"multimodal","score":69.7,"unit":"percent","family":"mmmu-pro","index":45.8612499066415,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmmu-pro","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmmuPro","benchmarkName":"Massive Multi-discipline Multimodal Understanding Pro","category":"multimodal","score":69.1,"unit":"percent","family":"mmmu-pro","index":45.08040254756998,"aboveFit":false,"run":null,"source":"MMMU-Pro authors","sourceUrl":"https://arxiv.org/abs/2409.02813","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":63.7,"unit":"percent","family":"lcr","index":51.00137893897726,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"medXpertQaMm","benchmarkName":"MedXpertQA Multimodal","category":"multimodal","score":48.7,"unit":"percent","family":"medXpertQaMm","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-portuguese:overall","benchmarkName":"EuroEval Portuguese","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":45.50625,"unit":"percent","family":"euroeval","index":33.96174601869062,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/portuguese","measuredAt":null,"stderr":0.8562928821378816,"costPerTest":null,"latency":null},{"benchmarkId":"extractBenchLlamaIndex","benchmarkName":"ExtractBench: schema-guided enterprise document extraction (LlamaIndex leaderboard)","category":"multimodal","score":45.39,"unit":"percent","family":"extractBenchLlamaIndex","index":12.001593164372693,"aboveFit":false,"run":null,"source":"Boyang Zhang, Adrian Lyjak, Eli Stewart, Zhaoqi Li and Simon Suo (LlamaIndex)","sourceUrl":"https://arxiv.org/abs/2607.29677","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-dutch:overall","benchmarkName":"EuroEval Dutch","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":44.875555555555565,"unit":"percent","family":"euroeval","index":33.1060387762368,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/dutch","measuredAt":null,"stderr":0.7476506413302146,"costPerTest":null,"latency":null},{"benchmarkId":"mrcrv2","benchmarkName":"MRCRv2","category":"reasoning","score":43.4,"unit":"percent","family":"mrcrv2","index":36.754036698896485,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":43.11,"unit":"percent","family":"euroeval","index":30.710586130107504,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.7807845348083672,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-french:overall","benchmarkName":"EuroEval French","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":42.78625,"unit":"percent","family":"euroeval","index":30.271331872860237,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/french","measuredAt":null,"stderr":0.9375799965869578,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":42.00125,"unit":"percent","family":"euroeval","index":29.20626749621433,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":1.0029456615390486,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-italian:overall","benchmarkName":"EuroEval Italian","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":41.5075,"unit":"percent","family":"euroeval","index":28.53636235485266,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/italian","measuredAt":null,"stderr":0.9097887804320297,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":37.315,"unit":"percent","family":"euroeval","index":22.848104521619618,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.9850951730670494,"costPerTest":null,"latency":null},{"benchmarkId":"tau2Bench","benchmarkName":"τ²-Bench Tool-Agent-User Evaluation","category":"agentic","score":36.3,"unit":"percent","family":"tau2-bench","index":26.877144108093162,"aboveFit":false,"run":null,"source":"Victor Barres, Honghua Dong, Soham Ray, Xujie Si, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2506.07982","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-spanish:overall","benchmarkName":"EuroEval Spanish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":32.301249999999996,"unit":"percent","family":"euroeval","index":16.04559940262805,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/spanish","measuredAt":null,"stderr":0.8574918002523406,"costPerTest":null,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":15.7,"unit":"percent","family":"hle","index":40.138340410281245,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hleNoTools","benchmarkName":"Humanity's Last Exam without tools","category":"knowledge","score":5.2,"unit":"percent","family":"hle","index":29.603766587468407,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gdpvalAaNormalized","benchmarkName":"GDPval-AA normalized","category":"agentic","score":0,"unit":"percent","family":"gdpvalAaNormalized","index":34.03377491719279,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":0,"unit":"percent","family":"critpt","index":36.8491160127729,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":100,"benchmarkCount":16,"categoryCount":8,"confidence":"medium","state":"available"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"deepseek-r1","providerId":"deepseek","slug":"deepseek-r1","name":"DeepSeek-R1","provider":"DeepSeek","description":"DeepSeek-R1 is a reasoning model from DeepSeek. 21 benchmarks count toward its index, in 6 categories.","logoUrl":null,"releaseDate":"2025-01-20","contextWindowTokens":64000,"modalities":["text"],"openWeights":true,"details":{"apiId":"deepseek/deepseek-r1","maxOutputTokens":16000,"knowledgeCutoff":"2024-07-31","retiresAt":null,"tools":true,"structuredOutput":false,"cacheReadPerMillion":null,"weightsUrl":"https://huggingface.co/deepseek-ai/DeepSeek-R1"},"reasoning":true,"overallScore":38.86551959080503,"overallScoreRange":{"low":33.83693630568903,"high":43.89410287592103},"badges":[],"capabilityScores":{"agentic":27.58919873920363,"coding":42.56239285086471,"reasoning":39.777166797785284,"multimodal":null,"knowledge":43.451824233121094,"multilingual":null,"instruction":28.48595278065974,"math":43.773589691800936},"capabilityCounts":{"math":5,"knowledge":6,"instruction":2,"coding":2,"reasoning":4,"agentic":2},"benchmarks":[{"benchmarkId":"aaMath500","benchmarkName":"Artificial Analysis MATH-500","category":"math","score":98.3,"unit":"percent","family":"math-500","index":48.00611553823449,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/math-500","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"math500","benchmarkName":"MATH-500 Problem Set","category":"math","score":97.3,"unit":"percent","family":"math-500","index":46.949357325804606,"aboveFit":false,"run":null,"source":"Dan Hendrycks, Collin Burns, Saurav Kadavath, Akul Arora, Steven Basart, Eric Tang, Dawn Song, Jacob Steinhardt","sourceUrl":"https://arxiv.org/abs/2103.03874","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-math-level-5:overall","benchmarkName":"MATH level 5","category":"math","domain":null,"eligible":true,"ranked":true,"score":93.05135951661632,"unit":"percent","family":"epoch-ai:epoch-math-level-5:overall","index":44.19810942278887,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0.6990872160031428,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:mgsm:overall","benchmarkName":"MGSM","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":92.254,"unit":"percent","family":"vals-ai:mgsm:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mgsm","measuredAt":"2026-01-09","stderr":1.655,"costPerTest":0.004123,"latency":10.167},{"benchmarkId":"vals-ai:math500:overall","benchmarkName":"MATH 500","category":"math","domain":null,"eligible":true,"ranked":true,"score":92.2,"unit":"percent","family":"math-500","index":41.559890442412225,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/math500","measuredAt":"2026-01-09","stderr":2.364,"costPerTest":0.010886,"latency":156.474},{"benchmarkId":"mmlu","benchmarkName":"Massive Multitask Language Understanding","category":"knowledge","score":90.8,"unit":"percent","family":"mmlu","index":39.94677872098683,"aboveFit":false,"run":null,"source":"Dan Hendrycks, Collin Burns, Steven Basart, Andy Zou, Mantas Mazeika, Dawn Song, Jacob Steinhardt","sourceUrl":"https://arxiv.org/abs/2009.03300","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:medqa:overall","benchmarkName":"MedQA","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":90.8,"unit":"percent","family":"vals-ai:medqa:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medqa","measuredAt":"2026-04-16","stderr":0.518,"costPerTest":0.005903,"latency":41.569},{"benchmarkId":"aaGlobalMmluLite","benchmarkName":"Artificial Analysis Global-MMLU-Lite","category":"multilingual","score":86,"unit":"percent","family":"aaGlobalMmluLite","index":null,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/global-mmlu-lite","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaMmluPro","benchmarkName":"Artificial Analysis MMLU-Pro","category":"knowledge","score":84.9,"unit":"percent","family":"mmlu-pro","index":51.44456615817433,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmlu-pro","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmluPro","benchmarkName":"Massive Multitask Language Understanding Professional","category":"knowledge","score":84,"unit":"percent","family":"mmlu-pro","index":50.31067388540514,"aboveFit":false,"run":null,"source":"Yubo Wang, Xueguang Ma, Ge Zhang, Yuansheng Ni, Abhranil Chandra, Shiguang Guo, Weiming Ren, Aaran Arulraj, Xuan He, Ziyan Jiang, Tianle Li, Max Ku, Kai Wang, Alex Zhuang, Rongqi Fan, Xiang Yue, Wenhu Chen","sourceUrl":"https://arxiv.org/abs/2406.01574","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"ifeval","benchmarkName":"Instruction-Following Eval","category":"instruction","score":83.3,"unit":"percent","family":"ifeval","index":30.94165582200626,"aboveFit":false,"run":null,"source":"Jeffrey Zhou, Tianjian Lu, Swaroop Mishra, Siddhartha Brahma, Sujoy Basu, Yi Luan, Denny Zhou, Le Hou","sourceUrl":"https://arxiv.org/abs/2311.07911","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:mmlu_pro:overall","benchmarkName":"MMLU Pro","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":83.184,"unit":"percent","family":"mmlu-pro","index":49.28261155809441,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","measuredAt":"2026-09-01","stderr":0.444,"costPerTest":0.012339,"latency":27.278},{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":81.3,"unit":"percent","family":"gpqa-diamond","index":51.77523920882805,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aime2024","benchmarkName":"American Invitational Mathematics Examination 2024","category":"math","score":79.8,"unit":"percent","family":"aime2024","index":39.946778720986835,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:aime:overall","benchmarkName":"AIME","category":"math","domain":null,"eligible":true,"ranked":true,"score":73.958,"unit":"percent","family":"vals-ai:aime:overall","index":47.825596340428355,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/aime","measuredAt":"2026-04-16","stderr":1.343,"costPerTest":0.053393,"latency":153.912},{"benchmarkId":"vals-ai:tax_eval_v2:overall","benchmarkName":"TaxEval v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":72.281,"unit":"percent","family":"vals-ai:tax_eval_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","measuredAt":"2026-09-01","stderr":0.874,"costPerTest":0.012239,"latency":152.958},{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":71.71717171717171,"unit":"percent","family":"gpqa-diamond","index":42.98613095936946,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":3.070485693321271,"costPerTest":null,"latency":null},{"benchmarkId":"gpqa","benchmarkName":"Graduate-Level Google-Proof Q&A","category":"knowledge","score":71.5,"unit":"percent","family":"gpqa-diamond","index":42.78694699461416,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:lcb:overall","benchmarkName":"LiveCodeBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":70.221,"unit":"percent","family":"livecodebench","index":47.609025345300125,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","measuredAt":"2026-09-01","stderr":1.15,"costPerTest":0.040788,"latency":86.074},{"benchmarkId":"vals-ai:legal_bench:overall","benchmarkName":"LegalBench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":67.323,"unit":"percent","family":"vals-ai:legal_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","measuredAt":"2026-10-13","stderr":0.535,"costPerTest":null,"latency":18.549},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":55.7,"unit":"percent","family":"lcr","index":45.51273473429198,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:corp_fin_v2:overall","benchmarkName":"CorpFin v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":54.118,"unit":"percent","family":"vals-ai:corp_fin_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","measuredAt":"2026-08-12","stderr":0.979,"costPerTest":0.086732,"latency":46.774},{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":53.333333333333336,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":39.948106648995996,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":7.521014330903548,"costPerTest":null,"latency":null},{"benchmarkId":"sweVerified","benchmarkName":"Software Engineering Benchmark Verified","category":"coding","score":49.2,"unit":"percent","family":"swe-bench-verified","index":37.5157603564293,"aboveFit":false,"run":null,"source":"Carlos E. Jimenez, John Yang, Alexander Wettig, Shunyu Yao, Kexin Pei, Ofir Press, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2310.06770","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaIfBench","benchmarkName":"Artificial Analysis IFBench","category":"instruction","score":39.6,"unit":"percent","family":"ifbench-family","index":26.030249739313216,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"tau2Bench","benchmarkName":"τ²-Bench Tool-Agent-User Evaluation","category":"agentic","score":36.5,"unit":"percent","family":"tau2-bench","index":27.009858730477486,"aboveFit":false,"run":null,"source":"Victor Barres, Honghua Dong, Soham Ray, Xujie Si, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2506.07982","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"omniscienceAccuracy","benchmarkName":"Artificial Analysis Omniscience Accuracy","category":"knowledge","score":30.5,"unit":"percent","family":"omniscienceAccuracy","index":49.12775798044044,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"simpleQa","benchmarkName":"Measuring Short-Form Factuality in Large Language Models","category":"knowledge","score":30.1,"unit":"percent","family":"simpleQa","index":38.100934167740476,"aboveFit":false,"run":null,"source":"Jason Wei, Najoung Kim, Hyung Won Chung, Yu-An Chung, Siddhartha Papay, Yifeng Lu, Hannaneh Hajishirzi, Luke Zettlemoyer","sourceUrl":"https://arxiv.org/abs/2411.04368","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":15.8,"unit":"percent","family":"hle","index":40.23866968478423,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"arc-prize:v1-semi-private:overall","benchmarkName":"ARC-AGI-1 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":15.8,"unit":"percent","family":"arc-prize:v1-semi-private:overall","index":34.25118928718628,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.06,"latency":null},{"benchmarkId":"vals-ai:terminal-bench:overall","benchmarkName":"Terminal-Bench 1.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":13.75,"unit":"percent","family":"vals-ai:terminal-bench:overall","index":28.168538747929773,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench","measuredAt":"2026-01-12","stderr":3.85,"costPerTest":0.359797,"latency":145.869},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":1.4,"unit":"percent","family":"critpt","index":39.55935265615135,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"arc-prize:v2-semi-private:overall","benchmarkName":"ARC-AGI-2 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":1.3,"unit":"percent","family":"arc-prize:v2-semi-private:overall","index":39.78539051351155,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.08,"latency":null},{"benchmarkId":"arc-prize:v2-public-eval:overall","benchmarkName":"ARC-AGI-2 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":0.3,"unit":"percent","family":"arc-prize:v2-public-eval:overall","index":null,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.08,"latency":null}],"price":{"inputPerMillion":0.7,"outputPerMillion":2.5,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":18,"timeToFirstToken":1.332,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":{"elo":1372.6822896691267,"eloUpper":1377.531410989047,"eloLower":1367.8331683492065,"votes":18524,"rank":186,"rankUpper":null,"rankLower":null,"arenaName":"DeepSeek-R1","organization":"deepseek","updatedAt":"2026-11-09"},"quality":{"coveragePercent":85,"benchmarkCount":21,"categoryCount":6,"confidence":"medium","state":"available"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:arc-prize-foundation","name":"ARC Prize Foundation","url":"https://arcprize.org","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gpt-4-5","providerId":"openai","slug":"gpt-4-5","name":"GPT-4.5","provider":"OpenAI","description":"GPT-4.5 is a standard model from OpenAI. 9 benchmarks count toward its index, in 5 categories.","logoUrl":null,"releaseDate":"2025-02-27","contextWindowTokens":128000,"modalities":["text"],"openWeights":false,"details":null,"reasoning":false,"overallScore":38.812122703488185,"overallScoreRange":{"low":29.922122703488185,"high":47.702122703488186},"badges":[],"capabilityScores":{"agentic":null,"coding":29.12105210203614,"reasoning":35.63923194192528,"multimodal":null,"knowledge":48.77597686186021,"multilingual":31.77076642263611,"instruction":null,"math":34.04429409543347},"capabilityCounts":{"knowledge":3,"coding":1,"multilingual":1,"math":2,"reasoning":2},"benchmarks":[{"benchmarkId":"mmlu","benchmarkName":"Massive Multitask Language Understanding","category":"knowledge","score":90.8,"unit":"percent","family":"mmlu","index":39.94677872098683,"aboveFit":false,"run":null,"source":"Dan Hendrycks, Collin Burns, Steven Basart, Andy Zou, Mantas Mazeika, Dawn Song, Jacob Steinhardt","sourceUrl":"https://arxiv.org/abs/2009.03300","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"humaneval","benchmarkName":"Evaluating Large Language Models Trained on Code","category":"coding","score":88.6,"unit":"percent","family":"humaneval","index":29.12105210203614,"aboveFit":false,"run":null,"source":"Mark Chen, Jerry Tworek, Heewoo Jun, Qiming Yuan, Henrique Ponde de Oliveira Pinto, Jared Kaplan, Harri Edwards, Yuri Burda, Nicholas Joseph, Greg Brockman, Alex Ray, Raul Puri, Gretchen Krueger, Michael Petrov, Heidy Khlaaf, Girish Sastry, Pamela Mishkin, Brooke Chan, Scott Gray, Nick Ryder, Mikhail Pavlov, Alethea Power, Lukasz Kaiser, Mohammad Bavarian, Clemens Winter, Philippe Tillet, Felipe Petroski Such, Dave Cummings, Matthias Plappert, Fotios Chantzis, Elizabeth Barnes, Ariel Herbert-Voss, William Hebgen Guss, Alex Nichol, Alex Paino, Nikolas Tezak, Jie Tang, Igor Babuschkin, Suchir Balaji, Shantanu Jain, William Saunders, Christopher Hesse, Andrew N. Carr, Jan Leike, Josh Achiam, Vedant Misra, Evan Morikawa, Alec Radford, Matthew Knight, Miles Brundage, Mira Murati, Katie Mayer, Peter Welinder, Bob McGrew, Dario Amodei, Sam McCandlish, Ilya Sutskever, Wojciech Zaremba","sourceUrl":"https://arxiv.org/abs/2107.03374","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mgsm","benchmarkName":"Multilingual Grade School Math","category":"multilingual","score":86.9,"unit":"percent","family":"mgsm","index":31.77076642263611,"aboveFit":false,"run":null,"source":"Freda Shi, Mirac Suzgun, Markus Freitag, Xuezhi Wang, Suraj Srivats, Soroush Vosoughi, Hyung Won Chung, Yi Tay, Sebastian Ruder, Denny Zhou, Dipanjan Das, Jason Wei","sourceUrl":"https://arxiv.org/abs/2210.03057","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"drop","benchmarkName":"Discrete Reasoning Over Paragraphs","category":"reasoning","score":83.4,"unit":"percent","family":"drop","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-math-level-5:overall","benchmarkName":"MATH level 5","category":"math","domain":null,"eligible":true,"ranked":true,"score":78.62537764350454,"unit":"percent","family":"epoch-ai:epoch-math-level-5:overall","index":37.190611142163405,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.1270694428136296,"costPerTest":null,"latency":null},{"benchmarkId":"gpqa","benchmarkName":"Graduate-Level Google-Proof Q&A","category":"knowledge","score":69.5,"unit":"percent","family":"gpqa-diamond","index":40.95260164477459,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":68.68686868686868,"unit":"percent","family":"gpqa-diamond","index":40.2068198232489,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":3.3042050878136546,"costPerTest":null,"latency":null},{"benchmarkId":"simpleQa","benchmarkName":"Measuring Short-Form Factuality in Large Language Models","category":"knowledge","score":62.5,"unit":"percent","family":"simpleQa","index":65.80144113058206,"aboveFit":true,"run":null,"source":"Jason Wei, Najoung Kim, Hyung Won Chung, Yu-An Chung, Siddhartha Papay, Yifeng Lu, Hannaneh Hajishirzi, Luke Zettlemoyer","sourceUrl":"https://arxiv.org/abs/2411.04368","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":37.77777777777778,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":30.897977048703545,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":7.309112127323451,"costPerTest":null,"latency":null},{"benchmarkId":"arc-prize:v1-semi-private:overall","benchmarkName":"ARC-AGI-1 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":10.299999999999999,"unit":"percent","family":"arc-prize:v1-semi-private:overall","index":31.722640720827094,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.29,"latency":null},{"benchmarkId":"arc-prize:v2-semi-private:overall","benchmarkName":"ARC-AGI-2 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":0.8,"unit":"percent","family":"arc-prize:v2-semi-private:overall","index":39.55582316302347,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":2.1,"latency":null},{"benchmarkId":"arc-prize:v2-public-eval:overall","benchmarkName":"ARC-AGI-2 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":0,"unit":"percent","family":"arc-prize:v2-public-eval:overall","index":null,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":2.07,"latency":null}],"price":{"inputPerMillion":75,"outputPerMillion":150,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":65,"benchmarkCount":9,"categoryCount":5,"confidence":"low","state":"available"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:arc-prize-foundation","name":"ARC Prize Foundation","url":"https://arcprize.org","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"nemotron-3-nano-omni-30b-a3b","providerId":"nvidia","slug":"nemotron-3-nano-omni-30b-a3b","name":"Nemotron 3 Nano Omni 30B A3B","provider":"NVIDIA","description":"Nemotron 3 Nano Omni 30B A3B is a reasoning model from NVIDIA in the Nemotron 3 Nano Omni family. 16 benchmarks count toward its index, in 7 categories.","logoUrl":null,"releaseDate":"2026-04-28","contextWindowTokens":256000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":true,"overallScore":38.56967220531448,"overallScoreRange":{"low":34.027172205314486,"high":43.11217220531448},"badges":[],"capabilityScores":{"agentic":33.4415385162902,"coding":37.557915127782735,"reasoning":35.69228116884716,"multimodal":40.756858221560044,"knowledge":38.1669644037682,"multilingual":null,"instruction":53.224508630851766,"math":43.556801280906555},"capabilityCounts":{"multimodal":6,"math":1,"knowledge":3,"instruction":1,"agentic":2,"reasoning":2,"coding":1},"benchmarks":[{"benchmarkId":"refcocoAvg","benchmarkName":"RefCOCO average","category":"multimodal","score":90.5,"unit":"percent","family":"refcocoAvg","index":47.981799469113234,"aboveFit":false,"run":null,"source":"RefCOCO dataset authors","sourceUrl":"https://github.com/lichengunc/refer","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"ai2dTest","benchmarkName":"AI2D test split","category":"multimodal","score":88.5,"unit":"percent","family":"ai2dTest","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aime2025","benchmarkName":"American Invitational Mathematics Examination 2025","category":"math","score":82.1,"unit":"percent","family":"aime2025","index":43.556801280906555,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmluPro","benchmarkName":"Massive Multitask Language Understanding Professional","category":"knowledge","score":77.3,"unit":"percent","family":"mmlu-pro","index":41.86947585479009,"aboveFit":false,"run":null,"source":"Yubo Wang, Xueguang Ma, Ge Zhang, Yuansheng Ni, Abhranil Chandra, Shiguang Guo, Weiming Ren, Aaran Arulraj, Xuan He, Ziyan Jiang, Tianle Li, Max Ku, Kai Wang, Alex Zhuang, Rongqi Fan, Xiang Yue, Wenhu Chen","sourceUrl":"https://arxiv.org/abs/2406.01574","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"charxiv","benchmarkName":"CharXiv Reasoning","category":"multimodal","score":76.25,"unit":"percent","family":"charxiv","index":50.99470598085682,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"ifBench","benchmarkName":"Instruction Following Benchmark","category":"instruction","score":74.2,"unit":"percent","family":"ifbench-family","index":58.364317012414034,"aboveFit":true,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"videoMmeNoSub","benchmarkName":"Video-MME without subtitle","category":"multimodal","score":72.2,"unit":"percent","family":"videoMmeNoSub","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gpqa","benchmarkName":"Graduate-Level Google-Proof Q&A","category":"knowledge","score":72.2,"unit":"percent","family":"gpqa-diamond","index":43.42896786705801,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gpqaDiamond","benchmarkName":"GPQA Diamond","category":"knowledge","score":72.2,"unit":"percent","family":"gpqa-diamond","index":43.42896786705801,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmmu","benchmarkName":"Massive Multi-discipline Multimodal Understanding","category":"multimodal","score":70.8,"unit":"percent","family":"mmmu","index":38.65292372443727,"aboveFit":false,"run":null,"source":"MMMU authors","sourceUrl":"https://arxiv.org/abs/2401.05508","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"liveCodeBenchV5","benchmarkName":"LiveCodeBench v5","category":"coding","score":63.2,"unit":"percent","family":"liveCodeBenchV5","index":null,"aboveFit":false,"run":null,"source":"LiveCodeBench maintainers","sourceUrl":"https://github.com/LiveCodeBench/LiveCodeBench","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaIfBench","benchmarkName":"Artificial Analysis IFBench","category":"instruction","score":63.2,"unit":"percent","family":"ifbench-family","index":48.0847002492895,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"screenSpotPro","benchmarkName":"ScreenSpot Pro","category":"multimodal","score":57.8,"unit":"percent","family":"screenSpotPro","index":41.12303694764893,"aboveFit":false,"run":null,"source":"Kaixin Li, Ziyang Meng, Hongzhan Lin, Ziyang Luo, Yuchen Tian, Jing Ma, Zhiyong Huang, Tat-Seng Chua","sourceUrl":"https://arxiv.org/abs/2504.07981","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmLongBenchDoc","benchmarkName":"MMLongBench-Doc","category":"multimodal","score":57.5,"unit":"percent","family":"mmLongBenchDoc","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaMmmuPro","benchmarkName":"Artificial Analysis MMMU-Pro","category":"multimodal","score":53.2,"unit":"percent","family":"mmmu-pro","index":24.38794753217527,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmmu-pro","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"parseBenchLlamaIndex","benchmarkName":"ParseBench: document parsing for AI agents (LlamaIndex leaderboard)","category":"multimodal","score":48.51,"unit":"percent","family":"parseBenchLlamaIndex","index":41.40073567512872,"aboveFit":false,"run":null,"source":"Boyang Zhang, Sebastián G. Acosta, Preston Carlson, Sacha Bron, Pierre-Loïc Doulcet, Daniel B. Ospina and Simon Suo (LlamaIndex)","sourceUrl":"https://arxiv.org/abs/2604.08538","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"osWorld","benchmarkName":"OSWorld","category":"agentic","score":47.4,"unit":"percent","family":"osWorld","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":46.9,"unit":"percent","family":"gpqa-diamond","index":20.224499191587448,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"tau2Bench","benchmarkName":"τ²-Bench Tool-Agent-User Evaluation","category":"agentic","score":45.3,"unit":"percent","family":"tau2-bench","index":32.849302115387616,"aboveFit":false,"run":null,"source":"Victor Barres, Honghua Dong, Soham Ray, Xujie Si, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2506.07982","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":39.7,"unit":"percent","family":"lcr","index":34.535446324921416,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"sciCode","benchmarkName":"Scientific Code Benchmark","category":"coding","score":32,"unit":"percent","family":"scicode-family","index":37.557915127782735,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":4.8,"unit":"percent","family":"hle","index":29.202449489456487,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gdpvalAaNormalized","benchmarkName":"GDPval-AA normalized","category":"agentic","score":0,"unit":"percent","family":"gdpvalAaNormalized","index":34.03377491719279,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":0,"unit":"percent","family":"critpt","index":36.8491160127729,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":228,"timeToFirstToken":9.53,"source":"BenchLM","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":95,"benchmarkCount":16,"categoryCount":7,"confidence":"medium","state":"available"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"zaya1-8b","providerId":"zyphra","slug":"zaya1-8b","name":"ZAYA1-8B","provider":"Zyphra","description":"ZAYA1-8B is a reasoning model from Zyphra in the ZAYA1 family. 9 benchmarks count toward its index, in 5 categories.","logoUrl":null,"releaseDate":"2026-05-05","contextWindowTokens":131000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":true,"overallScore":38.20921005565147,"overallScoreRange":{"low":29.819210055651467,"high":46.59921005565147},"badges":[],"capabilityScores":{"agentic":21.557892092619316,"coding":38.44263138591541,"reasoning":null,"multimodal":null,"knowledge":40.14610378620303,"multilingual":null,"instruction":36.837426290290296,"math":43.661457793769266},"capabilityCounts":{"math":3,"instruction":2,"knowledge":2,"coding":1,"agentic":1},"benchmarks":[{"benchmarkId":"aime2026","benchmarkName":"AIME 2026","category":"math","score":89.1,"unit":"percent","family":"aime2026","index":49.43767683534147,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"ifeval","benchmarkName":"Instruction-Following Eval","category":"instruction","score":85.58,"unit":"percent","family":"ifeval","index":35.533345273077,"aboveFit":false,"run":null,"source":"Jeffrey Zhou, Tianjian Lu, Swaroop Mishra, Siddhartha Brahma, Sujoy Basu, Yi Luan, Denny Zhou, Le Hou","sourceUrl":"https://arxiv.org/abs/2311.07911","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmluPro","benchmarkName":"Massive Multitask Language Understanding Professional","category":"knowledge","score":74.2,"unit":"percent","family":"mmlu-pro","index":37.96384691525179,"aboveFit":false,"run":null,"source":"Yubo Wang, Xueguang Ma, Ge Zhang, Yuansheng Ni, Abhranil Chandra, Shiguang Guo, Weiming Ren, Aaran Arulraj, Xuan He, Ziyan Jiang, Tianle Li, Max Ku, Kai Wang, Alex Zhuang, Rongqi Fan, Xiang Yue, Wenhu Chen","sourceUrl":"https://arxiv.org/abs/2406.01574","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hmmtFeb2026","benchmarkName":"Harvard-MIT Mathematics Tournament February 2026","category":"math","score":71.6,"unit":"percent","family":"hmmtFeb2026","index":43.50772056339473,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gpqa","benchmarkName":"Graduate-Level Google-Proof Q&A","category":"knowledge","score":71,"unit":"percent","family":"gpqa-diamond","index":42.32836065715426,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gpqaDiamond","benchmarkName":"GPQA Diamond","category":"knowledge","score":71,"unit":"percent","family":"gpqa-diamond","index":42.32836065715426,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"liveCodeBenchV6","benchmarkName":"LiveCodeBench v6","category":"coding","score":65.8,"unit":"percent","family":"liveCodeBenchV6","index":38.44263138591541,"aboveFit":false,"run":null,"source":"LiveCodeBench maintainers","sourceUrl":"https://github.com/LiveCodeBench/LiveCodeBench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"imoAnswerBench","benchmarkName":"IMOAnswerBench","category":"math","score":59.3,"unit":"percent","family":"imoAnswerBench","index":38.0389759825716,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"ifBench","benchmarkName":"Instruction Following Benchmark","category":"instruction","score":52.56,"unit":"percent","family":"ifbench-family","index":38.14150730750358,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"bfclV4","benchmarkName":"Berkeley Function Calling Leaderboard v4","category":"agentic","score":39.22,"unit":"percent","family":"bfclV4","index":21.557892092619316,"aboveFit":false,"run":null,"source":"Shishir G. Patil et al.","sourceUrl":"https://gorilla.cs.berkeley.edu/leaderboard.html","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"apex","benchmarkName":"Apex","category":"math","score":32.2,"unit":"percent","family":"apex","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":70,"benchmarkCount":9,"categoryCount":5,"confidence":"low","state":"available"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"qwen3-235b-a22b","providerId":"alibaba","slug":"qwen3-235b-a22b","name":"Qwen3 235B-A22B","provider":"Alibaba","description":"Qwen3 235B-A22B is a standard model from Alibaba. 14 benchmarks count toward its index, in 5 categories.","logoUrl":null,"releaseDate":"2025-04-29","contextWindowTokens":131072,"modalities":["text"],"openWeights":true,"details":{"apiId":"qwen/qwen3-235b-a22b","maxOutputTokens":8192,"knowledgeCutoff":"2025-03-31","retiresAt":"2026-10-09","tools":true,"structuredOutput":false,"cacheReadPerMillion":null,"weightsUrl":"https://huggingface.co/Qwen/Qwen3-235B-A22B"},"reasoning":false,"overallScore":38.122169996710056,"overallScoreRange":{"low":29.300555714986153,"high":46.94378427843396},"badges":[],"capabilityScores":{"agentic":null,"coding":44.04023933789553,"reasoning":null,"multimodal":null,"knowledge":41.14712539905987,"multilingual":41.998953455547365,"instruction":30.74026593380142,"math":32.17324500014879},"capabilityCounts":{"math":5,"knowledge":5,"instruction":1,"multilingual":1,"coding":2},"benchmarks":[{"benchmarkId":"vals-ai:math500:overall","benchmarkName":"MATH 500","category":"math","domain":null,"eligible":true,"ranked":true,"score":94.6,"unit":"percent","family":"math-500","index":44.096110152243924,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/math500","measuredAt":"2026-01-09","stderr":1.011,"costPerTest":0.003361,"latency":142.754},{"benchmarkId":"vals-ai:mgsm:overall","benchmarkName":"MGSM","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":92.473,"unit":"percent","family":"vals-ai:mgsm:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mgsm","measuredAt":"2026-01-09","stderr":1.625,"costPerTest":0.001108,"latency":31.899},{"benchmarkId":"math500","benchmarkName":"MATH-500 Problem Set","category":"math","score":91.2,"unit":"percent","family":"math-500","index":40.50313222998234,"aboveFit":false,"run":null,"source":"Dan Hendrycks, Collin Burns, Saurav Kadavath, Akul Arora, Steven Basart, Eric Tang, Dawn Song, Jacob Steinhardt","sourceUrl":"https://arxiv.org/abs/2103.03874","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:medqa:overall","benchmarkName":"MedQA","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":90.617,"unit":"percent","family":"vals-ai:medqa:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medqa","measuredAt":"2026-04-16","stderr":0.266,"costPerTest":0.001177,"latency":26.257},{"benchmarkId":"mmluRedux","benchmarkName":"MMLU-Redux","category":"knowledge","score":89.2,"unit":"percent","family":"mmluRedux","index":41.47859942649845,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"cEval","benchmarkName":"C-Eval","category":"knowledge","score":86.1,"unit":"percent","family":"cEval","index":40.56986757326472,"aboveFit":false,"run":null,"source":"C-Eval authors","sourceUrl":"https://arxiv.org/abs/2305.08322","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:aime:overall","benchmarkName":"AIME","category":"math","domain":null,"eligible":true,"ranked":true,"score":83.958,"unit":"percent","family":"vals-ai:aime:overall","index":52.19605394477574,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/aime","measuredAt":"2026-04-16","stderr":0.768,"costPerTest":0.011598,"latency":242.174},{"benchmarkId":"ifeval","benchmarkName":"Instruction-Following Eval","category":"instruction","score":83.2,"unit":"percent","family":"ifeval","index":30.74026593380142,"aboveFit":false,"run":null,"source":"Jeffrey Zhou, Tianjian Lu, Swaroop Mishra, Siddhartha Brahma, Sujoy Basu, Yi Luan, Denny Zhou, Le Hou","sourceUrl":"https://arxiv.org/abs/2311.07911","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:mmlu_pro:overall","benchmarkName":"MMLU Pro","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":81.246,"unit":"percent","family":"mmlu-pro","index":46.84096353073143,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","measuredAt":"2026-09-01","stderr":0.387,"costPerTest":0.002316,"latency":52.354},{"benchmarkId":"vals-ai:legal_bench:overall","benchmarkName":"LegalBench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":80.179,"unit":"percent","family":"vals-ai:legal_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","measuredAt":"2026-10-13","stderr":0.441,"costPerTest":null,"latency":12.366},{"benchmarkId":"mmmlu","benchmarkName":"MMMLU","category":"knowledge","score":79.8,"unit":"percent","family":"mmmlu","index":35.249739602236446,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"include","benchmarkName":"INCLUDE","category":"multilingual","score":75.6,"unit":"percent","family":"include","index":41.998953455547365,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":70.70707070707071,"unit":"percent","family":"gpqa-diamond","index":42.05969391399595,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.7124209393506975,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:tax_eval_v2:overall","benchmarkName":"TaxEval v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":70.646,"unit":"percent","family":"vals-ai:tax_eval_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","measuredAt":"2026-09-01","stderr":0.89,"costPerTest":0.004644,"latency":92.742},{"benchmarkId":"vals-ai:lcb:overall","benchmarkName":"LiveCodeBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":70.62,"unit":"percent","family":"livecodebench","index":47.94009209567222,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","measuredAt":"2026-09-01","stderr":1.145,"costPerTest":0.011344,"latency":429.483},{"benchmarkId":"vals-ai:gpqa:overall","benchmarkName":"GPQA Diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":70.202,"unit":"percent","family":"gpqa-diamond","index":41.596456862568274,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","measuredAt":"2026-09-01","stderr":2.373,"costPerTest":0.006472,"latency":306.218},{"benchmarkId":"epoch-ai:epoch-math-level-5:overall","benchmarkName":"MATH level 5","category":"math","domain":null,"eligible":true,"ranked":true,"score":68.85699899295065,"unit":"percent","family":"epoch-ai:epoch-math-level-5:overall","index":32.44556867639956,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0.9487067376018313,"costPerTest":null,"latency":null},{"benchmarkId":"bfclV3","benchmarkName":"Berkeley Function Calling Leaderboard v3","category":"agentic","score":68,"unit":"percent","family":"bfclV3","index":null,"aboveFit":false,"run":null,"source":"Shishir G. Patil et al.","sourceUrl":"https://gorilla.cs.berkeley.edu/leaderboard.html","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gpqa","benchmarkName":"Graduate-Level Google-Proof Q&A","category":"knowledge","score":62.9,"unit":"percent","family":"gpqa-diamond","index":34.899261990304005,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aime2024","benchmarkName":"American Invitational Mathematics Examination 2024","category":"math","score":40.1,"unit":"percent","family":"aime2024","index":21.514901765433162,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"liveCodeBenchV5","benchmarkName":"LiveCodeBench v5","category":"coding","score":35.3,"unit":"percent","family":"liveCodeBenchV5","index":null,"aboveFit":false,"run":null,"source":"LiveCodeBench maintainers","sourceUrl":"https://github.com/LiveCodeBench/LiveCodeBench","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aime2025","benchmarkName":"American Invitational Mathematics Examination 2025","category":"math","score":24.7,"unit":"percent","family":"aime2025","index":12.410079423022346,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:ioi-v1:overall","benchmarkName":"IOI v1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":0,"unit":"percent","family":"vals-ai:ioi-v1:overall","index":40.140386580118836,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/ioi-v1","measuredAt":"2026-08-09","stderr":0,"costPerTest":0.109574,"latency":352.565},{"benchmarkId":"epoch-ai:epoch-frontiermath-2025-02-28-public:overall","benchmarkName":"FrontierMath-2025-02-28-Public","category":"math","domain":null,"eligible":false,"ranked":true,"score":0,"unit":"percent","family":"epoch-ai:epoch-frontiermath-2025-02-28-public:overall","index":null,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":0.45499999999999996,"outputPerMillion":1.8199999999999998,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":44,"timeToFirstToken":0.663,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":{"elo":1366.243669791999,"eloUpper":1370.9458945177098,"eloLower":1361.5414450662884,"votes":25927,"rank":194,"rankUpper":null,"rankLower":null,"arenaName":"Qwen3-235B-A22B","organization":"alibaba","updatedAt":"2026-11-09"},"quality":{"coveragePercent":55.00000000000001,"benchmarkCount":14,"categoryCount":5,"confidence":"low","state":"available"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gpt-4-1","providerId":"openai","slug":"gpt-4-1","name":"GPT-4.1","provider":"OpenAI","description":"GPT-4.1 is a standard model from OpenAI. 29 benchmarks count toward its index, in 8 categories.","logoUrl":null,"releaseDate":"2025-04-14","contextWindowTokens":1047576,"modalities":["image","text","file"],"openWeights":false,"details":{"apiId":"openai/gpt-4.1","maxOutputTokens":32768,"knowledgeCutoff":"2024-06-30","retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.5,"weightsUrl":null},"reasoning":false,"overallScore":38.047931413572336,"overallScoreRange":{"low":35.045237795249406,"high":41.050625031895265},"badges":[],"capabilityScores":{"agentic":37.82528352732705,"coding":34.213864783049225,"reasoning":40.44751352888734,"multimodal":42.07804445194009,"knowledge":41.23900708458065,"multilingual":45.053127534794804,"instruction":34.203113534069466,"math":35.94847895015652},"capabilityCounts":{"knowledge":6,"instruction":2,"math":7,"agentic":4,"multilingual":2,"multimodal":1,"reasoning":5,"coding":2},"benchmarks":[{"benchmarkId":"vals-ai:medqa:overall","benchmarkName":"MedQA","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":91.183,"unit":"percent","family":"vals-ai:medqa:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medqa","measuredAt":"2026-04-16","stderr":0.259,"costPerTest":0.002952,"latency":3.083},{"benchmarkId":"mmlu","benchmarkName":"Massive Multitask Language Understanding","category":"knowledge","score":90.2,"unit":"percent","family":"mmlu","index":39.23038566920084,"aboveFit":false,"run":null,"source":"Dan Hendrycks, Collin Burns, Steven Basart, Andy Zou, Mantas Mazeika, Dawn Song, Jacob Steinhardt","sourceUrl":"https://arxiv.org/abs/2009.03300","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:mgsm:overall","benchmarkName":"MGSM","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":87.673,"unit":"percent","family":"vals-ai:mgsm:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mgsm","measuredAt":"2026-01-09","stderr":1.921,"costPerTest":0.001684,"latency":2.19},{"benchmarkId":"ifeval","benchmarkName":"Instruction-Following Eval","category":"instruction","score":87.4,"unit":"percent","family":"ifeval","index":39.198641238405415,"aboveFit":false,"run":null,"source":"Jeffrey Zhou, Tianjian Lu, Swaroop Mishra, Siddhartha Brahma, Sujoy Basu, Yi Luan, Denny Zhou, Le Hou","sourceUrl":"https://arxiv.org/abs/2311.07911","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:math500:overall","benchmarkName":"MATH 500","category":"math","domain":null,"eligible":true,"ranked":true,"score":87.2,"unit":"percent","family":"math-500","index":36.27609938026282,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/math500","measuredAt":"2026-01-09","stderr":1.494,"costPerTest":0.009426,"latency":12.53},{"benchmarkId":"vals-ai:legal_bench:overall","benchmarkName":"LegalBench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":83.1,"unit":"percent","family":"vals-ai:legal_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","measuredAt":"2026-10-13","stderr":0.427,"costPerTest":null,"latency":0.509},{"benchmarkId":"epoch-ai:epoch-math-level-5:overall","benchmarkName":"MATH level 5","category":"math","domain":null,"eligible":true,"ranked":true,"score":83.00604229607251,"unit":"percent","family":"epoch-ai:epoch-math-level-5:overall","index":39.31854255722244,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0.8490146044318322,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:mmlu_pro:overall","benchmarkName":"MMLU Pro","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":80.495,"unit":"percent","family":"mmlu-pro","index":45.89479342312071,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","measuredAt":"2026-09-01","stderr":0.385,"costPerTest":0.007141,"latency":8.177},{"benchmarkId":"vals-ai:tax_eval_v2:overall","benchmarkName":"TaxEval v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":75.061,"unit":"percent","family":"vals-ai:tax_eval_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","measuredAt":"2026-09-01","stderr":0.845,"costPerTest":0.006214,"latency":7.89},{"benchmarkId":"tau2-bench:retail:overall","benchmarkName":"τ²-bench Retail","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":74,"unit":"percent","family":"tau2-bench","index":51.89385042753771,"aboveFit":false,"run":"Sierra","source":"Sierra Research","sourceUrl":"https://github.com/sierra-research/tau2-bench","measuredAt":"2026-04-30","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmluProX","benchmarkName":"MMLU-ProX","category":"multilingual","score":72.7,"unit":"percent","family":"mmluProX","index":42.231681640366126,"aboveFit":false,"run":null,"source":"MMLU-ProX authors","sourceUrl":"https://arxiv.org/abs/2503.10497","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:mmmu:overall","benchmarkName":"MMMU Pro","category":"multimodal","domain":null,"eligible":true,"ranked":true,"score":72.386,"unit":"percent","family":"mmmu-pro","index":49.35684325075156,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmmu","measuredAt":"2026-09-01","stderr":1.075,"costPerTest":0.007959,"latency":12},{"benchmarkId":"vals-ai:case_law_v2:overall","benchmarkName":"CaseLaw v2","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":69.882,"unit":"percent","family":"vals-ai:case_law_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/case_law_v2","measuredAt":"2026-05-04","stderr":0.284,"costPerTest":0.046588,"latency":6.656},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":68.3,"unit":"percent","family":"lcr","index":54.15734935667129,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":66.91919191919192,"unit":"percent","family":"gpqa-diamond","index":38.58555499384524,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.821440093855908,"costPerTest":null,"latency":null},{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":66.6,"unit":"percent","family":"gpqa-diamond","index":38.29280088750721,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gpqa","benchmarkName":"Graduate-Level Google-Proof Q&A","category":"knowledge","score":66.3,"unit":"percent","family":"gpqa-diamond","index":38.01764908503127,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:mortgage_tax:overall","benchmarkName":"MortgageTax","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":65.938,"unit":"percent","family":"vals-ai:mortgage_tax:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mortgage_tax","measuredAt":"2026-09-01","stderr":0.922,"costPerTest":0.006212,"latency":5.034},{"benchmarkId":"vals-ai:gpqa:overall","benchmarkName":"GPQA Diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":65.404,"unit":"percent","family":"gpqa-diamond","index":37.195862368303146,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","measuredAt":"2026-09-01","stderr":2.402,"costPerTest":0.013852,"latency":23.14},{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":63.84375,"unit":"percent","family":"euroeval","index":58.84151418014818,"aboveFit":true,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":0.6503208342810494,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:corp_fin_v2:overall","benchmarkName":"CorpFin v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":63.054,"unit":"percent","family":"vals-ai:corp_fin_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","measuredAt":"2026-08-12","stderr":0.951,"costPerTest":0.13202,"latency":10.14},{"benchmarkId":"euroeval:euroeval-dutch:overall","benchmarkName":"EuroEval Dutch","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":62.62555555555556,"unit":"percent","family":"euroeval","index":57.188704617593125,"aboveFit":true,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/dutch","measuredAt":null,"stderr":0.45733122772433976,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-french:overall","benchmarkName":"EuroEval French","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":62.300000000000004,"unit":"percent","family":"euroeval","index":56.747000636903465,"aboveFit":true,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/french","measuredAt":null,"stderr":0.31727600366242636,"costPerTest":null,"latency":null},{"benchmarkId":"aaMmmuPro","benchmarkName":"Artificial Analysis MMMU-Pro","category":"multimodal","score":61.2,"unit":"percent","family":"mmmu-pro","index":34.799245653128594,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmmu-pro","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-italian:overall","benchmarkName":"EuroEval Italian","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":57.0775,"unit":"percent","family":"euroeval","index":49.66126979991847,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/italian","measuredAt":null,"stderr":0.40761501444377635,"costPerTest":null,"latency":null},{"benchmarkId":"tau2-bench:airline:overall","benchmarkName":"τ²-bench Airline","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":56,"unit":"percent","family":"tau2-bench","index":39.9495344129488,"aboveFit":false,"run":"Sierra","source":"Sierra Research","sourceUrl":"https://github.com/sierra-research/tau2-bench","measuredAt":"2026-03-02","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:lcb:overall","benchmarkName":"LiveCodeBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":54.666,"unit":"percent","family":"livecodebench","index":34.702400528162514,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","measuredAt":"2026-09-01","stderr":1.118,"costPerTest":0.00931,"latency":30.565},{"benchmarkId":"sweVerified","benchmarkName":"Software Engineering Benchmark Verified","category":"coding","score":54.6,"unit":"percent","family":"swe-bench-verified","index":41.49857127646123,"aboveFit":false,"run":null,"source":"Carlos E. Jimenez, John Yang, Alexander Wettig, Shunyu Yao, Kexin Pei, Ofir Press, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2310.06770","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-portuguese:overall","benchmarkName":"EuroEval Portuguese","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":54.44375,"unit":"percent","family":"euroeval","index":46.087877058528484,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/portuguese","measuredAt":null,"stderr":0.4952524608722303,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":54.111428571428576,"unit":"percent","family":"euroeval","index":45.63699334493931,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.5152213710867299,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-spanish:overall","benchmarkName":"EuroEval Spanish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":52.768750000000004,"unit":"percent","family":"euroeval","index":43.815287465048385,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/spanish","measuredAt":null,"stderr":0.48716655262856456,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":51.86625,"unit":"percent","family":"euroeval","index":42.590802624382235,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.5947215735115047,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-swe-bench-verified:overall","benchmarkName":"SWE-Bench verified","category":"coding","domain":null,"eligible":true,"ranked":true,"score":48.541666666666664,"unit":"percent","family":"swe-bench-verified","index":37.030201617721694,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.2835863746433964,"costPerTest":null,"latency":null},{"benchmarkId":"tau2Bench","benchmarkName":"τ²-Bench Tool-Agent-User Evaluation","category":"agentic","score":47.1,"unit":"percent","family":"tau2-bench","index":34.04373371684651,"aboveFit":false,"run":null,"source":"Victor Barres, Honghua Dong, Soham Ray, Xujie Si, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2506.07982","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaIfBench","benchmarkName":"Artificial Analysis IFBench","category":"instruction","score":43,"unit":"percent","family":"ifbench-family","index":29.207585829733524,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:aime:overall","benchmarkName":"AIME","category":"math","domain":null,"eligible":true,"ranked":true,"score":39.583,"unit":"percent","family":"vals-ai:aime:overall","index":32.802148325484204,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/aime","measuredAt":"2026-04-16","stderr":1.386,"costPerTest":0.050106,"latency":161.075},{"benchmarkId":"swe-bench:swe-bench-verified:overall","benchmarkName":"SWE-bench Verified","category":"coding","domain":null,"eligible":true,"ranked":true,"score":39.58,"unit":"percent","family":"swe-bench-verified","index":30.420456458150177,"aboveFit":false,"run":"mini-SWE-agent","source":"SWE-bench team","sourceUrl":"https://www.swebench.com/","measuredAt":"2026-02-26","stderr":null,"costPerTest":0.146313124,"latency":null},{"benchmarkId":"swebench-experiments:swe-bench-verified:overall","benchmarkName":"SWE-bench Verified","category":"coding","domain":null,"eligible":true,"ranked":true,"score":39.58,"unit":"percent","family":"swe-bench-verified","index":30.420456458150177,"aboveFit":false,"run":"mini-SWE-agent","source":"SWE-bench team","sourceUrl":"https://github.com/SWE-bench/experiments/tree/main/evaluation/verified","measuredAt":"2026-09-01","stderr":null,"costPerTest":0.146313124,"latency":null},{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":38.333333333333336,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":31.221195962999705,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":6.324056203789373,"costPerTest":null,"latency":null},{"benchmarkId":"tau2-bench:telecom:overall","benchmarkName":"τ²-bench Telecom","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":34,"unit":"percent","family":"tau2-bench","index":25.350925950673474,"aboveFit":false,"run":"Sierra","source":"Sierra Research","sourceUrl":"https://github.com/sierra-research/tau2-bench","measuredAt":"2026-03-02","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:terminal-bench:overall","benchmarkName":"Terminal-Bench 1.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":33.75,"unit":"percent","family":"vals-ai:terminal-bench:overall","index":42.64220395318109,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench","measuredAt":"2026-01-12","stderr":5.287,"costPerTest":1.292226,"latency":325.463},{"benchmarkId":"swebench-experiments:swe-bench-multimodal:overall","benchmarkName":"SWE-bench Multimodal","category":"multimodal","domain":null,"eligible":true,"ranked":true,"score":31.14119922630561,"unit":"percent","family":"swe-bench-multimodal","index":null,"aboveFit":false,"run":"GUIRepair","source":"SWE-bench team","sourceUrl":"https://github.com/SWE-bench/experiments/tree/main/evaluation/multimodal","measuredAt":"2025-11-17","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"swe-bench:swe-bench-multimodal:overall","benchmarkName":"SWE-bench Multimodal","category":"multimodal","domain":null,"eligible":true,"ranked":true,"score":31.14,"unit":"percent","family":"swe-bench-multimodal","index":null,"aboveFit":false,"run":"GUIRepair","source":"SWE-bench team","sourceUrl":"https://www.swebench.com/","measuredAt":"2025-11-17","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-simpleqa-verified:overall","benchmarkName":"SimpleQA Verified","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":31.1,"unit":"percent","family":"epoch-ai:epoch-simpleqa-verified:overall","index":49.82809776297482,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.4645596385722692,"costPerTest":null,"latency":null},{"benchmarkId":"omniscienceAccuracy","benchmarkName":"Artificial Analysis Omniscience Accuracy","category":"knowledge","score":27.8,"unit":"percent","family":"omniscienceAccuracy","index":45.72506682347969,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gertLabs","benchmarkName":"Gert Labs Composite Game Benchmark","category":"agentic","score":25.65,"unit":"percent","family":"gertLabs","index":36.31599246472965,"aboveFit":false,"run":null,"source":"Gert Labs","sourceUrl":"https://gertlabs.com/rankings","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:terminal-bench-2:overall","benchmarkName":"Terminal-Bench 2.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":14.607,"unit":"percent","family":"vals-ai:terminal-bench-2:overall","index":35.346303626499804,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2","measuredAt":"2026-06-04","stderr":3.765,"costPerTest":1.078347,"latency":589.207},{"benchmarkId":"arc-prize:v1-public-eval:overall","benchmarkName":"ARC-AGI-1 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":11.75,"unit":"percent","family":"arc-prize:v1-public-eval:overall","index":null,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.032,"latency":null},{"benchmarkId":"epoch-ai:epoch-chess-puzzles:overall","benchmarkName":"Chess Puzzles","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":11,"unit":"percent","family":"epoch-ai:epoch-chess-puzzles:overall","index":42.33384287178998,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":3.1446603773522015,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tiers-1-3-v2-private:overall","benchmarkName":"FrontierMath-Tiers-1-3-v2-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":5.964912280701754,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tiers-1-3-v2-private:overall","index":34.03433809201685,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.4053607746337793,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-2025-02-28-private:overall","benchmarkName":"FrontierMath-2025-02-28-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":5.517241379310345,"unit":"percent","family":"epoch-ai:epoch-frontiermath-2025-02-28-private:overall","index":34.784670605944015,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.3430381628597847,"costPerTest":null,"latency":null},{"benchmarkId":"arc-prize:v1-semi-private:overall","benchmarkName":"ARC-AGI-1 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":5.5,"unit":"percent","family":"arc-prize:v1-semi-private:overall","index":29.515907426549994,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.039,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":4.2,"unit":"percent","family":"hle","index":28.60047384243861,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"arc-prize:v2-semi-private:overall","benchmarkName":"ARC-AGI-2 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":0.42,"unit":"percent","family":"arc-prize:v2-semi-private:overall","index":39.38135197665253,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.0691,"latency":null},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":0,"unit":"percent","family":"critpt","index":36.8491160127729,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"arc-prize:v2-public-eval:overall","benchmarkName":"ARC-AGI-2 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":0,"unit":"percent","family":"arc-prize:v2-public-eval:overall","index":null,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.069,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-private:overall","benchmarkName":"FrontierMath-Tier-4-2025-07-01-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":0,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-private:overall","index":43.20235772716562,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-public:overall","benchmarkName":"FrontierMath-Tier-4-2025-07-01-Public","category":"math","domain":null,"eligible":false,"ranked":true,"score":0,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-public:overall","index":null,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-2025-02-28-public:overall","benchmarkName":"FrontierMath-2025-02-28-Public","category":"math","domain":null,"eligible":false,"ranked":true,"score":0,"unit":"percent","family":"epoch-ai:epoch-frontiermath-2025-02-28-public:overall","index":null,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":2,"outputPerMillion":8,"batch":{"inputPerMillion":1,"outputPerMillion":4},"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":74,"timeToFirstToken":0.997,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":{"elo":1382.9078482014697,"eloUpper":1386.604597224974,"eloLower":1379.2110991779655,"votes":50235,"rank":177,"rankUpper":null,"rankLower":null,"arenaName":"GPT-4.1-2025-04-14","organization":"openai","updatedAt":"2026-11-09"},"quality":{"coveragePercent":100,"benchmarkCount":29,"categoryCount":8,"confidence":"high","state":"available"},"hosts":[{"host":"digitalocean","name":"DigitalOcean","kind":"cloud","url":"https://docs.digitalocean.com/products/gradient-ai-platform/details/models/","inputPerMillion":2,"outputPerMillion":8,"cacheReadPerMillion":0.5,"contextTokens":1047576,"maxOutputTokens":32768,"deprecated":false},{"host":"azure","name":"Microsoft Azure","kind":"cloud","url":"https://learn.microsoft.com/en-us/azure/ai-services/openai/concepts/models","inputPerMillion":2,"outputPerMillion":8,"cacheReadPerMillion":0.5,"contextTokens":1047576,"maxOutputTokens":32768,"deprecated":true},{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":2,"outputPerMillion":8,"cacheReadPerMillion":0.5,"contextTokens":1047576,"maxOutputTokens":32768,"deprecated":false},{"host":"sap-ai-core","name":"SAP AI Core","kind":"cloud","url":"https://help.sap.com/docs/sap-ai-core","inputPerMillion":2,"outputPerMillion":8,"cacheReadPerMillion":0.5,"contextTokens":1047576,"maxOutputTokens":32768,"deprecated":true},{"host":"snowflake","name":"Snowflake Cortex","kind":"cloud","url":"https://docs.snowflake.com/en/user-guide/snowflake-cortex/cortex-rest-api","inputPerMillion":2,"outputPerMillion":8,"cacheReadPerMillion":0.5,"contextTokens":128000,"maxOutputTokens":32000,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:sierra-research","name":"Sierra Research","url":"https://github.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:swe-bench-team","name":"SWE-bench team","url":"https://www.swebench.com","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:arc-prize-foundation","name":"ARC Prize Foundation","url":"https://arcprize.org","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"north-mini-code-1-0","providerId":"cohere","slug":"north-mini-code-1-0","name":"North Mini Code","provider":"Cohere","description":"North Mini Code is a reasoning model from Cohere. 9 benchmarks count toward its index, in 5 categories.","logoUrl":null,"releaseDate":"2026-06-09","contextWindowTokens":256000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":true,"overallScore":38.015756970694895,"overallScoreRange":{"low":30.125756970694894,"high":45.905756970694895},"badges":[],"capabilityScores":{"agentic":30.820424724199857,"coding":46.42301492340291,"reasoning":35.1593670356492,"multimodal":null,"knowledge":38.680308861913836,"multilingual":null,"instruction":42.85144080624428,"math":null},"capabilityCounts":{"knowledge":3,"instruction":1,"coding":1,"agentic":2,"reasoning":2},"benchmarks":[{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":75.7,"unit":"percent","family":"gpqa-diamond","index":46.63907222927726,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaIfBench","benchmarkName":"Artificial Analysis IFBench","category":"instruction","score":57.6,"unit":"percent","family":"ifbench-family","index":42.85144080624428,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaSciCode","benchmarkName":"Artificial Analysis SciCode","category":"coding","score":38.8,"unit":"percent","family":"scicode-family","index":46.42301492340291,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/scicode","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"tau2Bench","benchmarkName":"τ²-Bench Tool-Agent-User Evaluation","category":"agentic","score":37.4,"unit":"percent","family":"tau2-bench","index":27.60707453120693,"aboveFit":false,"run":null,"source":"Victor Barres, Honghua Dong, Soham Ray, Xujie Si, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2506.07982","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":37.3,"unit":"percent","family":"lcr","index":32.888853063515825,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"omniscienceAccuracy","benchmarkName":"Artificial Analysis Omniscience Accuracy","category":"knowledge","score":18.4,"unit":"percent","family":"omniscienceAccuracy","index":33.87866057332005,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":11.1,"unit":"percent","family":"hle","index":35.523193783144194,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":0.3,"unit":"percent","family":"critpt","index":37.42988100778257,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gdpvalAaNormalized","benchmarkName":"GDPval-AA normalized","category":"agentic","score":0,"unit":"percent","family":"gdpvalAaNormalized","index":34.03377491719279,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":0,"outputPerMillion":0,"batch":null,"freeTier":true,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":70,"timeToFirstToken":28.92,"source":"BenchLM","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":75,"benchmarkCount":9,"categoryCount":5,"confidence":"low","state":"available"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"deepseek-v3-1","providerId":"deepseek","slug":"deepseek-v3-1","name":"DeepSeek V3.1","provider":"DeepSeek","description":"DeepSeek V3.1 is a standard model from DeepSeek. 14 benchmarks count toward its index, in 6 categories.","logoUrl":null,"releaseDate":"2025-08-21","contextWindowTokens":128000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":37.934011370036835,"overallScoreRange":{"low":32.11239708831293,"high":43.75562565176074},"badges":[],"capabilityScores":{"agentic":38.01117383240896,"coding":43.023949741224286,"reasoning":38.196475087234816,"multimodal":null,"knowledge":41.52664810369816,"multilingual":null,"instruction":24.348130632620105,"math":29.854499705604002},"capabilityCounts":{"knowledge":4,"math":2,"coding":2,"reasoning":2,"agentic":3,"instruction":1},"benchmarks":[{"benchmarkId":"mmluPro","benchmarkName":"Massive Multitask Language Understanding Professional","category":"knowledge","score":83.7,"unit":"percent","family":"mmlu-pro","index":49.93270979448208,"aboveFit":false,"run":null,"source":"Yubo Wang, Xueguang Ma, Ge Zhang, Yuansheng Ni, Abhranil Chandra, Shiguang Guo, Weiming Ren, Aaran Arulraj, Xuan He, Ziyan Jiang, Tianle Li, Max Ku, Kai Wang, Alex Zhuang, Rongqi Fan, Xiang Yue, Wenhu Chen","sourceUrl":"https://arxiv.org/abs/2406.01574","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gpqa","benchmarkName":"Graduate-Level Google-Proof Q&A","category":"knowledge","score":74.9,"unit":"percent","family":"gpqa-diamond","index":45.90533408934144,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":73.5,"unit":"percent","family":"gpqa-diamond","index":44.62129234445373,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aime2024","benchmarkName":"American Invitational Mathematics Examination 2024","category":"math","score":66.3,"unit":"percent","family":"aime2024","index":33.67901199806556,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"sweVerified","benchmarkName":"Software Engineering Benchmark Verified","category":"coding","score":66,"unit":"percent","family":"swe-bench-verified","index":49.90672766319531,"aboveFit":false,"run":null,"source":"Carlos E. Jimenez, John Yang, Alexander Wettig, Shunyu Yao, Kexin Pei, Ofir Press, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2310.06770","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"liveCodeBench","benchmarkName":"LiveCodeBench: Holistic and Contamination Free Evaluation of Large Language Models for Code","category":"coding","score":56.4,"unit":"percent","family":"livecodebench","index":36.14117181925326,"aboveFit":false,"run":null,"source":"Naman Jain, King Han, Alex Gu, Wen-Ding Li, Fanjia Yan, Tianjun Zhang, Sida Wang, Armando Solar-Lezama, Koushik Sen, Ion Stoica","sourceUrl":"https://arxiv.org/abs/2403.07974","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:case_law_v2:overall","benchmarkName":"CaseLaw v2","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":53.912,"unit":"percent","family":"vals-ai:case_law_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/case_law_v2","measuredAt":"2026-05-04","stderr":0.567,"costPerTest":0.016723,"latency":40.682},{"benchmarkId":"vals-ai:corp_fin_v2:overall","benchmarkName":"CorpFin v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":51.476,"unit":"percent","family":"vals-ai:corp_fin_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","measuredAt":"2026-08-12","stderr":0.982,"costPerTest":0.037369,"latency":62.999},{"benchmarkId":"aime2025","benchmarkName":"American Invitational Mathematics Examination 2025","category":"math","score":49.8,"unit":"percent","family":"aime2025","index":26.029987413142443,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":47,"unit":"percent","family":"lcr","index":39.543834161696736,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:terminal-bench:overall","benchmarkName":"Terminal-Bench 1.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":41.25,"unit":"percent","family":"vals-ai:terminal-bench:overall","index":48.06982840515033,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench","measuredAt":"2026-01-12","stderr":5.504,"costPerTest":0.391033,"latency":641.23},{"benchmarkId":"aaIfBench","benchmarkName":"Artificial Analysis IFBench","category":"instruction","score":37.8,"unit":"percent","family":"ifbench-family","index":24.348130632620105,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"tau2Bench","benchmarkName":"τ²-Bench Tool-Agent-User Evaluation","category":"agentic","score":34.8,"unit":"percent","family":"tau2-bench","index":25.881784440210758,"aboveFit":false,"run":null,"source":"Victor Barres, Honghua Dong, Soham Ray, Xujie Si, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2506.07982","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"omniscienceAccuracy","benchmarkName":"Artificial Analysis Omniscience Accuracy","category":"knowledge","score":23.1,"unit":"percent","family":"omniscienceAccuracy","index":39.80186369839987,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:terminal-bench-2:overall","benchmarkName":"Terminal-Bench 2.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":22.472,"unit":"percent","family":"vals-ai:terminal-bench-2:overall","index":40.08190865186579,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2","measuredAt":"2026-06-04","stderr":4.449,"costPerTest":0.501536,"latency":837.847},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":6.7,"unit":"percent","family":"hle","index":31.108705705013097,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":0,"unit":"percent","family":"critpt","index":36.8491160127729,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":98,"timeToFirstToken":1.41,"source":"BenchLM","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":{"elo":1419.8929312942228,"eloUpper":1425.941900566439,"eloLower":1413.8439620220065,"votes":14586,"rank":118,"rankUpper":null,"rankLower":null,"arenaName":"DeepSeek-V3.1","organization":"deepseek","updatedAt":"2026-11-09"},"quality":{"coveragePercent":85,"benchmarkCount":14,"categoryCount":6,"confidence":"medium","state":"available"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"mistral-small-4","providerId":"mistral","slug":"mistral-small-4","name":"Mistral Small 4","provider":"Mistral","description":"Mistral Small 4 is a standard model from Mistral. 12 benchmarks count toward its index, in 6 categories.","logoUrl":null,"releaseDate":"2026-03-16","contextWindowTokens":262144,"modalities":["text","image"],"openWeights":true,"details":{"apiId":"mistralai/mistral-small-2603","maxOutputTokens":209715,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.015,"weightsUrl":"https://huggingface.co/mistralai/Mistral-Small-4-119B-2603"},"reasoning":false,"overallScore":37.8888873495691,"overallScoreRange":{"low":31.721010423170977,"high":44.056764275967225},"badges":[],"capabilityScores":{"agentic":33.62055829718513,"coding":44.3362687571822,"reasoning":39.41306629428029,"multimodal":29.07303168660426,"knowledge":39.906116817929366,"multilingual":null,"instruction":34.067041026846944,"math":null},"capabilityCounts":{"knowledge":3,"multimodal":1,"reasoning":2,"instruction":1,"agentic":3,"coding":2},"benchmarks":[{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":76.9,"unit":"percent","family":"gpqa-diamond","index":47.739679439181,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:sage:overall","benchmarkName":"SAGE","category":null,"domain":"Education","eligible":false,"ranked":true,"score":59.158,"unit":"percent","family":"vals-ai:sage:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/sage","measuredAt":"2026-11-05","stderr":2.125,"costPerTest":0.003493,"latency":14.094},{"benchmarkId":"aaMmmuPro","benchmarkName":"Artificial Analysis MMMU-Pro","category":"multimodal","score":56.8,"unit":"percent","family":"mmmu-pro","index":29.07303168660426,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmmu-pro","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":49.7,"unit":"percent","family":"lcr","index":41.39625158077802,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaIfBench","benchmarkName":"Artificial Analysis IFBench","category":"instruction","score":48.2,"unit":"percent","family":"ifbench-family","index":34.067041026846944,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"tau2Bench","benchmarkName":"τ²-Bench Tool-Agent-User Evaluation","category":"agentic","score":41.2,"unit":"percent","family":"tau2-bench","index":30.128652356509036,"aboveFit":false,"run":null,"source":"Victor Barres, Honghua Dong, Soham Ray, Xujie Si, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2506.07982","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaSciCode","benchmarkName":"Artificial Analysis SciCode","category":"coding","score":38.8,"unit":"percent","family":"scicode-family","index":46.42301492340291,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/scicode","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"omniscienceAccuracy","benchmarkName":"Artificial Analysis Omniscience Accuracy","category":"knowledge","score":21.4,"unit":"percent","family":"omniscienceAccuracy","index":37.659428525498654,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:terminal-bench-2:overall","benchmarkName":"Terminal-Bench 2.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":16.854,"unit":"percent","family":"vals-ai:terminal-bench-2:overall","index":36.699247617853565,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2","measuredAt":"2026-06-04","stderr":3.991,"costPerTest":0.105762,"latency":407.865},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":9.9,"unit":"percent","family":"hle","index":34.31924248910844,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":0.3,"unit":"percent","family":"critpt","index":37.42988100778257,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gdpvalAaNormalized","benchmarkName":"GDPval-AA normalized","category":"agentic","score":0,"unit":"percent","family":"gdpvalAaNormalized","index":34.03377491719279,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:vibe-code:overall","benchmarkName":"Vibe Code Bench v1.1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":0,"unit":"percent","family":"vals-ai:vibe-code:overall","index":42.24952259096148,"aboveFit":false,"run":"OpenHands","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","measuredAt":"2026-11-09","stderr":0,"costPerTest":6.473001,"latency":1814.681}],"price":{"inputPerMillion":0.15,"outputPerMillion":0.6,"batch":{"inputPerMillion":0.075,"outputPerMillion":0.3},"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":114,"timeToFirstToken":0.571,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":85,"benchmarkCount":12,"categoryCount":6,"confidence":"low","state":"available"},"hosts":[{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":0.15,"outputPerMillion":0.6,"cacheReadPerMillion":0.015,"contextTokens":262144,"maxOutputTokens":209715,"deprecated":false},{"host":"sap-ai-core","name":"SAP AI Core","kind":"cloud","url":"https://help.sap.com/docs/sap-ai-core","inputPerMillion":0.15,"outputPerMillion":0.6,"cacheReadPerMillion":null,"contextTokens":128000,"maxOutputTokens":128000,"deprecated":true},{"host":"regolo","name":"Regolo","kind":"host","url":"https://docs.regolo.ai/","inputPerMillion":0.56075,"outputPerMillion":2.35515,"cacheReadPerMillion":null,"contextTokens":256000,"maxOutputTokens":16384,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"claude-3-7-sonnet","providerId":"anthropic","slug":"claude-3-7-sonnet","name":"Claude 3.7 Sonnet","provider":"Anthropic","description":"Claude 3.7 Sonnet is a standard model from Anthropic. 21 benchmarks count toward its index, in 7 categories.","logoUrl":null,"releaseDate":"2025-02-24","contextWindowTokens":200000,"modalities":["text"],"openWeights":false,"details":null,"reasoning":false,"overallScore":37.7429916396002,"overallScoreRange":{"low":33.7144083544842,"high":41.7715749247162},"badges":[],"capabilityScores":{"agentic":40.679464836062564,"coding":42.13616363507273,"reasoning":39.80876432236112,"multimodal":40.26965038909164,"knowledge":39.98488943839692,"multilingual":null,"instruction":38.093996940966775,"math":29.954574745859915},"capabilityCounts":{"instruction":2,"math":6,"knowledge":5,"agentic":1,"multimodal":2,"coding":3,"reasoning":2},"benchmarks":[{"benchmarkId":"vals-ai:mgsm:overall","benchmarkName":"MGSM","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":92.4,"unit":"percent","family":"vals-ai:mgsm:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mgsm","measuredAt":"2026-01-09","stderr":1.626,"costPerTest":0.00566,"latency":4.861},{"benchmarkId":"ifeval","benchmarkName":"Instruction-Following Eval","category":"instruction","score":90.8,"unit":"percent","family":"ifeval","index":46.04589743737053,"aboveFit":false,"run":null,"source":"Jeffrey Zhou, Tianjian Lu, Swaroop Mishra, Siddhartha Brahma, Sujoy Basu, Yi Luan, Denny Zhou, Le Hou","sourceUrl":"https://arxiv.org/abs/2311.07911","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-math-level-5:overall","benchmarkName":"MATH level 5","category":"math","domain":null,"eligible":true,"ranked":true,"score":88.14199395770393,"unit":"percent","family":"epoch-ai:epoch-math-level-5:overall","index":41.81335869901581,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0.8236774684678254,"costPerTest":null,"latency":null},{"benchmarkId":"mmmlu","benchmarkName":"MMMLU","category":"knowledge","score":83.2,"unit":"percent","family":"mmmlu","index":39.67565391638687,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"math500","benchmarkName":"MATH-500 Problem Set","category":"math","score":82.2,"unit":"percent","family":"math-500","index":30.992308318113416,"aboveFit":false,"run":null,"source":"Dan Hendrycks, Collin Burns, Saurav Kadavath, Akul Arora, Steven Basart, Eric Tang, Dawn Song, Jacob Steinhardt","sourceUrl":"https://arxiv.org/abs/2103.03874","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:mmlu_pro:overall","benchmarkName":"MMLU Pro","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":80.663,"unit":"percent","family":"mmlu-pro","index":46.10645331403761,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","measuredAt":"2026-09-01","stderr":0.383,"costPerTest":0.009664,"latency":6.302},{"benchmarkId":"vals-ai:legal_bench:overall","benchmarkName":"LegalBench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":80.001,"unit":"percent","family":"vals-ai:legal_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","measuredAt":"2026-10-13","stderr":0.484,"costPerTest":null,"latency":1.199},{"benchmarkId":"vals-ai:math500:overall","benchmarkName":"MATH 500","category":"math","domain":null,"eligible":true,"ranked":true,"score":76.8,"unit":"percent","family":"math-500","index":25.285813970992056,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/math500","measuredAt":"2026-01-09","stderr":1.888,"costPerTest":0.006868,"latency":5.527},{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":76.76767676767676,"unit":"percent","family":"gpqa-diamond","index":47.61831618623706,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":3.0088629490217444,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:tax_eval_v2:overall","benchmarkName":"TaxEval v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":72.404,"unit":"percent","family":"vals-ai:tax_eval_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","measuredAt":"2026-09-01","stderr":0.879,"costPerTest":0.006427,"latency":6.28},{"benchmarkId":"tau2-bench:retail:overall","benchmarkName":"τ²-bench Retail","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":72.1,"unit":"percent","family":"tau2-bench","index":50.63306151488665,"aboveFit":false,"run":"Sierra","source":"Sierra Research","sourceUrl":"https://github.com/sierra-research/tau2-bench","measuredAt":"2026-04-30","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmmu","benchmarkName":"Massive Multi-discipline Multimodal Understanding","category":"multimodal","score":71.8,"unit":"percent","family":"mmmu","index":39.741195288987896,"aboveFit":false,"run":null,"source":"MMMU authors","sourceUrl":"https://arxiv.org/abs/2401.05508","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:mmmu:overall","benchmarkName":"MMMU Pro","category":"multimodal","domain":null,"eligible":true,"ranked":true,"score":71.519,"unit":"percent","family":"mmmu-pro","index":48.228518816893256,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmmu","measuredAt":"2026-09-01","stderr":1.085,"costPerTest":0.012358,"latency":8.323},{"benchmarkId":"vals-ai:mortgage_tax:overall","benchmarkName":"MortgageTax","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":68.68,"unit":"percent","family":"vals-ai:mortgage_tax:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mortgage_tax","measuredAt":"2026-09-01","stderr":1.775,"costPerTest":0.015796,"latency":5.67},{"benchmarkId":"gpqa","benchmarkName":"Graduate-Level Google-Proof Q&A","category":"knowledge","score":68,"unit":"percent","family":"gpqa-diamond","index":39.576842632394914,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:gpqa:overall","benchmarkName":"GPQA Diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":67.424,"unit":"percent","family":"gpqa-diamond","index":39.04855117164112,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","measuredAt":"2026-09-01","stderr":2.354,"costPerTest":0.011972,"latency":9.218},{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":65.6,"unit":"percent","family":"gpqa-diamond","index":37.37562821258742,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"tau2-bench:airline:overall","benchmarkName":"τ²-bench Airline","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":64.2,"unit":"percent","family":"tau2-bench","index":45.39083393070597,"aboveFit":false,"run":"Sierra","source":"Sierra Research","sourceUrl":"https://github.com/sierra-research/tau2-bench","measuredAt":"2026-03-02","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"swebench-experiments:swe-bench-verified:overall","benchmarkName":"SWE-bench Verified","category":"coding","domain":null,"eligible":true,"ranked":true,"score":62.4,"unit":"percent","family":"swe-bench-verified","index":47.251520383174025,"aboveFit":false,"run":"SWE-agent","source":"SWE-bench team","sourceUrl":"https://github.com/SWE-bench/experiments/tree/main/evaluation/verified","measuredAt":"2026-09-01","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"sweVerified","benchmarkName":"Software Engineering Benchmark Verified","category":"coding","score":62.3,"unit":"percent","family":"swe-bench-verified","index":47.17776462539565,"aboveFit":false,"run":null,"source":"Carlos E. Jimenez, John Yang, Alexander Wettig, Shunyu Yao, Kexin Pei, Ofir Press, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2310.06770","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-swe-bench-verified:overall","benchmarkName":"SWE-Bench verified","category":"coding","domain":null,"eligible":true,"ranked":true,"score":60.9504132231405,"unit":"percent","family":"swe-bench-verified","index":46.182366671246264,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.2198466971122266,"costPerTest":null,"latency":null},{"benchmarkId":"aaMmmuPro","benchmarkName":"Artificial Analysis MMMU-Pro","category":"multimodal","score":60.1,"unit":"percent","family":"mmmu-pro","index":33.36769216149751,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmmu-pro","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:lcb:overall","benchmarkName":"LiveCodeBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":56.662,"unit":"percent","family":"livecodebench","index":36.35856402125198,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","measuredAt":"2026-09-01","stderr":1.146,"costPerTest":0.020551,"latency":16.232},{"benchmarkId":"swe-bench:swe-bench-verified:overall","benchmarkName":"SWE-bench Verified","category":"coding","domain":null,"eligible":true,"ranked":true,"score":52.8,"unit":"percent","family":"swe-bench-verified","index":40.170967636450584,"aboveFit":false,"run":"mini-SWE-agent","source":"SWE-bench team","sourceUrl":"https://www.swebench.com/","measuredAt":"2026-02-26","stderr":null,"costPerTest":0.3546154482,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":51.7,"unit":"percent","family":"lcr","index":42.76841263194934,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"tau2Bench","benchmarkName":"τ²-Bench Tool-Agent-User Evaluation","category":"agentic","score":50,"unit":"percent","family":"tau2-bench","index":35.96809574141916,"aboveFit":false,"run":null,"source":"Victor Barres, Honghua Dong, Soham Ray, Xujie Si, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2506.07982","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":50,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":38.00879316321904,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":7.521014330903548,"costPerTest":null,"latency":null},{"benchmarkId":"tau2-bench:telecom:overall","benchmarkName":"τ²-bench Telecom","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":49,"unit":"percent","family":"tau2-bench","index":35.30452262949756,"aboveFit":false,"run":"Sierra","source":"Sierra Research","sourceUrl":"https://github.com/sierra-research/tau2-bench","measuredAt":"2026-03-02","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"swe-bench:swe-bench-lite:overall","benchmarkName":"SWE-bench Lite","category":"coding","domain":null,"eligible":true,"ranked":true,"score":48,"unit":"percent","family":"swe-bench-lite","index":43.369861235645246,"aboveFit":false,"run":"SWE-agent","source":"SWE-bench team","sourceUrl":"https://www.swebench.com/","measuredAt":"2025-09-11","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"swebench-experiments:swe-bench-lite:overall","benchmarkName":"SWE-bench Lite","category":"coding","domain":null,"eligible":true,"ranked":true,"score":48,"unit":"percent","family":"swe-bench-lite","index":43.369861235645246,"aboveFit":false,"run":"SWE-agent","source":"SWE-bench team","sourceUrl":"https://github.com/SWE-bench/experiments/tree/main/evaluation/lite","measuredAt":"2025-09-11","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaIfBench","benchmarkName":"Artificial Analysis IFBench","category":"instruction","score":44,"unit":"percent","family":"ifbench-family","index":30.142096444563027,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"swebench-experiments:swe-bench-test:overall","benchmarkName":"SWE-bench (full test split)","category":"coding","domain":null,"eligible":true,"ranked":true,"score":33.82737576285963,"unit":"percent","family":"swebench-experiments:swe-bench-test:overall","index":null,"aboveFit":false,"run":"SWE-agent 1.0","source":"SWE-bench team","sourceUrl":"https://github.com/SWE-bench/experiments/tree/main/evaluation/test","measuredAt":"2025-12-19","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"swebench-experiments:swe-bench-multimodal:overall","benchmarkName":"SWE-bench Multimodal","category":"multimodal","domain":null,"eligible":true,"ranked":true,"score":30.947775628626694,"unit":"percent","family":"swe-bench-multimodal","index":null,"aboveFit":false,"run":"OpenHands-Versa","source":"SWE-bench team","sourceUrl":"https://github.com/SWE-bench/experiments/tree/main/evaluation/multimodal","measuredAt":"2025-11-17","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"swe-bench:swe-bench-multimodal:overall","benchmarkName":"SWE-bench Multimodal","category":"multimodal","domain":null,"eligible":true,"ranked":true,"score":30.945,"unit":"percent","family":"swe-bench-multimodal","index":null,"aboveFit":false,"run":"OpenHands-Versa","source":"SWE-bench team","sourceUrl":"https://www.swebench.com/","measuredAt":"2025-11-17","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"omniscienceAccuracy","benchmarkName":"Artificial Analysis Omniscience Accuracy","category":"knowledge","score":28.2,"unit":"percent","family":"omniscienceAccuracy","index":46.229169217103504,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aime2024","benchmarkName":"American Invitational Mathematics Examination 2024","category":"math","score":23.3,"unit":"percent","family":"aime2024","index":13.715014288020022,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:aime:overall","benchmarkName":"AIME","category":"math","domain":null,"eligible":true,"ranked":true,"score":22.292,"unit":"percent","family":"vals-ai:aime:overall","index":25.245190081807138,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/aime","measuredAt":"2026-04-16","stderr":1.352,"costPerTest":0.023201,"latency":18.928},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":4.2,"unit":"percent","family":"hle","index":28.60047384243861,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-2025-02-28-private:overall","benchmarkName":"FrontierMath-2025-02-28-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":3.275862068965517,"unit":"percent","family":"epoch-ai:epoch-frontiermath-2025-02-28-private:overall","index":32.806031098544736,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.020064175308735,"costPerTest":null,"latency":null},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":0,"unit":"percent","family":"critpt","index":36.8491160127729,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-public:overall","benchmarkName":"FrontierMath-Tier-4-2025-07-01-Public","category":"math","domain":null,"eligible":false,"ranked":true,"score":0,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-public:overall","index":null,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-2025-02-28-public:overall","benchmarkName":"FrontierMath-2025-02-28-Public","category":"math","domain":null,"eligible":false,"ranked":true,"score":0,"unit":"percent","family":"epoch-ai:epoch-frontiermath-2025-02-28-public:overall","index":null,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":{"elo":1299.248800613048,"eloUpper":1303.236592684093,"eloLower":1295.2610085420029,"votes":42553,"rank":253,"rankUpper":null,"rankLower":null,"arenaName":"Claude 3.7 Sonnet-20250219","organization":"anthropic","updatedAt":"2026-11-09"},"quality":{"coveragePercent":95,"benchmarkCount":21,"categoryCount":7,"confidence":"medium","state":"available"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:sierra-research","name":"Sierra Research","url":"https://github.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:swe-bench-team","name":"SWE-bench team","url":"https://github.com","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"granite-4-2-8b","providerId":"ibm","slug":"granite-4-2-8b","name":"Granite 4.2 8B","provider":"IBM","description":"Granite 4.2 8B is a reasoning model from IBM in the Granite 4.2 family. 17 benchmarks count toward its index, in 6 categories.","logoUrl":null,"releaseDate":"2026-08-25","contextWindowTokens":131072,"modalities":["text"],"openWeights":true,"details":{"apiId":"ibm-granite/granite-4.2-8b","maxOutputTokens":117964,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.015,"weightsUrl":"https://huggingface.co/ibm-granite/granite-4.2-8b"},"reasoning":true,"overallScore":37.57313131284509,"overallScoreRange":{"low":32.15133025600759,"high":42.994932369682594},"badges":[],"capabilityScores":{"agentic":36.81341411921326,"coding":36.48757124310714,"reasoning":37.80077705915399,"multimodal":null,"knowledge":33.25036185040884,"multilingual":null,"instruction":63.15835646648938,"math":41.90009190976299},"capabilityCounts":{"math":2,"instruction":1,"knowledge":4,"coding":4,"agentic":4,"reasoning":2},"benchmarks":[{"benchmarkId":"aime2025","benchmarkName":"American Invitational Mathematics Examination 2025","category":"math","score":86.67,"unit":"percent","family":"aime2025","index":46.03660126157782,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"ifBench","benchmarkName":"Instruction Following Benchmark","category":"instruction","score":79.33,"unit":"percent","family":"ifbench-family","index":63.15835646648938,"aboveFit":true,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hmmtFeb2025","benchmarkName":"Harvard-MIT Mathematics Tournament February 2025","category":"math","score":78.33,"unit":"percent","family":"hmmtFeb2025","index":37.763582557948155,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmluPro","benchmarkName":"Massive Multitask Language Understanding Professional","category":"knowledge","score":74.04,"unit":"percent","family":"mmlu-pro","index":37.7622660667595,"aboveFit":false,"run":null,"source":"Yubo Wang, Xueguang Ma, Ge Zhang, Yuansheng Ni, Abhranil Chandra, Shiguang Guo, Weiming Ren, Aaran Arulraj, Xuan He, Ziyan Jiang, Tianle Li, Max Ku, Kai Wang, Alex Zhuang, Rongqi Fan, Xiang Yue, Wenhu Chen","sourceUrl":"https://arxiv.org/abs/2406.01574","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"liveCodeBenchV6","benchmarkName":"LiveCodeBench v6","category":"coding","score":73.24,"unit":"percent","family":"liveCodeBenchV6","index":44.55129655999524,"aboveFit":false,"run":null,"source":"LiveCodeBench maintainers","sourceUrl":"https://github.com/LiveCodeBench/LiveCodeBench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaOpennessIndex","benchmarkName":"Artificial Analysis Openness Index","category":"knowledge","score":72.2,"unit":"percent","family":"aaOpennessIndex","index":null,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-openness-index","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gpqa","benchmarkName":"Graduate-Level Google-Proof Q&A","category":"knowledge","score":64.14,"unit":"percent","family":"gpqa-diamond","index":36.03655610720455,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":63.1,"unit":"percent","family":"gpqa-diamond","index":35.08269652528797,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"tau3Bench","benchmarkName":"τ³-Bench Tool-Agent-User Evaluation","category":"agentic","score":58.06,"unit":"percent","family":"tau3Bench","index":46.259257288929234,"aboveFit":false,"run":null,"source":"Sierra Research","sourceUrl":"https://github.com/sierra-research/tau2-bench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"bfclV4","benchmarkName":"Berkeley Function Calling Leaderboard v4","category":"agentic","score":52.39,"unit":"percent","family":"bfclV4","index":33.794876769811566,"aboveFit":false,"run":null,"source":"Shishir G. Patil et al.","sourceUrl":"https://gorilla.cs.berkeley.edu/leaderboard.html","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"sweVerified","benchmarkName":"Software Engineering Benchmark Verified","category":"coding","score":47.67,"unit":"percent","family":"swe-bench-verified","index":36.38729726242025,"aboveFit":false,"run":null,"source":"Carlos E. Jimenez, John Yang, Alexander Wettig, Shunyu Yao, Kexin Pei, Ofir Press, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2310.06770","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":45,"unit":"percent","family":"lcr","index":38.17167311052541,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"sciCode","benchmarkName":"Scientific Code Benchmark","category":"coding","score":36.09,"unit":"percent","family":"scicode-family","index":42.890011916618995,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaSciCode","benchmarkName":"Artificial Analysis SciCode","category":"coding","score":31.5,"unit":"percent","family":"scicode-family","index":36.90606955457537,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/scicode","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"terminalBench21","benchmarkName":"Terminal-Bench 2.1","category":"agentic","score":20.56,"unit":"percent","family":"terminal-bench-2-1","index":33.16574750091945,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"swePro","benchmarkName":"SWE-bench Pro","category":"coding","score":19.11,"unit":"percent","family":"swePro","index":25.11365041441589,"aboveFit":false,"run":null,"source":"Xiang Deng, Jeff Da, Edwin Pan, Yannis Yiming He, Charles Ide, Kanak Garg, Niklas Lauffer, Andrew Park, Nitin Pasari, Chetan Rane, Karmini Sampath, Maya Krishnan, Srivatsa Kundurthy, Sean Hendryx, Zifan Wang, Chen Bo Calvin Zhang, Noah Jacobson, Bing Liu, Brad Kenstler","sourceUrl":"https://arxiv.org/abs/2509.16941","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"omniscienceAccuracy","benchmarkName":"Artificial Analysis Omniscience Accuracy","category":"knowledge","score":11.8,"unit":"percent","family":"omniscienceAccuracy","index":25.56097107852711,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":9.7,"unit":"percent","family":"hle","index":34.11858394010248,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":0.3,"unit":"percent","family":"critpt","index":37.42988100778257,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gdpvalAaNormalized","benchmarkName":"GDPval-AA normalized","category":"agentic","score":0,"unit":"percent","family":"gdpvalAaNormalized","index":34.03377491719279,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":0.06,"outputPerMillion":0.15,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":101,"timeToFirstToken":0.2945,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":{"elo":1318.8220227076013,"eloUpper":1330.0032507545748,"eloLower":1307.6407946606278,"votes":3196,"rank":242,"rankUpper":null,"rankLower":null,"arenaName":"Granite 4.2 8B","organization":"ibm","updatedAt":"2026-11-09"},"quality":{"coveragePercent":85,"benchmarkCount":17,"categoryCount":6,"confidence":"medium","state":"available"},"hosts":[{"host":"coreweave","name":"CoreWeave","kind":"host","url":"https://docs.wandb.ai/inference","inputPerMillion":0.06,"outputPerMillion":0.15,"cacheReadPerMillion":0.015,"contextTokens":131072,"maxOutputTokens":131072,"deprecated":false},{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":0.06,"outputPerMillion":0.15,"cacheReadPerMillion":0.015,"contextTokens":131072,"maxOutputTokens":117964,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"deepseek-v3-0324","providerId":"deepseek","slug":"deepseek-v3-0324","name":"DeepSeek V3 0324","provider":"DeepSeek","description":"DeepSeek V3 0324 is a standard model from DeepSeek in the DeepSeek family. 16 benchmarks count toward its index, in 7 categories.","logoUrl":null,"releaseDate":"2025-03-24","contextWindowTokens":128000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":37.10681732635228,"overallScoreRange":{"low":32.06431732635228,"high":42.149317326352275},"badges":[],"capabilityScores":{"agentic":34.03875431701965,"coding":40.883118983895315,"reasoning":36.03532143163999,"multimodal":null,"knowledge":37.63187708457632,"multilingual":40.064497965431336,"instruction":27.338564600074516,"math":38.03639482703741},"capabilityCounts":{"math":2,"knowledge":4,"multilingual":1,"coding":4,"agentic":2,"instruction":1,"reasoning":2},"benchmarks":[{"benchmarkId":"vals-ai:mgsm:overall","benchmarkName":"MGSM","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":91.673,"unit":"percent","family":"vals-ai:mgsm:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mgsm","measuredAt":"2026-01-09","stderr":1.696,"costPerTest":0.000395,"latency":18.032},{"benchmarkId":"vals-ai:math500:overall","benchmarkName":"MATH 500","category":"math","domain":null,"eligible":true,"ranked":true,"score":88.6,"unit":"percent","family":"math-500","index":37.75556087766464,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/math500","measuredAt":"2026-01-09","stderr":1.421,"costPerTest":0.00074,"latency":12.545},{"benchmarkId":"vals-ai:medqa:overall","benchmarkName":"MedQA","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":82,"unit":"percent","family":"vals-ai:medqa:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medqa","measuredAt":"2026-04-16","stderr":0.351,"costPerTest":0.000614,"latency":12.876},{"benchmarkId":"vals-ai:mmlu_pro:overall","benchmarkName":"MMLU Pro","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":79.474,"unit":"percent","family":"mmlu-pro","index":44.60845563367922,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","measuredAt":"2026-09-01","stderr":0.396,"costPerTest":0.00147,"latency":25.051},{"benchmarkId":"vals-ai:legal_bench:overall","benchmarkName":"LegalBench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":77.727,"unit":"percent","family":"vals-ai:legal_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","measuredAt":"2026-10-13","stderr":0.418,"costPerTest":null,"latency":7.712},{"benchmarkId":"vals-ai:tax_eval_v2:overall","benchmarkName":"TaxEval v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":71.096,"unit":"percent","family":"vals-ai:tax_eval_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","measuredAt":"2026-09-01","stderr":0.883,"costPerTest":0.000652,"latency":34.009},{"benchmarkId":"mmluProX","benchmarkName":"MMLU-ProX","category":"multilingual","score":70.5,"unit":"percent","family":"mmluProX","index":40.064497965431336,"aboveFit":false,"run":null,"source":"MMLU-ProX authors","sourceUrl":"https://arxiv.org/abs/2503.10497","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":65.5,"unit":"percent","family":"gpqa-diamond","index":37.28391094509545,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:lcb:overall","benchmarkName":"LiveCodeBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":65.478,"unit":"percent","family":"livecodebench","index":43.673562696140124,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","measuredAt":"2026-09-01","stderr":1.164,"costPerTest":0.001776,"latency":26.93},{"benchmarkId":"vals-ai:gpqa:overall","benchmarkName":"GPQA Diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":61.616,"unit":"percent","family":"gpqa-diamond","index":33.721612275707,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","measuredAt":"2026-09-01","stderr":2.447,"costPerTest":0.001552,"latency":23.475},{"benchmarkId":"vals-ai:corp_fin_v2:overall","benchmarkName":"CorpFin v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":54.74,"unit":"percent","family":"vals-ai:corp_fin_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","measuredAt":"2026-08-12","stderr":0.978,"costPerTest":0.052049,"latency":36.732},{"benchmarkId":"vals-ai:aime:overall","benchmarkName":"AIME","category":"math","domain":null,"eligible":true,"ranked":true,"score":52.202,"unit":"percent","family":"vals-ai:aime:overall","index":38.317228776410175,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/aime","measuredAt":"2026-04-16","stderr":1.712,"costPerTest":0.003192,"latency":50.575},{"benchmarkId":"tau2Bench","benchmarkName":"τ²-Bench Tool-Agent-User Evaluation","category":"agentic","score":47.1,"unit":"percent","family":"tau2-bench","index":34.04373371684651,"aboveFit":false,"run":null,"source":"Victor Barres, Honghua Dong, Soham Ray, Xujie Si, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2506.07982","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"swebench-experiments:swe-bench-verified:overall","benchmarkName":"SWE-bench Verified","category":"coding","domain":null,"eligible":true,"ranked":true,"score":42,"unit":"percent","family":"swe-bench-verified","index":32.20534579638671,"aboveFit":false,"run":"SWE-Exp","source":"SWE-bench team","sourceUrl":"https://github.com/SWE-bench/experiments/tree/main/evaluation/verified","measuredAt":"2026-09-01","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaIfBench","benchmarkName":"Artificial Analysis IFBench","category":"instruction","score":41,"unit":"percent","family":"ifbench-family","index":27.338564600074516,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":40.7,"unit":"percent","family":"lcr","index":35.221526850507075,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaSciCode","benchmarkName":"Artificial Analysis SciCode","category":"coding","score":39,"unit":"percent","family":"scicode-family","index":46.68375315268586,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/scicode","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"omniscienceAccuracy","benchmarkName":"Artificial Analysis Omniscience Accuracy","category":"knowledge","score":24.3,"unit":"percent","family":"omniscienceAccuracy","index":41.314170879271316,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":4.7,"unit":"percent","family":"hle","index":29.10212021495351,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:ioi-v1:overall","benchmarkName":"IOI v1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":1.667,"unit":"percent","family":"vals-ai:ioi-v1:overall","index":40.96981429036857,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/ioi-v1","measuredAt":"2026-08-09","stderr":0.9,"costPerTest":0.606484,"latency":171.479},{"benchmarkId":"gdpvalAaNormalized","benchmarkName":"GDPval-AA normalized","category":"agentic","score":0,"unit":"percent","family":"gdpvalAaNormalized","index":34.03377491719279,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":0,"unit":"percent","family":"critpt","index":36.8491160127729,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":{"elo":1374.622683490949,"eloUpper":1378.4808419239343,"eloLower":1370.7645250579635,"votes":44918,"rank":184,"rankUpper":null,"rankLower":null,"arenaName":"DeepSeek-V3-0324","organization":"deepseek","updatedAt":"2026-11-09"},"quality":{"coveragePercent":90,"benchmarkCount":16,"categoryCount":7,"confidence":"medium","state":"available"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:swe-bench-team","name":"SWE-bench team","url":"https://github.com","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"minicpm5-2b","providerId":"openbmb","slug":"minicpm5-2b","name":"MiniCPM5-2B","provider":"OpenBMB","description":"MiniCPM5-2B is a reasoning model from OpenBMB in the MiniCPM5 family. 21 benchmarks count toward its index, in 6 categories.","logoUrl":null,"releaseDate":"2026-09-06","contextWindowTokens":131000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":true,"overallScore":37.10047147141234,"overallScoreRange":{"low":32.07188818629634,"high":42.12905475652834},"badges":[],"capabilityScores":{"agentic":38.21286476000911,"coding":31.958040526516736,"reasoning":39.12772191617433,"multimodal":null,"knowledge":34.68989841885007,"multilingual":null,"instruction":44.38529758811618,"math":43.91385602396838},"capabilityCounts":{"math":4,"instruction":2,"knowledge":5,"coding":4,"agentic":3,"reasoning":3},"benchmarks":[{"benchmarkId":"math500","benchmarkName":"MATH-500 Problem Set","category":"math","score":94.6,"unit":"percent","family":"math-500","index":44.096110152243924,"aboveFit":false,"run":null,"source":"Dan Hendrycks, Collin Burns, Saurav Kadavath, Akul Arora, Steven Basart, Eric Tang, Dawn Song, Jacob Steinhardt","sourceUrl":"https://arxiv.org/abs/2103.03874","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"ifeval","benchmarkName":"Instruction-Following Eval","category":"instruction","score":86.7,"unit":"percent","family":"ifeval","index":37.78891202097141,"aboveFit":false,"run":null,"source":"Jeffrey Zhou, Tianjian Lu, Swaroop Mishra, Siddhartha Brahma, Sujoy Basu, Yi Luan, Denny Zhou, Le Hou","sourceUrl":"https://arxiv.org/abs/2311.07911","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aime2025","benchmarkName":"American Invitational Mathematics Examination 2025","category":"math","score":86.5,"unit":"percent","family":"aime2025","index":45.94435487280012,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aime2026","benchmarkName":"AIME 2026","category":"math","score":86.5,"unit":"percent","family":"aime2026","index":47.56583149412873,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmluRedux","benchmarkName":"MMLU-Redux","category":"knowledge","score":84.7,"unit":"percent","family":"mmluRedux","index":33.84501265386743,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmluPro","benchmarkName":"Massive Multitask Language Understanding Professional","category":"knowledge","score":70.8,"unit":"percent","family":"mmlu-pro","index":33.680253884790424,"aboveFit":false,"run":null,"source":"Yubo Wang, Xueguang Ma, Ge Zhang, Yuansheng Ni, Abhranil Chandra, Shiguang Guo, Weiming Ren, Aaran Arulraj, Xuan He, Ziyan Jiang, Tianle Li, Max Ku, Kai Wang, Alex Zhuang, Rongqi Fan, Xiang Yue, Wenhu Chen","sourceUrl":"https://arxiv.org/abs/2406.01574","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gpqaDiamond","benchmarkName":"GPQA Diamond","category":"knowledge","score":70.2,"unit":"percent","family":"gpqa-diamond","index":41.59462251721844,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":70.2,"unit":"percent","family":"gpqa-diamond","index":41.59462251721844,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"liveCodeBenchV6","benchmarkName":"LiveCodeBench v6","category":"coding","score":69.1,"unit":"percent","family":"liveCodeBenchV6","index":41.152119971192754,"aboveFit":false,"run":null,"source":"LiveCodeBench maintainers","sourceUrl":"https://github.com/LiveCodeBench/LiveCodeBench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"bfclV4","benchmarkName":"Berkeley Function Calling Leaderboard v4","category":"agentic","score":66.6,"unit":"percent","family":"bfclV4","index":46.998183699416856,"aboveFit":false,"run":null,"source":"Shishir G. Patil et al.","sourceUrl":"https://gorilla.cs.berkeley.edu/leaderboard.html","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"ifBench","benchmarkName":"Instruction Following Benchmark","category":"instruction","score":66.3,"unit":"percent","family":"ifbench-family","index":50.98168315526095,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hmmtFeb2026","benchmarkName":"Harvard-MIT Mathematics Tournament February 2026","category":"math","score":63.8,"unit":"percent","family":"hmmtFeb2026","index":38.04912757670074,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":59,"unit":"percent","family":"lcr","index":47.77680046872466,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"sweVerified","benchmarkName":"Software Engineering Benchmark Verified","category":"coding","score":46.4,"unit":"percent","family":"swe-bench-verified","index":35.450599138634956,"aboveFit":false,"run":null,"source":"Carlos E. Jimenez, John Yang, Alexander Wettig, Shunyu Yao, Kexin Pei, Ofir Press, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2310.06770","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"longBenchV2","benchmarkName":"LongBench v2","category":"reasoning","score":43.7,"unit":"percent","family":"longBenchV2","index":32.17648427201577,"aboveFit":false,"run":null,"source":"LongBench v2 authors","sourceUrl":"https://arxiv.org/abs/2412.15204","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"superGpqa","benchmarkName":"SuperGPQA: Scaling LLM Evaluation Across 285 Graduate Disciplines","category":"knowledge","score":40.8,"unit":"percent","family":"superGpqa","index":31.013653294295437,"aboveFit":false,"run":null,"source":"Xiaoxuan Du, Yao Yao, Kexin Ma, Bowen Wang, Tianyu Zheng, Kaiyan Zhu, Yiming Zhang, Yutao Zhu, Jiawei Zhou, Jingren Zhou","sourceUrl":"https://arxiv.org/abs/2502.14739","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"sciCode","benchmarkName":"Scientific Code Benchmark","category":"coding","score":26.3,"unit":"percent","family":"scicode-family","index":30.126875593218756,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaSciCode","benchmarkName":"Artificial Analysis SciCode","category":"coding","score":26.3,"unit":"percent","family":"scicode-family","index":30.126875593218756,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/scicode","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"swePro","benchmarkName":"SWE-bench Pro","category":"coding","score":14.4,"unit":"percent","family":"swePro","index":21.10256740302049,"aboveFit":false,"run":null,"source":"Xiang Deng, Jeff Da, Edwin Pan, Yannis Yiming He, Charles Ide, Kanak Garg, Niklas Lauffer, Andrew Park, Nitin Pasari, Chetan Rane, Karmini Sampath, Maya Krishnan, Srivatsa Kundurthy, Sean Hendryx, Zifan Wang, Chen Bo Calvin Zhang, Noah Jacobson, Bing Liu, Brad Kenstler","sourceUrl":"https://arxiv.org/abs/2509.16941","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gdpvalAaNormalized","benchmarkName":"GDPval-AA normalized","category":"agentic","score":10.4,"unit":"percent","family":"gdpvalAaNormalized","index":41.46953323878784,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hle","benchmarkName":"Humanity's Last Exam","category":"knowledge","score":8.9,"unit":"percent","family":"hle","index":33.315949744078644,"aboveFit":false,"run":null,"source":"Center for AI Safety, Scale AI, and thousands of expert contributors","sourceUrl":"https://lastexam.ai/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":8.9,"unit":"percent","family":"hle","index":33.315949744078644,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"terminalBench21","benchmarkName":"Terminal-Bench 2.1","category":"agentic","score":8.6,"unit":"percent","family":"terminal-bench-2-1","index":26.170877341822642,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":0.3,"unit":"percent","family":"critpt","index":37.42988100778257,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":85,"benchmarkCount":21,"categoryCount":6,"confidence":"medium","state":"available"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"nemotron-3-5-lightning-30b-a3b-nvfp4","providerId":"nvidia","slug":"nemotron-3-5-lightning-30b-a3b-nvfp4","name":"Nemotron 3.5 Lightning 30B A3B NVFP4","provider":"NVIDIA","description":"Nemotron 3.5 Lightning 30B A3B NVFP4 is a reasoning model from NVIDIA in the Nemotron 3.5 Lightning family. 14 benchmarks count toward its index, in 5 categories.","logoUrl":null,"releaseDate":"2026-08-11","contextWindowTokens":1000000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":true,"overallScore":36.958800683327695,"overallScoreRange":{"low":30.137186401603792,"high":43.7804149650516},"badges":[],"capabilityScores":{"agentic":30.166842237240225,"coding":38.69496153308274,"reasoning":38.947733262751115,"multimodal":null,"knowledge":39.5010113400305,"multilingual":null,"instruction":57.13076300083908,"math":null},"capabilityCounts":{"agentic":5,"knowledge":4,"instruction":1,"coding":2,"reasoning":2},"benchmarks":[{"benchmarkId":"pinchBench","benchmarkName":"PinchBench","category":"agentic","score":83.43,"unit":"percent","family":"pinchBench","index":43.01564544601116,"aboveFit":false,"run":null,"source":"Kilo Code","sourceUrl":"https://pinchbench.com/about","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaOpennessIndex","benchmarkName":"Artificial Analysis Openness Index","category":"knowledge","score":83.3,"unit":"percent","family":"aaOpennessIndex","index":null,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-openness-index","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmluPro","benchmarkName":"Massive Multitask Language Understanding Professional","category":"knowledge","score":81.62,"unit":"percent","family":"mmlu-pro","index":47.312158764082184,"aboveFit":false,"run":null,"source":"Yubo Wang, Xueguang Ma, Ge Zhang, Yuansheng Ni, Abhranil Chandra, Shiguang Guo, Weiming Ren, Aaran Arulraj, Xuan He, Ziyan Jiang, Tianle Li, Max Ku, Kai Wang, Alex Zhuang, Rongqi Fan, Xiang Yue, Wenhu Chen","sourceUrl":"https://arxiv.org/abs/2406.01574","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gpqa","benchmarkName":"Graduate-Level Google-Proof Q&A","category":"knowledge","score":75.57,"unit":"percent","family":"gpqa-diamond","index":46.51983978153768,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gpqaDiamond","benchmarkName":"GPQA Diamond","category":"knowledge","score":75.57,"unit":"percent","family":"gpqa-diamond","index":46.51983978153768,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":74.3,"unit":"percent","family":"gpqa-diamond","index":45.35503048438956,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"ifBench","benchmarkName":"Instruction Following Benchmark","category":"instruction","score":72.88,"unit":"percent","family":"ifbench-family","index":57.13076300083908,"aboveFit":true,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"sweVerified","benchmarkName":"Software Engineering Benchmark Verified","category":"coding","score":52.8,"unit":"percent","family":"swe-bench-verified","index":40.170967636450584,"aboveFit":false,"run":null,"source":"Carlos E. Jimenez, John Yang, Alexander Wettig, Shunyu Yao, Kexin Pei, Ofir Press, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2310.06770","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":49.19,"unit":"percent","family":"lcr","index":41.046350512729326,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"browseComp","benchmarkName":"BrowseComp","category":"agentic","score":36.81,"unit":"percent","family":"browseComp","index":28.989623109432447,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaSciCode","benchmarkName":"Artificial Analysis SciCode","category":"coding","score":32.1,"unit":"percent","family":"scicode-family","index":37.68828424242421,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/scicode","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"sciCode","benchmarkName":"Scientific Code Benchmark","category":"coding","score":31.38,"unit":"percent","family":"scicode-family","index":36.749626617005596,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"terminalBench21","benchmarkName":"Terminal-Bench 2.1","category":"agentic","score":23.46,"unit":"percent","family":"terminal-bench-2-1","index":34.86182805789109,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"omniscienceAccuracy","benchmarkName":"Artificial Analysis Omniscience Accuracy","category":"knowledge","score":14.7,"unit":"percent","family":"omniscienceAccuracy","index":29.215713432299765,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":10.6,"unit":"percent","family":"hle","index":35.021547410629296,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hleNoTools","benchmarkName":"Humanity's Last Exam without tools","category":"knowledge","score":10.47,"unit":"percent","family":"hle","index":34.89111935377542,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"tau3Bench","benchmarkName":"τ³-Bench Tool-Agent-User Evaluation","category":"agentic","score":9.48,"unit":"percent","family":"tau3Bench","index":5,"aboveFit":false,"run":null,"source":"Sierra Research","sourceUrl":"https://github.com/sierra-research/tau2-bench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gdpvalAaNormalized","benchmarkName":"GDPval-AA normalized","category":"agentic","score":6.9,"unit":"percent","family":"gdpvalAaNormalized","index":38.96711457286643,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":0,"unit":"percent","family":"critpt","index":36.8491160127729,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":75,"benchmarkCount":14,"categoryCount":5,"confidence":"low","state":"available"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"ling-2-6-flash","providerId":"inclusionai","slug":"ling-2-6-flash","name":"Ling 2.6 Flash","provider":"InclusionAI","description":"Ling 2.6 Flash is a standard model from InclusionAI in the Ling 2.6 family. 9 benchmarks count toward its index, in 5 categories.","logoUrl":null,"releaseDate":"2026-04-21","contextWindowTokens":262000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":36.95295207419833,"overallScoreRange":{"low":29.06295207419833,"high":44.84295207419833},"badges":[],"capabilityScores":{"agentic":46.94525134389488,"coding":31.03945939570907,"reasoning":32.81074296138739,"multimodal":null,"knowledge":30.839065628103114,"multilingual":null,"instruction":42.477636560312476,"math":null},"capabilityCounts":{"agentic":2,"knowledge":3,"instruction":1,"reasoning":2,"coding":1},"benchmarks":[{"benchmarkId":"tau2Bench","benchmarkName":"τ²-Bench Tool-Agent-User Evaluation","category":"agentic","score":86,"unit":"percent","family":"tau2-bench","index":59.856727770596976,"aboveFit":true,"run":null,"source":"Victor Barres, Honghua Dong, Soham Ray, Xujie Si, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2506.07982","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":59.3,"unit":"percent","family":"gpqa-diamond","index":31.597440360592778,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gpqa","benchmarkName":"Graduate-Level Google-Proof Q&A","category":"knowledge","score":59,"unit":"percent","family":"gpqa-diamond","index":31.322288558116846,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaIfBench","benchmarkName":"Artificial Analysis IFBench","category":"instruction","score":57.4,"unit":"percent","family":"ifbench-family","index":42.66453868327837,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"ifBench","benchmarkName":"Instruction Following Benchmark","category":"instruction","score":57,"unit":"percent","family":"ifbench-family","index":42.29073443734657,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":31.3,"unit":"percent","family":"lcr","index":28.77236991000187,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"sciCode","benchmarkName":"Scientific Code Benchmark","category":"coding","score":27,"unit":"percent","family":"scicode-family","index":31.03945939570907,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"omniscienceAccuracy","benchmarkName":"Artificial Analysis Omniscience Accuracy","category":"knowledge","score":15.6,"unit":"percent","family":"omniscienceAccuracy","index":30.34994381795335,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":6.3,"unit":"percent","family":"hle","index":30.70738860700118,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gdpvalAaNormalized","benchmarkName":"GDPval-AA normalized","category":"agentic","score":0,"unit":"percent","family":"gdpvalAaNormalized","index":34.03377491719279,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":0,"unit":"percent","family":"critpt","index":36.8491160127729,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":104,"timeToFirstToken":1.07,"source":"BenchLM","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":75,"benchmarkCount":9,"categoryCount":5,"confidence":"low","state":"available"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"sarvam-105b","providerId":"sarvam","slug":"sarvam-105b","name":"Sarvam 105B","provider":"Sarvam","description":"Sarvam 105B is a reasoning model from Sarvam. 15 benchmarks count toward its index, in 6 categories.","logoUrl":null,"releaseDate":"2026-03-06","contextWindowTokens":128000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":true,"overallScore":36.71167484674234,"overallScoreRange":{"low":31.036598799530744,"high":42.38675089395394},"badges":[],"capabilityScores":{"agentic":36.429818115416936,"coding":34.418018529737786,"reasoning":22.073582735971804,"multimodal":null,"knowledge":42.421827983627715,"multilingual":null,"instruction":27.566649343639458,"math":46.629277788734825},"capabilityCounts":{"math":4,"knowledge":4,"instruction":2,"agentic":2,"coding":1,"reasoning":2},"benchmarks":[{"benchmarkId":"math500","benchmarkName":"MATH-500 Problem Set","category":"math","score":98.6,"unit":"percent","family":"math-500","index":48.32314300196345,"aboveFit":false,"run":null,"source":"Dan Hendrycks, Collin Burns, Saurav Kadavath, Akul Arora, Steven Basart, Eric Tang, Dawn Song, Jacob Steinhardt","sourceUrl":"https://arxiv.org/abs/2103.03874","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmlu","benchmarkName":"Massive Multitask Language Understanding","category":"knowledge","score":90.6,"unit":"percent","family":"mmlu","index":39.70798103705816,"aboveFit":false,"run":null,"source":"Dan Hendrycks, Collin Burns, Steven Basart, Andy Zou, Mantas Mazeika, Dawn Song, Jacob Steinhardt","sourceUrl":"https://arxiv.org/abs/2009.03300","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aime2025","benchmarkName":"American Invitational Mathematics Examination 2025","category":"math","score":88.3,"unit":"percent","family":"aime2025","index":46.92108134221112,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hmmtFeb2025","benchmarkName":"Harvard-MIT Mathematics Tournament February 2025","category":"math","score":85.8,"unit":"percent","family":"hmmtFeb2025","index":44.5517603604542,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hmmtNov2025","benchmarkName":"Harvard-MIT Mathematics Tournament November 2025","category":"math","score":85.8,"unit":"percent","family":"hmmtNov2025","index":46.72112645031052,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"ifeval","benchmarkName":"Instruction-Following Eval","category":"instruction","score":84.8,"unit":"percent","family":"ifeval","index":33.962504145079116,"aboveFit":false,"run":null,"source":"Jeffrey Zhou, Tianjian Lu, Swaroop Mishra, Siddhartha Brahma, Sujoy Basu, Yi Luan, Denny Zhou, Le Hou","sourceUrl":"https://arxiv.org/abs/2311.07911","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmluPro","benchmarkName":"Massive Multitask Language Understanding Professional","category":"knowledge","score":81.7,"unit":"percent","family":"mmlu-pro","index":47.412949188328334,"aboveFit":false,"run":null,"source":"Yubo Wang, Xueguang Ma, Ge Zhang, Yuansheng Ni, Abhranil Chandra, Shiguang Guo, Weiming Ren, Aaran Arulraj, Xuan He, Ziyan Jiang, Tianle Li, Max Ku, Kai Wang, Alex Zhuang, Rongqi Fan, Xiang Yue, Wenhu Chen","sourceUrl":"https://arxiv.org/abs/2406.01574","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gpqaDiamond","benchmarkName":"GPQA Diamond","category":"reasoning","score":78.7,"unit":"percent","family":"gpqa-diamond","index":49.39059025403662,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":73.8,"unit":"percent","family":"gpqa-diamond","index":44.89644414692966,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"browseComp","benchmarkName":"BrowseComp","category":"agentic","score":49.5,"unit":"percent","family":"browseComp","index":39.01497444756385,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"tau2Bench","benchmarkName":"τ²-Bench Tool-Agent-User Evaluation","category":"agentic","score":46.8,"unit":"percent","family":"tau2-bench","index":33.84466178327003,"aboveFit":false,"run":null,"source":"Victor Barres, Honghua Dong, Soham Ray, Xujie Si, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2506.07982","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"sweVerified","benchmarkName":"Software Engineering Benchmark Verified","category":"coding","score":45,"unit":"percent","family":"swe-bench-verified","index":34.418018529737786,"aboveFit":false,"run":null,"source":"Carlos E. Jimenez, John Yang, Alexander Wettig, Shunyu Yao, Kexin Pei, Ofir Press, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2310.06770","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaIfBench","benchmarkName":"Artificial Analysis IFBench","category":"instruction","score":34.4,"unit":"percent","family":"ifbench-family","index":21.170794542199793,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":11,"unit":"percent","family":"hle","index":35.42286450864121,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":0,"unit":"percent","family":"lcr","index":7.298049459170706,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":0,"unit":"percent","family":"critpt","index":36.8491160127729,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":191,"timeToFirstToken":12.11,"source":"BenchLM","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":85,"benchmarkCount":15,"categoryCount":6,"confidence":"medium","state":"available"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"qwq-32b","providerId":"alibaba","slug":"qwq-32b","name":"QwQ-32B","provider":"Alibaba","description":"QwQ-32B is a reasoning model from Alibaba. 13 benchmarks count toward its index, in 5 categories.","logoUrl":null,"releaseDate":"2025-03-06","contextWindowTokens":131000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":true,"overallScore":36.346240529794066,"overallScoreRange":{"low":26.86148944331309,"high":45.83099161627504},"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":25.616399492307835,"multimodal":null,"knowledge":38.42404436860277,"multilingual":33.01883581969889,"instruction":28.716318199342513,"math":41.88955108366844},"capabilityCounts":{"math":4,"knowledge":5,"instruction":2,"multilingual":1,"reasoning":1},"benchmarks":[{"benchmarkId":"math500","benchmarkName":"MATH-500 Problem Set","category":"math","score":98,"unit":"percent","family":"math-500","index":47.68908807450553,"aboveFit":false,"run":null,"source":"Dan Hendrycks, Collin Burns, Saurav Kadavath, Akul Arora, Steven Basart, Eric Tang, Dawn Song, Jacob Steinhardt","sourceUrl":"https://arxiv.org/abs/2103.03874","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmluRedux","benchmarkName":"MMLU-Redux","category":"knowledge","score":90,"unit":"percent","family":"mmluRedux","index":42.83568151941062,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"cEval","benchmarkName":"C-Eval","category":"knowledge","score":88.4,"unit":"percent","family":"cEval","index":43.65387952980562,"aboveFit":false,"run":null,"source":"C-Eval authors","sourceUrl":"https://arxiv.org/abs/2305.08322","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"ifeval","benchmarkName":"Instruction-Following Eval","category":"instruction","score":83.9,"unit":"percent","family":"ifeval","index":32.149995151235416,"aboveFit":false,"run":null,"source":"Jeffrey Zhou, Tianjian Lu, Swaroop Mishra, Siddhartha Brahma, Sujoy Basu, Yi Luan, Denny Zhou, Le Hou","sourceUrl":"https://arxiv.org/abs/2311.07911","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmmlu","benchmarkName":"MMMLU","category":"knowledge","score":80.9,"unit":"percent","family":"mmmlu","index":36.68165305681454,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aime2024","benchmarkName":"American Invitational Mathematics Examination 2024","category":"math","score":79.5,"unit":"percent","family":"aime2024","index":39.80749501603303,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"include","benchmarkName":"INCLUDE","category":"multilingual","score":69.7,"unit":"percent","family":"include","index":33.01883581969889,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aime2025","benchmarkName":"American Invitational Mathematics Examination 2025","category":"math","score":69.5,"unit":"percent","family":"aime2025","index":36.71971599502953,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"bfclV3","benchmarkName":"Berkeley Function Calling Leaderboard v3","category":"agentic","score":66.4,"unit":"percent","family":"bfclV3","index":null,"aboveFit":false,"run":null,"source":"Shishir G. Patil et al.","sourceUrl":"https://gorilla.cs.berkeley.edu/leaderboard.html","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gpqa","benchmarkName":"Graduate-Level Google-Proof Q&A","category":"knowledge","score":65.6,"unit":"percent","family":"gpqa-diamond","index":37.37562821258742,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":65.3409090909091,"unit":"percent","family":"gpqa-diamond","index":37.13799711044912,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.81399081227253,"costPerTest":null,"latency":null},{"benchmarkId":"liveCodeBenchV5","benchmarkName":"LiveCodeBench v5","category":"coding","score":62.7,"unit":"percent","family":"liveCodeBenchV5","index":null,"aboveFit":false,"run":null,"source":"LiveCodeBench maintainers","sourceUrl":"https://github.com/LiveCodeBench/LiveCodeBench","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":59.3,"unit":"percent","family":"gpqa-diamond","index":31.597440360592778,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":59.166666666666664,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":43.341905249105665,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":6.348963965431165,"costPerTest":null,"latency":null},{"benchmarkId":"aaIfBench","benchmarkName":"Artificial Analysis IFBench","category":"instruction","score":38.8,"unit":"percent","family":"ifbench-family","index":25.282641247449607,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":26.7,"unit":"percent","family":"lcr","index":25.616399492307835,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":7.4,"unit":"percent","family":"hle","index":31.811010626533953,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":{"elo":1329.0749139022373,"eloUpper":1333.5125821402876,"eloLower":1324.637245664187,"votes":25166,"rank":234,"rankUpper":null,"rankLower":null,"arenaName":"QwQ-32B","organization":"alibaba","updatedAt":"2026-11-09"},"quality":{"coveragePercent":50,"benchmarkCount":13,"categoryCount":5,"confidence":"low","state":"available"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"grok41fastnonreasoning","providerId":"xai","slug":"xai-grok-4-1-fast-non-reasoning","name":"Grok 4.1 Fast Non Reasoning","provider":"xAI","description":"Grok 4.1 Fast Non Reasoning is a model from xAI. 8 benchmarks count toward its index, in 5 categories.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":36.1714270506851,"overallScoreRange":{"low":27.95446872479186,"high":44.388385376578334},"badges":[],"capabilityScores":{"agentic":35.48609193452864,"coding":34.332077072391726,"reasoning":null,"multimodal":38.05147490366138,"knowledge":38.103050198397554,"multilingual":null,"instruction":null,"math":27.33907632004997},"capabilityCounts":{"knowledge":2,"multimodal":1,"coding":2,"math":1,"agentic":2},"benchmarks":[{"benchmarkId":"vals-ai:mgsm:overall","benchmarkName":"MGSM","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":87.564,"unit":"percent","family":"vals-ai:mgsm:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mgsm","measuredAt":"2026-01-09","stderr":2.036,"costPerTest":0.000147,"latency":3.683},{"benchmarkId":"vals-ai:legal_bench:overall","benchmarkName":"LegalBench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":79.113,"unit":"percent","family":"vals-ai:legal_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","measuredAt":"2026-10-13","stderr":0.46,"costPerTest":0.000091,"latency":1.757},{"benchmarkId":"vals-ai:medscribe:overall","benchmarkName":"MedScribe","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":77.464,"unit":"percent","family":"vals-ai:medscribe:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medscribe","measuredAt":"2026-10-29","stderr":2.04,"costPerTest":0.001782,"latency":19.865},{"benchmarkId":"vals-ai:medqa:overall","benchmarkName":"MedQA","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":76.025,"unit":"percent","family":"vals-ai:medqa:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medqa","measuredAt":"2026-04-16","stderr":0.39,"costPerTest":0.000165,"latency":6.552},{"benchmarkId":"vals-ai:mmlu_pro:overall","benchmarkName":"MMLU Pro","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":75.214,"unit":"percent","family":"mmlu-pro","index":39.24136554257174,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","measuredAt":"2026-09-01","stderr":0.419,"costPerTest":0.000217,"latency":3.098},{"benchmarkId":"vals-ai:gpqa:overall","benchmarkName":"GPQA Diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":65.152,"unit":"percent","family":"gpqa-diamond","index":36.964734854223366,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","measuredAt":"2026-09-01","stderr":2.398,"costPerTest":0.000445,"latency":8.66},{"benchmarkId":"vals-ai:mmmu:overall","benchmarkName":"MMMU Pro","category":"multimodal","domain":null,"eligible":true,"ranked":true,"score":63.699,"unit":"percent","family":"mmmu-pro","index":38.05147490366138,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmmu","measuredAt":"2026-09-01","stderr":1.156,"costPerTest":0.000628,"latency":51.426},{"benchmarkId":"vals-ai:tax_eval_v2:overall","benchmarkName":"TaxEval v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":61.856,"unit":"percent","family":"vals-ai:tax_eval_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","measuredAt":"2026-09-01","stderr":0.914,"costPerTest":0.000372,"latency":9.277},{"benchmarkId":"vals-ai:corp_fin_v2:overall","benchmarkName":"CorpFin v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":52.486,"unit":"percent","family":"vals-ai:corp_fin_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","measuredAt":"2026-08-12","stderr":0.971,"costPerTest":0.015705,"latency":13.465},{"benchmarkId":"vals-ai:finance_agent:overall","benchmarkName":"Finance Agent v1.1","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":44.362,"unit":"percent","family":"vals-ai:finance_agent:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/finance_agent","measuredAt":"2026-06-04","stderr":2.738,"costPerTest":0.089182,"latency":93.211},{"benchmarkId":"vals-ai:lcb:overall","benchmarkName":"LiveCodeBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":42.622,"unit":"percent","family":"livecodebench","index":24.70899716605352,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","measuredAt":"2026-09-01","stderr":1.092,"costPerTest":0.000436,"latency":11.962},{"benchmarkId":"vals-ai:mortgage_tax:overall","benchmarkName":"MortgageTax","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":34.26,"unit":"percent","family":"vals-ai:mortgage_tax:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mortgage_tax","measuredAt":"2026-09-01","stderr":0.938,"costPerTest":0.001018,"latency":8.886},{"benchmarkId":"vals-ai:medcode:overall","benchmarkName":"MedCode","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":28.349,"unit":"percent","family":"vals-ai:medcode:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medcode","measuredAt":"2026-10-29","stderr":1.921,"costPerTest":0.002193,"latency":4.496},{"benchmarkId":"vals-ai:aime:overall","benchmarkName":"AIME","category":"math","domain":null,"eligible":true,"ranked":true,"score":27.083,"unit":"percent","family":"vals-ai:aime:overall","index":27.33907632004997,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/aime","measuredAt":"2026-04-16","stderr":1.935,"costPerTest":0.000788,"latency":17.366},{"benchmarkId":"vals-ai:terminal-bench:overall","benchmarkName":"Terminal-Bench 1.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":21.25,"unit":"percent","family":"vals-ai:terminal-bench:overall","index":33.59616319989902,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench","measuredAt":"2026-01-12","stderr":4.574,"costPerTest":0.02719,"latency":133.157},{"benchmarkId":"vals-ai:terminal-bench-2:overall","benchmarkName":"Terminal-Bench 2.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":17.978,"unit":"percent","family":"vals-ai:terminal-bench-2:overall","index":37.37602066915826,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2","measuredAt":"2026-06-04","stderr":4.093,"costPerTest":0.032811,"latency":253.172},{"benchmarkId":"vals-ai:ioi-v1:overall","benchmarkName":"IOI v1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":7.667,"unit":"percent","family":"vals-ai:ioi-v1:overall","index":43.95515697872993,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/ioi-v1","measuredAt":"2026-08-09","stderr":3.198,"costPerTest":0.06955,"latency":7675.292}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":75,"benchmarkCount":8,"categoryCount":5,"confidence":"low","state":"available"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"llama4maverickinstructbasic","providerId":"meta","slug":"meta-llama4-maverick-instruct-basic","name":"Llama4 Maverick Instruct Basic","provider":"Meta","description":"Llama4 Maverick Instruct Basic is a model from Meta. 8 benchmarks count toward its index, in 5 categories.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":35.99194385056438,"overallScoreRange":{"low":27.774985524671145,"high":44.20890217645762},"badges":[],"capabilityScores":{"agentic":29.393279740542543,"coding":28.5498693151072,"reasoning":null,"multimodal":48.454964551023984,"knowledge":42.717681335388384,"multilingual":null,"instruction":null,"math":30.341099237318947},"capabilityCounts":{"math":2,"knowledge":2,"multimodal":1,"coding":1,"agentic":2},"benchmarks":[{"benchmarkId":"vals-ai:mgsm:overall","benchmarkName":"MGSM","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":92.436,"unit":"percent","family":"vals-ai:mgsm:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mgsm","measuredAt":"2026-01-09","stderr":1.635,"costPerTest":0.000218,"latency":2.714},{"benchmarkId":"vals-ai:math500:overall","benchmarkName":"MATH 500","category":"math","domain":null,"eligible":true,"ranked":true,"score":85.2,"unit":"percent","family":"math-500","index":34.16258295540306,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/math500","measuredAt":"2026-01-09","stderr":1.588,"costPerTest":0.00059,"latency":7.469},{"benchmarkId":"vals-ai:mmlu_pro:overall","benchmarkName":"MMLU Pro","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":79.415,"unit":"percent","family":"mmlu-pro","index":44.53412269579769,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","measuredAt":"2026-09-01","stderr":0.395,"costPerTest":0.000686,"latency":6.803},{"benchmarkId":"vals-ai:legal_bench:overall","benchmarkName":"LegalBench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":77.812,"unit":"percent","family":"vals-ai:legal_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","measuredAt":"2026-10-13","stderr":0.42,"costPerTest":null,"latency":0.472},{"benchmarkId":"vals-ai:mmmu:overall","benchmarkName":"MMMU Pro","category":"multimodal","domain":null,"eligible":true,"ranked":true,"score":71.693,"unit":"percent","family":"mmmu-pro","index":48.454964551023984,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmmu","measuredAt":"2026-09-01","stderr":1.083,"costPerTest":0.001078,"latency":7.908},{"benchmarkId":"vals-ai:gpqa:overall","benchmarkName":"GPQA Diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":69.444,"unit":"percent","family":"gpqa-diamond","index":40.901239974979084,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","measuredAt":"2026-09-01","stderr":2.35,"costPerTest":0.000964,"latency":10.071},{"benchmarkId":"vals-ai:tax_eval_v2:overall","benchmarkName":"TaxEval v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":66.558,"unit":"percent","family":"vals-ai:tax_eval_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","measuredAt":"2026-09-01","stderr":0.919,"costPerTest":0.000527,"latency":25.143},{"benchmarkId":"vals-ai:mortgage_tax:overall","benchmarkName":"MortgageTax","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":58.506,"unit":"percent","family":"vals-ai:mortgage_tax:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mortgage_tax","measuredAt":"2026-09-01","stderr":0.967,"costPerTest":0.000752,"latency":2.716},{"benchmarkId":"vals-ai:medscribe:overall","benchmarkName":"MedScribe","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":54.219,"unit":"percent","family":"vals-ai:medscribe:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medscribe","measuredAt":"2026-10-29","stderr":1.871,"costPerTest":0.00246,"latency":25.048},{"benchmarkId":"vals-ai:corp_fin_v2:overall","benchmarkName":"CorpFin v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":49.728,"unit":"percent","family":"vals-ai:corp_fin_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","measuredAt":"2026-08-12","stderr":0.981,"costPerTest":0.014178,"latency":4.724},{"benchmarkId":"vals-ai:lcb:overall","benchmarkName":"LiveCodeBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":47.251,"unit":"percent","family":"livecodebench","index":28.5498693151072,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","measuredAt":"2026-09-01","stderr":1.169,"costPerTest":0.000832,"latency":9.442},{"benchmarkId":"vals-ai:medqa:overall","benchmarkName":"MedQA","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":43.3,"unit":"percent","family":"vals-ai:medqa:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medqa","measuredAt":"2026-04-16","stderr":0.452,"costPerTest":0.000687,"latency":7.751},{"benchmarkId":"vals-ai:medcode:overall","benchmarkName":"MedCode","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":36.514,"unit":"percent","family":"vals-ai:medcode:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medcode","measuredAt":"2026-10-29","stderr":1.994,"costPerTest":0.002888,"latency":21.241},{"benchmarkId":"vals-ai:aime:overall","benchmarkName":"AIME","category":"math","domain":null,"eligible":true,"ranked":true,"score":25.208,"unit":"percent","family":"vals-ai:aime:overall","index":26.519615519234833,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/aime","measuredAt":"2026-04-16","stderr":0.784,"costPerTest":0.001142,"latency":15.502},{"benchmarkId":"vals-ai:terminal-bench:overall","benchmarkName":"Terminal-Bench 1.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":17.5,"unit":"percent","family":"vals-ai:terminal-bench:overall","index":30.882350973914395,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench","measuredAt":"2026-01-12","stderr":4.248,"costPerTest":0.03501,"latency":55.424},{"benchmarkId":"vals-ai:terminal-bench-2:overall","benchmarkName":"Terminal-Bench 2.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":2.247,"unit":"percent","family":"vals-ai:terminal-bench-2:overall","index":27.904208507170686,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2","measuredAt":"2026-06-04","stderr":1.58,"costPerTest":0.03402,"latency":1193.58}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":75,"benchmarkCount":8,"categoryCount":5,"confidence":"low","state":"available"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"nemotron-ultra-253b","providerId":"nvidia","slug":"nemotron-ultra-253b","name":"Nemotron Ultra 253B","provider":"NVIDIA","description":"Nemotron Ultra 253B is a reasoning model from NVIDIA. 9 benchmarks count toward its index, in 6 categories.","logoUrl":null,"releaseDate":"2025-04-07","contextWindowTokens":32000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":true,"overallScore":35.92070583401043,"overallScoreRange":{"low":29.030705834010426,"high":42.81070583401043},"badges":[],"capabilityScores":{"agentic":10.354173621245181,"coding":44.36390739867041,"reasoning":36.8491160127729,"multimodal":null,"knowledge":38.631172120895044,"multilingual":null,"instruction":34.02453441257844,"math":42.48996165306176},"capabilityCounts":{"math":2,"instruction":2,"knowledge":2,"coding":1,"agentic":1,"reasoning":1},"benchmarks":[{"benchmarkId":"math500","benchmarkName":"MATH-500 Problem Set","category":"math","score":97,"unit":"percent","family":"math-500","index":46.63232986207564,"aboveFit":false,"run":null,"source":"Dan Hendrycks, Collin Burns, Saurav Kadavath, Akul Arora, Steven Basart, Eric Tang, Dawn Song, Jacob Steinhardt","sourceUrl":"https://arxiv.org/abs/2103.03874","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"ifeval","benchmarkName":"Instruction-Following Eval","category":"instruction","score":89.45,"unit":"percent","family":"ifeval","index":43.327133946604974,"aboveFit":false,"run":null,"source":"Jeffrey Zhou, Tianjian Lu, Swaroop Mishra, Siddhartha Brahma, Sujoy Basu, Yi Luan, Denny Zhou, Le Hou","sourceUrl":"https://arxiv.org/abs/2311.07911","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gpqa","benchmarkName":"Graduate-Level Google-Proof Q&A","category":"knowledge","score":76.01,"unit":"percent","family":"gpqa-diamond","index":46.923395758502394,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":72.8,"unit":"percent","family":"gpqa-diamond","index":43.97927147200988,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aime2025","benchmarkName":"American Invitational Mathematics Examination 2025","category":"math","score":72.5,"unit":"percent","family":"aime2025","index":38.347593444047874,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"liveCodeBench","benchmarkName":"LiveCodeBench: Holistic and Contamination Free Evaluation of Large Language Models for Code","category":"coding","score":66.31,"unit":"percent","family":"livecodebench","index":44.36390739867041,"aboveFit":false,"run":null,"source":"Naman Jain, King Han, Alex Gu, Wen-Ding Li, Fanjia Yan, Tianjun Zhang, Sida Wang, Armando Solar-Lezama, Koushik Sen, Ion Stoica","sourceUrl":"https://arxiv.org/abs/2403.07974","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaIfBench","benchmarkName":"Artificial Analysis IFBench","category":"instruction","score":38.2,"unit":"percent","family":"ifbench-family","index":24.721934878551913,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"tau2Bench","benchmarkName":"τ²-Bench Tool-Agent-User Evaluation","category":"agentic","score":11.4,"unit":"percent","family":"tau2-bench","index":10.354173621245181,"aboveFit":false,"run":null,"source":"Victor Barres, Honghua Dong, Soham Ray, Xujie Si, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2506.07982","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":7.4,"unit":"percent","family":"hle","index":31.811010626533953,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":0,"unit":"percent","family":"critpt","index":36.8491160127729,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":85,"benchmarkCount":9,"categoryCount":6,"confidence":"low","state":"available"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gemini-2-5-flash","providerId":"google","slug":"gemini-2-5-flash","name":"Gemini 2.5 Flash","provider":"Google","description":"Gemini 2.5 Flash is a standard model from Google. 13 benchmarks count toward its index, in 7 categories.","logoUrl":null,"releaseDate":"2025-06-17","contextWindowTokens":1048576,"modalities":["file","image","text","audio","video"],"openWeights":false,"details":{"apiId":"google/gemini-2.5-flash","maxOutputTokens":65535,"knowledgeCutoff":"2025-01-31","retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.03,"weightsUrl":null},"reasoning":false,"overallScore":35.006915133549754,"overallScoreRange":{"low":30.022164047068777,"high":39.99166622003073},"badges":[],"capabilityScores":{"agentic":12.6766795129708,"coding":31.928732472933937,"reasoning":40.54641017102325,"multimodal":40.395318393141,"knowledge":37.512248766800944,"multilingual":null,"instruction":25.46954337041551,"math":44.133637979217326},"capabilityCounts":{"math":3,"knowledge":3,"multimodal":1,"reasoning":2,"instruction":1,"coding":2,"agentic":1},"benchmarks":[{"benchmarkId":"vals-ai:medscribe:overall","benchmarkName":"MedScribe","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":82.869,"unit":"percent","family":"vals-ai:medscribe:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medscribe","measuredAt":"2026-10-29","stderr":1.909,"costPerTest":0.014869,"latency":22.786},{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":71.94444444444446,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":50.77594027791733,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":5.532781604070319,"costPerTest":null,"latency":null},{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":68.3,"unit":"percent","family":"gpqa-diamond","index":39.851994434870846,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaMmmuPro","benchmarkName":"Artificial Analysis MMMU-Pro","category":"multimodal","score":65.5,"unit":"percent","family":"mmmu-pro","index":40.395318393141,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmmu-pro","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":49.9,"unit":"percent","family":"lcr","index":41.53346768589515,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaIfBench","benchmarkName":"Artificial Analysis IFBench","category":"instruction","score":39,"unit":"percent","family":"ifbench-family","index":25.46954337041551,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:medcode:overall","benchmarkName":"MedCode","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":38.425,"unit":"percent","family":"vals-ai:medcode:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medcode","measuredAt":"2026-10-29","stderr":1.923,"costPerTest":0.003698,"latency":22.145},{"benchmarkId":"swe-bench:swe-bench-verified:overall","benchmarkName":"SWE-bench Verified","category":"coding","domain":null,"eligible":true,"ranked":true,"score":28.73,"unit":"percent","family":"swe-bench-verified","index":22.41795673919712,"aboveFit":false,"run":"mini-SWE-agent","source":"SWE-bench team","sourceUrl":"https://www.swebench.com/","measuredAt":"2026-02-26","stderr":null,"costPerTest":0.13388571835,"latency":null},{"benchmarkId":"swebench-experiments:swe-bench-verified:overall","benchmarkName":"SWE-bench Verified","category":"coding","domain":null,"eligible":true,"ranked":true,"score":28.73,"unit":"percent","family":"swe-bench-verified","index":22.41795673919712,"aboveFit":false,"run":"mini-SWE-agent","source":"SWE-bench team","sourceUrl":"https://github.com/SWE-bench/experiments/tree/main/evaluation/verified","measuredAt":"2026-09-01","stderr":null,"costPerTest":0.13388571835,"latency":null},{"benchmarkId":"omniscienceAccuracy","benchmarkName":"Artificial Analysis Omniscience Accuracy","category":"knowledge","score":26.1,"unit":"percent","family":"omniscienceAccuracy","index":43.582631650578485,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"tau2Bench","benchmarkName":"τ²-Bench Tool-Agent-User Evaluation","category":"agentic","score":14.9,"unit":"percent","family":"tau2-bench","index":12.6766795129708,"aboveFit":false,"run":null,"source":"Victor Barres, Honghua Dong, Soham Ray, Xujie Si, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2506.07982","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-2025-02-28-private:overall","benchmarkName":"FrontierMath-2025-02-28-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":4.844290657439446,"unit":"percent","family":"epoch-ai:epoch-frontiermath-2025-02-28-private:overall","index":34.19060476502137,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.265133273845462,"costPerTest":null,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":4.7,"unit":"percent","family":"hle","index":29.10212021495351,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-private:overall","benchmarkName":"FrontierMath-Tier-4-2025-07-01-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":4.166666666666666,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-private:overall","index":47.43436889471329,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.914766351555603,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:ioi-v1:overall","benchmarkName":"IOI v1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":2.611,"unit":"percent","family":"vals-ai:ioi-v1:overall","index":41.43950820667075,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/ioi-v1","measuredAt":"2026-08-09","stderr":1.03,"costPerTest":0.036998,"latency":350.186},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":1.4,"unit":"percent","family":"critpt","index":39.55935265615135,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":0.3,"outputPerMillion":2.5,"batch":{"inputPerMillion":0.15,"outputPerMillion":1.25},"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":95,"timeToFirstToken":0.593,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":{"elo":1416.987722764275,"eloUpper":1419.3948721866486,"eloLower":1414.5805733419015,"votes":124953,"rank":129,"rankUpper":null,"rankLower":null,"arenaName":"Gemini 2.5 Flash","organization":"google","updatedAt":"2026-11-09"},"quality":{"coveragePercent":95,"benchmarkCount":13,"categoryCount":7,"confidence":"medium","state":"available"},"hosts":[{"host":"databricks","name":"Databricks","kind":"cloud","url":"https://docs.databricks.com/aws/en/machine-learning/foundation-models/","inputPerMillion":0.3,"outputPerMillion":2.5,"cacheReadPerMillion":0.03,"contextTokens":1048576,"maxOutputTokens":65536,"deprecated":true},{"host":"google-ai-studio","name":"Google AI Studio","kind":"maker","url":"https://ai.google.dev/gemini-api/docs/models","inputPerMillion":0.3,"outputPerMillion":2.5,"cacheReadPerMillion":0.03,"contextTokens":1048576,"maxOutputTokens":65536,"deprecated":false},{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":0.3,"outputPerMillion":2.5,"cacheReadPerMillion":0.03,"contextTokens":1048576,"maxOutputTokens":65535,"deprecated":true},{"host":"sap-ai-core","name":"SAP AI Core","kind":"cloud","url":"https://help.sap.com/docs/sap-ai-core","inputPerMillion":0.3,"outputPerMillion":2.5,"cacheReadPerMillion":0.03,"contextTokens":1048576,"maxOutputTokens":65536,"deprecated":true}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:swe-bench-team","name":"SWE-bench team","url":"https://www.swebench.com","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gpt-4-1-mini","providerId":"openai","slug":"gpt-4-1-mini","name":"GPT-4.1 mini","provider":"OpenAI","description":"GPT-4.1 mini is a standard model from OpenAI in the GPT-4.1 family. 25 benchmarks count toward its index, in 7 categories.","logoUrl":null,"releaseDate":"2025-04-14","contextWindowTokens":1047576,"modalities":["image","text","file"],"openWeights":false,"details":{"apiId":"openai/gpt-4.1-mini","maxOutputTokens":32768,"knowledgeCutoff":"2024-06-30","retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.09999999999999999,"weightsUrl":null},"reasoning":false,"overallScore":34.52227436350671,"overallScoreRange":{"low":30.78827436350671,"high":38.25627436350671},"badges":[],"capabilityScores":{"agentic":35.299543229670704,"coding":28.242456420726654,"reasoning":36.82754210330863,"multimodal":39.24812348143847,"knowledge":35.968736864062414,"multilingual":null,"instruction":33.11465797434684,"math":36.469532217742284},"capabilityCounts":{"instruction":2,"math":6,"knowledge":6,"multimodal":1,"agentic":2,"coding":2,"reasoning":6},"benchmarks":[{"benchmarkId":"ifeval","benchmarkName":"Instruction-Following Eval","category":"instruction","score":88.5,"unit":"percent","family":"ifeval","index":41.413930008658824,"aboveFit":false,"run":null,"source":"Jeffrey Zhou, Tianjian Lu, Swaroop Mishra, Siddhartha Brahma, Sujoy Basu, Yi Luan, Denny Zhou, Le Hou","sourceUrl":"https://arxiv.org/abs/2311.07911","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:math500:overall","benchmarkName":"MATH 500","category":"math","domain":null,"eligible":true,"ranked":true,"score":88,"unit":"percent","family":"math-500","index":37.12150595020672,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/math500","measuredAt":"2026-01-09","stderr":1.453,"costPerTest":0.00122,"latency":5.76},{"benchmarkId":"vals-ai:mgsm:overall","benchmarkName":"MGSM","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":87.782,"unit":"percent","family":"vals-ai:mgsm:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mgsm","measuredAt":"2026-01-09","stderr":1.928,"costPerTest":0.000394,"latency":2.748},{"benchmarkId":"mmlu","benchmarkName":"Massive Multitask Language Understanding","category":"knowledge","score":87.5,"unit":"percent","family":"mmlu","index":36.00661693616388,"aboveFit":false,"run":null,"source":"Dan Hendrycks, Collin Burns, Steven Basart, Andy Zou, Mantas Mazeika, Dawn Song, Jacob Steinhardt","sourceUrl":"https://arxiv.org/abs/2009.03300","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-math-level-5:overall","benchmarkName":"MATH level 5","category":"math","domain":null,"eligible":true,"ranked":true,"score":87.29229607250755,"unit":"percent","family":"epoch-ai:epoch-math-level-5:overall","index":41.4006133814397,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0.7298484758774334,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:medqa:overall","benchmarkName":"MedQA","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":84.633,"unit":"percent","family":"vals-ai:medqa:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medqa","measuredAt":"2026-04-16","stderr":0.329,"costPerTest":0.000519,"latency":1.858},{"benchmarkId":"vals-ai:legal_bench:overall","benchmarkName":"LegalBench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":78.044,"unit":"percent","family":"vals-ai:legal_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","measuredAt":"2026-10-13","stderr":0.419,"costPerTest":null,"latency":0.467},{"benchmarkId":"vals-ai:mmlu_pro:overall","benchmarkName":"MMLU Pro","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":77.225,"unit":"percent","family":"mmlu-pro","index":41.774984832059324,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","measuredAt":"2026-09-01","stderr":0.405,"costPerTest":0.001316,"latency":3.832},{"benchmarkId":"vals-ai:tax_eval_v2:overall","benchmarkName":"TaxEval v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":71.914,"unit":"percent","family":"vals-ai:tax_eval_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","measuredAt":"2026-09-01","stderr":0.881,"costPerTest":0.001233,"latency":5.921},{"benchmarkId":"vals-ai:mmmu:overall","benchmarkName":"MMMU Pro","category":"multimodal","domain":null,"eligible":true,"ranked":true,"score":70.537,"unit":"percent","family":"mmmu-pro","index":46.95053197254624,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmmu","measuredAt":"2026-09-01","stderr":1.096,"costPerTest":0.001788,"latency":8.225},{"benchmarkId":"vals-ai:gpqa:overall","benchmarkName":"GPQA Diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":67.929,"unit":"percent","family":"gpqa-diamond","index":39.51172337247561,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","measuredAt":"2026-09-01","stderr":2.351,"costPerTest":0.002324,"latency":9.282},{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":66.4,"unit":"percent","family":"gpqa-diamond","index":38.10936635252326,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":65.84595959595958,"unit":"percent","family":"gpqa-diamond","index":37.60121563313587,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.7051500965295916,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:mortgage_tax:overall","benchmarkName":"MortgageTax","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":65.501,"unit":"percent","family":"vals-ai:mortgage_tax:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mortgage_tax","measuredAt":"2026-09-01","stderr":0.938,"costPerTest":0.002797,"latency":4.598},{"benchmarkId":"gpqa","benchmarkName":"Graduate-Level Google-Proof Q&A","category":"knowledge","score":64.2,"unit":"percent","family":"gpqa-diamond","index":36.09158646769973,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"tau2-bench:retail:overall","benchmarkName":"τ²-bench Retail","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":61.4,"unit":"percent","family":"tau2-bench","index":43.53282921732547,"aboveFit":false,"run":"Sierra","source":"Sierra Research","sourceUrl":"https://github.com/sierra-research/tau2-bench","measuredAt":"2026-04-30","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaMmmuPro","benchmarkName":"Artificial Analysis MMMU-Pro","category":"multimodal","score":58.7,"unit":"percent","family":"mmmu-pro","index":31.54571499033068,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmmu-pro","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:lcb:overall","benchmarkName":"LiveCodeBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":58.158,"unit":"percent","family":"livecodebench","index":37.59985689984008,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","measuredAt":"2026-09-01","stderr":1.181,"costPerTest":0.001178,"latency":27.592},{"benchmarkId":"vals-ai:corp_fin_v2:overall","benchmarkName":"CorpFin v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":57.926,"unit":"percent","family":"vals-ai:corp_fin_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","measuredAt":"2026-08-12","stderr":0.972,"costPerTest":0.026113,"latency":6.443},{"benchmarkId":"tau2Bench","benchmarkName":"τ²-Bench Tool-Agent-User Evaluation","category":"agentic","score":52.9,"unit":"percent","family":"tau2-bench","index":37.89245776599182,"aboveFit":false,"run":null,"source":"Victor Barres, Honghua Dong, Soham Ray, Xujie Si, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2506.07982","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:aime:overall","benchmarkName":"AIME","category":"math","domain":null,"eligible":true,"ranked":true,"score":49.375,"unit":"percent","family":"vals-ai:aime:overall","index":37.08170041166117,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/aime","measuredAt":"2026-04-16","stderr":1.122,"costPerTest":0.0079,"latency":33.139},{"benchmarkId":"tau2-bench:telecom:overall","benchmarkName":"τ²-bench Telecom","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":48.9,"unit":"percent","family":"tau2-bench","index":35.2381653183054,"aboveFit":false,"run":"Sierra","source":"Sierra Research","sourceUrl":"https://github.com/sierra-research/tau2-bench","measuredAt":"2026-03-02","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"tau2-bench:airline:overall","benchmarkName":"τ²-bench Airline","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":48.7,"unit":"percent","family":"tau2-bench","index":35.10545069592108,"aboveFit":false,"run":"Sierra","source":"Sierra Research","sourceUrl":"https://github.com/sierra-research/tau2-bench","measuredAt":"2026-03-02","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":44.72222222222222,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":34.938213477405526,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":6.148017573965347,"costPerTest":null,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":44,"unit":"percent","family":"lcr","index":37.48559258493975,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaIfBench","benchmarkName":"Artificial Analysis IFBench","category":"instruction","score":38.3,"unit":"percent","family":"ifbench-family","index":24.815385940034858,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"swe-bench:swe-bench-verified:overall","benchmarkName":"SWE-bench Verified","category":"coding","domain":null,"eligible":true,"ranked":true,"score":23.94,"unit":"percent","family":"swe-bench-verified","index":18.88505594161323,"aboveFit":false,"run":"mini-SWE-agent","source":"SWE-bench team","sourceUrl":"https://www.swebench.com/","measuredAt":"2026-02-26","stderr":null,"costPerTest":0.43515415279999997,"latency":null},{"benchmarkId":"swebench-experiments:swe-bench-verified:overall","benchmarkName":"SWE-bench Verified","category":"coding","domain":null,"eligible":true,"ranked":true,"score":23.94,"unit":"percent","family":"swe-bench-verified","index":18.88505594161323,"aboveFit":false,"run":"mini-SWE-agent","source":"SWE-bench team","sourceUrl":"https://github.com/SWE-bench/experiments/tree/main/evaluation/verified","measuredAt":"2026-09-01","stderr":null,"costPerTest":0.43515415279999997,"latency":null},{"benchmarkId":"sweVerified","benchmarkName":"Software Engineering Benchmark Verified","category":"coding","score":23.6,"unit":"percent","family":"swe-bench-verified","index":18.634286365166783,"aboveFit":false,"run":null,"source":"Carlos E. Jimenez, John Yang, Alexander Wettig, Shunyu Yao, Kexin Pei, Ofir Press, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2310.06770","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"omniscienceAccuracy","benchmarkName":"Artificial Analysis Omniscience Accuracy","category":"knowledge","score":20.3,"unit":"percent","family":"omniscienceAccuracy","index":36.27314694303317,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-simpleqa-verified:overall","benchmarkName":"SimpleQA Verified","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":12.7,"unit":"percent","family":"epoch-ai:epoch-simpleqa-verified:overall","index":34.499273441826084,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.0534798620855645,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-2025-02-28-public:overall","benchmarkName":"FrontierMath-2025-02-28-Public","category":"math","domain":null,"eligible":false,"ranked":true,"score":10,"unit":"percent","family":"epoch-ai:epoch-frontiermath-2025-02-28-public:overall","index":null,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":10,"costPerTest":null,"latency":null},{"benchmarkId":"arc-prize:v1-public-eval:overall","benchmarkName":"ARC-AGI-1 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":7.249999999999999,"unit":"percent","family":"arc-prize:v1-public-eval:overall","index":null,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.0066,"latency":null},{"benchmarkId":"epoch-ai:epoch-mystery-game-puzzles:overall","benchmarkName":"Mystery Game Puzzles","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":7.000000000000001,"unit":"percent","family":"epoch-ai:epoch-mystery-game-puzzles:overall","index":44.1019024790064,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.564323999762428,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tiers-1-3-v2-private:overall","benchmarkName":"FrontierMath-Tiers-1-3-v2-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":6.666666666666667,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tiers-1-3-v2-private:overall","index":34.40370771398087,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.480176785956286,"costPerTest":null,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":5,"unit":"percent","family":"hle","index":29.403108038462445,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-2025-02-28-private:overall","benchmarkName":"FrontierMath-2025-02-28-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":4.482758620689655,"unit":"percent","family":"epoch-ai:epoch-frontiermath-2025-02-28-private:overall","index":33.87145237175973,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.217207560947464,"costPerTest":null,"latency":null},{"benchmarkId":"arc-prize:v1-semi-private:overall","benchmarkName":"ARC-AGI-1 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":3.5000000000000004,"unit":"percent","family":"arc-prize:v1-semi-private:overall","index":28.5964352206012,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.0078,"latency":null},{"benchmarkId":"epoch-ai:epoch-chess-puzzles:overall","benchmarkName":"Chess Puzzles","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":1,"unit":"percent","family":"epoch-ai:epoch-chess-puzzles:overall","index":34.74369092028902,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0.9999999999999999,"costPerTest":null,"latency":null},{"benchmarkId":"gdpvalAaNormalized","benchmarkName":"GDPval-AA normalized","category":"agentic","score":0,"unit":"percent","family":"gdpvalAaNormalized","index":34.03377491719279,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":0,"unit":"percent","family":"critpt","index":36.8491160127729,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"arc-prize:v2-public-eval:overall","benchmarkName":"ARC-AGI-2 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":0,"unit":"percent","family":"arc-prize:v2-public-eval:overall","index":null,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.0136,"latency":null},{"benchmarkId":"arc-prize:v2-semi-private:overall","benchmarkName":"ARC-AGI-2 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":0,"unit":"percent","family":"arc-prize:v2-semi-private:overall","index":39.188515402242544,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.0139,"latency":null}],"price":{"inputPerMillion":0.39999999999999997,"outputPerMillion":1.5999999999999999,"batch":{"inputPerMillion":0.19999999999999998,"outputPerMillion":0.7999999999999999},"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":72,"timeToFirstToken":0.95,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":{"elo":1340.5315219529643,"eloUpper":1344.8401826417094,"eloLower":1336.2228612642189,"votes":38657,"rank":216,"rankUpper":null,"rankLower":null,"arenaName":"GPT-4.1 mini-2025-04-14","organization":"openai","updatedAt":"2026-11-09"},"quality":{"coveragePercent":95,"benchmarkCount":25,"categoryCount":7,"confidence":"medium","state":"available"},"hosts":[{"host":"azure","name":"Microsoft Azure","kind":"cloud","url":"https://learn.microsoft.com/en-us/azure/ai-services/openai/concepts/models","inputPerMillion":0.4,"outputPerMillion":1.6,"cacheReadPerMillion":0.1,"contextTokens":1047576,"maxOutputTokens":32768,"deprecated":true},{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":0.4,"outputPerMillion":1.6,"cacheReadPerMillion":0.1,"contextTokens":1047576,"maxOutputTokens":32768,"deprecated":false},{"host":"sap-ai-core","name":"SAP AI Core","kind":"cloud","url":"https://help.sap.com/docs/sap-ai-core","inputPerMillion":0.4,"outputPerMillion":1.6,"cacheReadPerMillion":0.1,"contextTokens":1047576,"maxOutputTokens":32768,"deprecated":true}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:sierra-research","name":"Sierra Research","url":"https://github.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:swe-bench-team","name":"SWE-bench team","url":"https://www.swebench.com","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:arc-prize-foundation","name":"ARC Prize Foundation","url":"https://arcprize.org","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"llama-4-maverick","providerId":"meta","slug":"llama-4-maverick","name":"Llama 4 Maverick","provider":"Meta","description":"Llama 4 Maverick is a standard model from Meta. 15 benchmarks count toward its index, in 7 categories.","logoUrl":null,"releaseDate":"2025-04-05","contextWindowTokens":524288,"modalities":["text","image"],"openWeights":true,"details":{"apiId":"meta-llama/llama-4-maverick","maxOutputTokens":32768,"knowledgeCutoff":"2024-08-31","retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.16999999999999998,"weightsUrl":"https://huggingface.co/meta-llama/Llama-4-Maverick-17B-128E-Instruct"},"reasoning":false,"overallScore":34.44409085196595,"overallScoreRange":{"low":29.769014804754356,"high":39.11916689917755},"badges":[],"capabilityScores":{"agentic":36.87116126769469,"coding":37.16680778385831,"reasoning":36.66017753617196,"multimodal":35.97051669173584,"knowledge":34.000100515516785,"multilingual":null,"instruction":29.207585829733524,"math":28.62255608825591},"capabilityCounts":{"math":3,"knowledge":2,"multimodal":1,"reasoning":4,"instruction":1,"coding":1,"agentic":3},"benchmarks":[{"benchmarkId":"epoch-ai:epoch-math-level-5:overall","benchmarkName":"MATH level 5","category":"math","domain":null,"eligible":true,"ranked":true,"score":73.01737160120845,"unit":"percent","family":"epoch-ai:epoch-math-level-5:overall","index":34.46649204616109,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.083573699636281,"costPerTest":null,"latency":null},{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":67.1,"unit":"percent","family":"gpqa-diamond","index":38.751387224967104,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":66.98232323232324,"unit":"percent","family":"gpqa-diamond","index":38.643457309181095,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.8244823529728174,"costPerTest":null,"latency":null},{"benchmarkId":"aaMmmuPro","benchmarkName":"Artificial Analysis MMMU-Pro","category":"multimodal","score":62.1,"unit":"percent","family":"mmmu-pro","index":35.97051669173584,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmmu-pro","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":50,"unit":"percent","family":"lcr","index":41.60207573845371,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaIfBench","benchmarkName":"Artificial Analysis IFBench","category":"instruction","score":43,"unit":"percent","family":"ifbench-family","index":29.207585829733524,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaSciCode","benchmarkName":"Artificial Analysis SciCode","category":"coding","score":31.7,"unit":"percent","family":"scicode-family","index":37.16680778385831,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/scicode","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":20.555555555555554,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":20.878190705522616,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":4.948528559541873,"costPerTest":null,"latency":null},{"benchmarkId":"tau2Bench","benchmarkName":"τ²-Bench Tool-Agent-User Evaluation","category":"agentic","score":17.8,"unit":"percent","family":"tau2-bench","index":14.601041537543459,"aboveFit":false,"run":null,"source":"Victor Barres, Honghua Dong, Soham Ray, Xujie Si, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2506.07982","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"arc-prize:v1-public-eval:overall","benchmarkName":"ARC-AGI-1 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":7.12,"unit":"percent","family":"arc-prize:v1-public-eval:overall","index":null,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.0065,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":4.9,"unit":"percent","family":"hle","index":29.302778763959466,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"arc-prize:v1-semi-private:overall","benchmarkName":"ARC-AGI-1 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":4.38,"unit":"percent","family":"arc-prize:v1-semi-private:overall","index":29.00100299121867,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.0078,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-2025-02-28-private:overall","benchmarkName":"FrontierMath-2025-02-28-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":0.6896551724137931,"unit":"percent","family":"epoch-ai:epoch-frontiermath-2025-02-28-private:overall","index":30.52298551308402,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0.4868154158215009,"costPerTest":null,"latency":null},{"benchmarkId":"gdpvalAaNormalized","benchmarkName":"GDPval-AA normalized","category":"agentic","score":0,"unit":"percent","family":"gdpvalAaNormalized","index":34.03377491719279,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mazeBench","benchmarkName":"MazeBench: visual spatial reasoning in a 3D open world","category":"agentic","score":0,"unit":"percent","family":"mazeBench","index":61.97866734834783,"aboveFit":false,"run":null,"source":"Jonathan Pappas and David Pappas","sourceUrl":"https://mazebench.com/blog?post=maze-bench-results","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":0,"unit":"percent","family":"critpt","index":36.8491160127729,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"arc-prize:v2-public-eval:overall","benchmarkName":"ARC-AGI-2 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":0,"unit":"percent","family":"arc-prize:v2-public-eval:overall","index":null,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.0126,"latency":null},{"benchmarkId":"arc-prize:v2-semi-private:overall","benchmarkName":"ARC-AGI-2 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":0,"unit":"percent","family":"arc-prize:v2-semi-private:overall","index":39.188515402242544,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.0121,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-2025-02-28-public:overall","benchmarkName":"FrontierMath-2025-02-28-Public","category":"math","domain":null,"eligible":false,"ranked":true,"score":0,"unit":"percent","family":"epoch-ai:epoch-frontiermath-2025-02-28-public:overall","index":null,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":0.35,"outputPerMillion":1,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":61,"timeToFirstToken":0.574,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":95,"benchmarkCount":15,"categoryCount":7,"confidence":"medium","state":"available"},"hosts":[{"host":"oracle-cloud","name":"Oracle Cloud","kind":"cloud","url":"https://docs.oracle.com/en-us/iaas/Content/generative-ai/pretrained-models.htm","inputPerMillion":0.72,"outputPerMillion":0.72,"cacheReadPerMillion":null,"contextTokens":512000,"maxOutputTokens":16384,"deprecated":false},{"host":"deepinfra","name":"Deep Infra","kind":"host","url":"https://deepinfra.com/models","inputPerMillion":0.2,"outputPerMillion":0.8,"cacheReadPerMillion":null,"contextTokens":1048576,"maxOutputTokens":16384,"deprecated":true},{"host":"io-net","name":"IO.NET","kind":"host","url":"https://io.net/docs/guides/intelligence/io-intelligence","inputPerMillion":0.26875,"outputPerMillion":0.82625,"cacheReadPerMillion":0.134375,"contextTokens":430000,"maxOutputTokens":4096,"deprecated":false},{"host":"novita-ai","name":"Novita AI","kind":"host","url":"https://novita.ai/docs/guides/introduction","inputPerMillion":0.27,"outputPerMillion":0.85,"cacheReadPerMillion":null,"contextTokens":1048576,"maxOutputTokens":8192,"deprecated":false},{"host":"digitalocean","name":"DigitalOcean","kind":"cloud","url":"https://docs.digitalocean.com/products/gradient-ai-platform/details/models/","inputPerMillion":0.25,"outputPerMillion":0.87,"cacheReadPerMillion":0.05,"contextTokens":128000,"maxOutputTokens":16384,"deprecated":false},{"host":"amazon-bedrock","name":"Amazon Bedrock","kind":"cloud","url":"https://docs.aws.amazon.com/bedrock/latest/userguide/models-supported.html","inputPerMillion":0.24,"outputPerMillion":0.97,"cacheReadPerMillion":null,"contextTokens":1000000,"maxOutputTokens":8192,"deprecated":true},{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":0.35,"outputPerMillion":1,"cacheReadPerMillion":0.17,"contextTokens":524288,"maxOutputTokens":32768,"deprecated":false},{"host":"ibm-watsonx","name":"IBM watsonx","kind":"cloud","url":"https://www.ibm.com/docs/en/watsonx/saas?topic=solutions-supported-foundation-models","inputPerMillion":0.371,"outputPerMillion":1.484,"cacheReadPerMillion":null,"contextTokens":131072,"maxOutputTokens":8192,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:arc-prize-foundation","name":"ARC Prize Foundation","url":"https://arcprize.org","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"mistral-large-3","providerId":"mistral","slug":"mistral-large-3","name":"Mistral Large 3","provider":"Mistral","description":"Mistral Large 3 is a standard model from Mistral. 17 benchmarks count toward its index, in 7 categories.","logoUrl":null,"releaseDate":"2025-12-02","contextWindowTokens":262144,"modalities":["text","image","file"],"openWeights":true,"details":{"apiId":"mistralai/mistral-large-2512","maxOutputTokens":209715,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.049999999999999996,"weightsUrl":"https://huggingface.co/mistralai/Mistral-Large-3-675B-Instruct-2512"},"reasoning":false,"overallScore":34.31501708248564,"overallScoreRange":{"low":29.893216025648133,"high":38.73681813932314},"badges":[],"capabilityScores":{"agentic":29.676729852047444,"coding":38.17686939571419,"reasoning":34.42303219651369,"multimodal":34.46413199486041,"knowledge":38.90521171821429,"multilingual":null,"instruction":22.85291364889291,"math":34.25925889077362},"capabilityCounts":{"knowledge":4,"multimodal":1,"coding":4,"math":1,"instruction":1,"reasoning":2,"agentic":4},"benchmarks":[{"benchmarkId":"vals-ai:mgsm:overall","benchmarkName":"MGSM","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":85.418,"unit":"percent","family":"vals-ai:mgsm:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mgsm","measuredAt":"2026-01-09","stderr":1.787,"costPerTest":0.000472,"latency":7.622},{"benchmarkId":"vals-ai:medqa:overall","benchmarkName":"MedQA","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":82.233,"unit":"percent","family":"vals-ai:medqa:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medqa","measuredAt":"2026-04-16","stderr":0.349,"costPerTest":0.000771,"latency":17.31},{"benchmarkId":"vals-ai:mmlu_pro:overall","benchmarkName":"MMLU Pro","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":79.823,"unit":"percent","family":"mmlu-pro","index":45.04815385945304,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","measuredAt":"2026-09-01","stderr":0.472,"costPerTest":0.001481,"latency":13.554},{"benchmarkId":"vals-ai:legal_bench:overall","benchmarkName":"LegalBench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":79.138,"unit":"percent","family":"vals-ai:legal_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","measuredAt":"2026-10-13","stderr":0.46,"costPerTest":0.000439,"latency":1.731},{"benchmarkId":"vals-ai:tax_eval_v2:overall","benchmarkName":"TaxEval v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":73.058,"unit":"percent","family":"vals-ai:tax_eval_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","measuredAt":"2026-09-01","stderr":0.867,"costPerTest":0.001601,"latency":29.531},{"benchmarkId":"vals-ai:gpqa:overall","benchmarkName":"GPQA Diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":68.434,"unit":"percent","family":"gpqa-diamond","index":39.97489557331009,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","measuredAt":"2026-09-01","stderr":2.372,"costPerTest":0.00215,"latency":21.216},{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":68,"unit":"percent","family":"gpqa-diamond","index":39.576842632394914,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:mmmu:overall","benchmarkName":"MMMU Pro","category":"multimodal","domain":null,"eligible":true,"ranked":true,"score":66.185,"unit":"percent","family":"mmmu-pro","index":41.28678579474763,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmmu","measuredAt":"2026-09-01","stderr":1.137,"costPerTest":0.001477,"latency":13.55},{"benchmarkId":"vals-ai:case_law_v2:overall","benchmarkName":"CaseLaw v2","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":61.415,"unit":"percent","family":"vals-ai:case_law_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/case_law_v2","measuredAt":"2026-05-04","stderr":0.965,"costPerTest":0.018968,"latency":40.839},{"benchmarkId":"vals-ai:corp_fin_v2:overall","benchmarkName":"CorpFin v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":61.033,"unit":"percent","family":"vals-ai:corp_fin_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","measuredAt":"2026-08-12","stderr":0.96,"costPerTest":0.024166,"latency":48.842},{"benchmarkId":"aaMmmuPro","benchmarkName":"Artificial Analysis MMMU-Pro","category":"multimodal","score":55.7,"unit":"percent","family":"mmmu-pro","index":27.641478194973182,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmmu-pro","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:lcb:overall","benchmarkName":"LiveCodeBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":55.337,"unit":"percent","family":"livecodebench","index":35.259156892823356,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","measuredAt":"2026-09-01","stderr":1.152,"costPerTest":0.002215,"latency":28.888},{"benchmarkId":"vals-ai:mortgage_tax:overall","benchmarkName":"MortgageTax","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":52.106,"unit":"percent","family":"vals-ai:mortgage_tax:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mortgage_tax","measuredAt":"2026-09-01","stderr":0.995,"costPerTest":0.003226,"latency":9.292},{"benchmarkId":"vals-ai:aime:overall","benchmarkName":"AIME","category":"math","domain":null,"eligible":true,"ranked":true,"score":42.917,"unit":"percent","family":"vals-ai:aime:overall","index":34.25925889077362,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/aime","measuredAt":"2026-04-16","stderr":0.884,"costPerTest":0.006086,"latency":104.325},{"benchmarkId":"vals-ai:swebench:overall","benchmarkName":"SWE-bench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":41.4,"unit":"percent","family":"swe-bench-verified","index":31.762811249716492,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/swebench","measuredAt":"2026-09-01","stderr":2.205,"costPerTest":0.321666,"latency":186.955},{"benchmarkId":"aaSciCode","benchmarkName":"Artificial Analysis SciCode","category":"coding","score":36.6,"unit":"percent","family":"scicode-family","index":43.554894401290504,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/scicode","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaIfBench","benchmarkName":"Artificial Analysis IFBench","category":"instruction","score":36.2,"unit":"percent","family":"ifbench-family","index":22.85291364889291,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":36,"unit":"percent","family":"lcr","index":31.996948380254473,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"omniscienceAccuracy","benchmarkName":"Artificial Analysis Omniscience Accuracy","category":"knowledge","score":25,"unit":"percent","family":"omniscienceAccuracy","index":42.19635006811299,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"tau2Bench","benchmarkName":"τ²-Bench Tool-Agent-User Evaluation","category":"agentic","score":24.6,"unit":"percent","family":"tau2-bench","index":19.113338698610384,"aboveFit":false,"run":null,"source":"Victor Barres, Honghua Dong, Soham Ray, Xujie Si, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2506.07982","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:terminal-bench:overall","benchmarkName":"Terminal-Bench 1.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":21.25,"unit":"percent","family":"vals-ai:terminal-bench:overall","index":33.59616319989902,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench","measuredAt":"2026-01-12","stderr":4.574,"costPerTest":0.057283,"latency":158.436},{"benchmarkId":"vals-ai:finance_agent:overall","benchmarkName":"Finance Agent v1.1","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":18.049,"unit":"percent","family":"vals-ai:finance_agent:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/finance_agent","measuredAt":"2026-06-04","stderr":2.176,"costPerTest":0.086052,"latency":91.152},{"benchmarkId":"vals-ai:terminal-bench-2:overall","benchmarkName":"Terminal-Bench 2.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":8.989,"unit":"percent","family":"vals-ai:terminal-bench-2:overall","index":31.963642592487588,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2","measuredAt":"2026-06-04","stderr":3.049,"costPerTest":0.030482,"latency":1180.302},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":4.2,"unit":"percent","family":"hle","index":28.60047384243861,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:ioi-v1:overall","benchmarkName":"IOI v1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":4,"unit":"percent","family":"vals-ai:ioi-v1:overall","index":42.13061503902641,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/ioi-v1","measuredAt":"2026-08-09","stderr":1.46,"costPerTest":3.073948,"latency":6788.576},{"benchmarkId":"gdpvalAaNormalized","benchmarkName":"GDPval-AA normalized","category":"agentic","score":0,"unit":"percent","family":"gdpvalAaNormalized","index":34.03377491719279,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":0,"unit":"percent","family":"critpt","index":36.8491160127729,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":0.5,"outputPerMillion":1.5,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":61,"timeToFirstToken":0.745,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":{"elo":1428.6218230728227,"eloUpper":1431.4143779684214,"eloLower":1425.8292681772236,"votes":88641,"rank":98,"rankUpper":null,"rankLower":null,"arenaName":"Mistral Large 3","organization":"mistral","updatedAt":"2026-11-09"},"quality":{"coveragePercent":95,"benchmarkCount":17,"categoryCount":7,"confidence":"medium","state":"available"},"hosts":[{"host":"amazon-bedrock","name":"Amazon Bedrock","kind":"cloud","url":"https://docs.aws.amazon.com/bedrock/latest/userguide/models-supported.html","inputPerMillion":0.5,"outputPerMillion":1.5,"cacheReadPerMillion":null,"contextTokens":262144,"maxOutputTokens":8192,"deprecated":false},{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":0.5,"outputPerMillion":1.5,"cacheReadPerMillion":0.05,"contextTokens":262144,"maxOutputTokens":209715,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"magistral-medium","providerId":"mistral","slug":"magistral-medium","name":"Magistral Medium","provider":"Mistral","description":"Magistral Medium is a reasoning model from Mistral in the Magistral family. 20 benchmarks count toward its index, in 7 categories.","logoUrl":null,"releaseDate":"2025-06-10","contextWindowTokens":40000,"modalities":["text"],"openWeights":false,"details":null,"reasoning":true,"overallScore":33.741258953705696,"overallScoreRange":{"low":29.625537034088538,"high":37.856980873322854},"badges":[],"capabilityScores":{"agentic":30.603765752597116,"coding":39.21540004391774,"reasoning":28.40521127449133,"multimodal":39.1797993375197,"knowledge":35.24000891961111,"multilingual":null,"instruction":12.47984582428542,"math":41.77130291385653},"capabilityCounts":{"math":4,"coding":3,"knowledge":4,"multimodal":1,"agentic":3,"instruction":1,"reasoning":4},"benchmarks":[{"benchmarkId":"math500","benchmarkName":"MATH-500 Problem Set","category":"math","score":94.3,"unit":"percent","family":"math-500","index":43.77908268851496,"aboveFit":false,"run":null,"source":"Dan Hendrycks, Collin Burns, Saurav Kadavath, Akul Arora, Steven Basart, Eric Tang, Dawn Song, Jacob Steinhardt","sourceUrl":"https://arxiv.org/abs/2103.03874","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:medqa:overall","benchmarkName":"MedQA","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":89.467,"unit":"percent","family":"vals-ai:medqa:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medqa","measuredAt":"2026-04-16","stderr":0.28,"costPerTest":0.006909,"latency":17.075},{"benchmarkId":"vals-ai:aime:overall","benchmarkName":"AIME","category":"math","domain":null,"eligible":true,"ranked":true,"score":83.542,"unit":"percent","family":"vals-ai:aime:overall","index":52.014242908434895,"aboveFit":true,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/aime","measuredAt":"2026-04-16","stderr":1.018,"costPerTest":0.086908,"latency":401.007},{"benchmarkId":"vals-ai:lcb:overall","benchmarkName":"LiveCodeBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":74.86,"unit":"percent","family":"livecodebench","index":51.45819490664383,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","measuredAt":"2026-09-01","stderr":1.132,"costPerTest":0.049906,"latency":209.97},{"benchmarkId":"vals-ai:mgsm:overall","benchmarkName":"MGSM","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":74.618,"unit":"percent","family":"vals-ai:mgsm:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mgsm","measuredAt":"2026-01-09","stderr":2.549,"costPerTest":0.005709,"latency":11.933},{"benchmarkId":"aime2024","benchmarkName":"American Invitational Mathematics Examination 2024","category":"math","score":73.6,"unit":"percent","family":"aime2024","index":37.068248818608176,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gpqa","benchmarkName":"Graduate-Level Google-Proof Q&A","category":"knowledge","score":70.8,"unit":"percent","family":"gpqa-diamond","index":42.144926122170304,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:mmlu_pro:overall","benchmarkName":"MMLU Pro","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":68.659,"unit":"percent","family":"mmlu-pro","index":30.982850155902852,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","measuredAt":"2026-09-01","stderr":0.433,"costPerTest":0.014664,"latency":31.988},{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":67.9,"unit":"percent","family":"gpqa-diamond","index":39.48512536490294,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aime2025","benchmarkName":"American Invitational Mathematics Examination 2025","category":"math","score":64.9,"unit":"percent","family":"aime2025","index":34.22363723986808,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:mmmu:overall","benchmarkName":"MMMU Pro","category":"multimodal","domain":null,"eligible":true,"ranked":true,"score":64.566,"unit":"percent","family":"mmmu-pro","index":39.1797993375197,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmmu","measuredAt":"2026-09-01","stderr":1.15,"costPerTest":0.021457,"latency":42.144},{"benchmarkId":"vals-ai:gpqa:overall","benchmarkName":"GPQA Diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":62.374,"unit":"percent","family":"gpqa-diamond","index":34.41682916329621,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","measuredAt":"2026-09-01","stderr":2.435,"costPerTest":0.03601,"latency":94.145},{"benchmarkId":"vals-ai:tax_eval_v2:overall","benchmarkName":"TaxEval v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":61.938,"unit":"percent","family":"vals-ai:tax_eval_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","measuredAt":"2026-09-01","stderr":0.964,"costPerTest":0.025214,"latency":46.254},{"benchmarkId":"liveCodeBenchV5","benchmarkName":"LiveCodeBench v5","category":"coding","score":59.4,"unit":"percent","family":"liveCodeBenchV5","index":null,"aboveFit":false,"run":null,"source":"LiveCodeBench maintainers","sourceUrl":"https://github.com/LiveCodeBench/LiveCodeBench","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:legal_bench:overall","benchmarkName":"LegalBench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":54.767,"unit":"percent","family":"vals-ai:legal_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","measuredAt":"2026-10-13","stderr":0.561,"costPerTest":null,"latency":11.734},{"benchmarkId":"vals-ai:mortgage_tax:overall","benchmarkName":"MortgageTax","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":51.868,"unit":"percent","family":"vals-ai:mortgage_tax:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mortgage_tax","measuredAt":"2026-09-01","stderr":0.993,"costPerTest":0.019966,"latency":25.948},{"benchmarkId":"liveCodeBenchV6","benchmarkName":"LiveCodeBench v6","category":"coding","score":50.3,"unit":"percent","family":"liveCodeBenchV6","index":25.716245606582454,"aboveFit":false,"run":null,"source":"LiveCodeBench maintainers","sourceUrl":"https://github.com/LiveCodeBench/LiveCodeBench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:corp_fin_v2:overall","benchmarkName":"CorpFin v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":47.397,"unit":"percent","family":"vals-ai:corp_fin_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","measuredAt":"2026-08-12","stderr":0.982,"costPerTest":0.125244,"latency":55.265},{"benchmarkId":"vals-ai:terminal-bench:overall","benchmarkName":"Terminal-Bench 1.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":28.75,"unit":"percent","family":"vals-ai:terminal-bench:overall","index":39.02378765186826,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench","measuredAt":"2026-01-12","stderr":5.06,"costPerTest":1.234078,"latency":746.79},{"benchmarkId":"aaIfBench","benchmarkName":"Artificial Analysis IFBench","category":"instruction","score":25.1,"unit":"percent","family":"ifbench-family","index":12.47984582428542,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"tau2Bench","benchmarkName":"τ²-Bench Tool-Agent-User Evaluation","category":"agentic","score":23.1,"unit":"percent","family":"tau2-bench","index":18.11797903072797,"aboveFit":false,"run":null,"source":"Victor Barres, Honghua Dong, Soham Ray, Xujie Si, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2506.07982","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"omniscienceAccuracy","benchmarkName":"Artificial Analysis Omniscience Accuracy","category":"knowledge","score":20.3,"unit":"percent","family":"omniscienceAccuracy","index":36.27314694303317,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:terminal-bench-2:overall","benchmarkName":"Terminal-Bench 2.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":13.483,"unit":"percent","family":"vals-ai:terminal-bench-2:overall","index":34.66953057519512,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2","measuredAt":"2026-06-04","stderr":3.641,"costPerTest":0.567192,"latency":898.203},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":9.8,"unit":"percent","family":"hle","index":34.218913214605465,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"arc-prize:v1-public-eval:overall","benchmarkName":"ARC-AGI-1 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":8.9,"unit":"percent","family":"arc-prize:v1-public-eval:overall","index":null,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.1069,"latency":null},{"benchmarkId":"arc-prize:v1-semi-private:overall","benchmarkName":"ARC-AGI-1 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":5.91,"unit":"percent","family":"arc-prize:v1-semi-private:overall","index":29.704399228769493,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.1015,"latency":null},{"benchmarkId":"vals-ai:ioi-v1:overall","benchmarkName":"IOI v1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":0.666,"unit":"percent","family":"vals-ai:ioi-v1:overall","index":40.471759618526946,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/ioi-v1","measuredAt":"2026-08-09","stderr":0.444,"costPerTest":6.344222,"latency":2211.783},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":0.3,"unit":"percent","family":"critpt","index":37.42988100778257,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":0,"unit":"percent","family":"lcr","index":7.298049459170706,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"arc-prize:v2-public-eval:overall","benchmarkName":"ARC-AGI-2 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":0,"unit":"percent","family":"arc-prize:v2-public-eval:overall","index":null,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.1061,"latency":null},{"benchmarkId":"arc-prize:v2-semi-private:overall","benchmarkName":"ARC-AGI-2 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":0,"unit":"percent","family":"arc-prize:v2-semi-private:overall","index":39.188515402242544,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.1079,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":{"elo":1254.7621153804248,"eloUpper":1261.2061132811386,"eloLower":1248.3181174797112,"votes":11496,"rank":299,"rankUpper":null,"rankLower":null,"arenaName":"Magistral Medium-2506","organization":"mistral","updatedAt":"2026-11-09"},"quality":{"coveragePercent":95,"benchmarkCount":20,"categoryCount":7,"confidence":"medium","state":"available"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:arc-prize-foundation","name":"ARC Prize Foundation","url":"https://arcprize.org","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"celeris-1","providerId":"celeris","slug":"celeris-1","name":"Celeris-1","provider":"Celeris","description":"Celeris-1 is a standard model from Celeris. 8 benchmarks count toward its index, in 5 categories.","logoUrl":null,"releaseDate":"2026-07-22","contextWindowTokens":128000,"modalities":["text"],"openWeights":false,"details":null,"reasoning":false,"overallScore":33.71657713389645,"overallScoreRange":{"low":25.499618808003213,"high":41.93353545978969},"badges":[],"capabilityScores":{"agentic":34.03377491719279,"coding":23.999527205069516,"reasoning":35.10911272209935,"multimodal":null,"knowledge":35.46579164509551,"multilingual":null,"instruction":25.906908616884845,"math":null},"capabilityCounts":{"instruction":1,"knowledge":3,"reasoning":2,"coding":1,"agentic":1},"benchmarks":[{"benchmarkId":"gsm8k","benchmarkName":"Grade School Math 8K","category":"math","score":93.7,"unit":"percent","family":"gsm8k","index":null,"aboveFit":false,"run":null,"source":"Karl Cobbe and colleagues (OpenAI)","sourceUrl":"https://arxiv.org/abs/2110.14168","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"drop","benchmarkName":"Discrete Reasoning Over Paragraphs","category":"reasoning","score":81.4,"unit":"percent","family":"drop","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"ifeval","benchmarkName":"Instruction-Following Eval","category":"instruction","score":80.8,"unit":"percent","family":"ifeval","index":25.906908616884845,"aboveFit":false,"run":null,"source":"Jeffrey Zhou, Tianjian Lu, Swaroop Mishra, Siddhartha Brahma, Sujoy Basu, Yi Luan, Denny Zhou, Le Hou","sourceUrl":"https://arxiv.org/abs/2311.07911","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmluPro","benchmarkName":"Massive Multitask Language Understanding Professional","category":"knowledge","score":75.9,"unit":"percent","family":"mmlu-pro","index":40.10564343048248,"aboveFit":false,"run":null,"source":"Yubo Wang, Xueguang Ma, Ge Zhang, Yuansheng Ni, Abhranil Chandra, Shiguang Guo, Weiming Ren, Aaran Arulraj, Xuan He, Ziyan Jiang, Tianle Li, Max Ku, Kai Wang, Alex Zhuang, Rongqi Fan, Xiang Yue, Wenhu Chen","sourceUrl":"https://arxiv.org/abs/2406.01574","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":63.1,"unit":"percent","family":"gpqa-diamond","index":35.08269652528797,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":38,"unit":"percent","family":"lcr","index":33.36910943142579,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaSciCode","benchmarkName":"Artificial Analysis SciCode","category":"coding","score":21.6,"unit":"percent","family":"scicode-family","index":23.999527205069516,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/scicode","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":6.8,"unit":"percent","family":"hle","index":31.20903497951608,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gdpvalAaNormalized","benchmarkName":"GDPval-AA normalized","category":"agentic","score":0,"unit":"percent","family":"gdpvalAaNormalized","index":34.03377491719279,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":0,"unit":"percent","family":"critpt","index":36.8491160127729,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":0.2,"outputPerMillion":0.7,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":1555,"timeToFirstToken":0.54,"source":"BenchLM","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":75,"benchmarkCount":8,"categoryCount":5,"confidence":"low","state":"available"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"o1-mini","providerId":"openai","slug":"o1-mini","name":"o1-mini","provider":"OpenAI","description":"o1-mini is a reasoning model from OpenAI. 12 benchmarks count toward its index, in 5 categories.","logoUrl":null,"releaseDate":"2024-09-12","contextWindowTokens":128000,"modalities":["text"],"openWeights":false,"details":null,"reasoning":true,"overallScore":33.547202590368045,"overallScoreRange":{"low":25.37932566396992,"high":41.71507951676617},"badges":[],"capabilityScores":{"agentic":null,"coding":32.738050244582276,"reasoning":36.49663075294256,"multimodal":null,"knowledge":28.159506450254508,"multilingual":33.4244331650364,"instruction":null,"math":37.228933594284754},"capabilityCounts":{"coding":1,"math":4,"multilingual":1,"knowledge":4,"reasoning":2},"benchmarks":[{"benchmarkId":"humaneval","benchmarkName":"Evaluating Large Language Models Trained on Code","category":"coding","score":92.4,"unit":"percent","family":"humaneval","index":32.738050244582276,"aboveFit":false,"run":null,"source":"Mark Chen, Jerry Tworek, Heewoo Jun, Qiming Yuan, Henrique Ponde de Oliveira Pinto, Jared Kaplan, Harri Edwards, Yuri Burda, Nicholas Joseph, Greg Brockman, Alex Ray, Raul Puri, Gretchen Krueger, Michael Petrov, Heidy Khlaaf, Girish Sastry, Pamela Mishkin, Brooke Chan, Scott Gray, Nick Ryder, Mikhail Pavlov, Alethea Power, Lukasz Kaiser, Mohammad Bavarian, Clemens Winter, Philippe Tillet, Felipe Petroski Such, Dave Cummings, Matthias Plappert, Fotios Chantzis, Elizabeth Barnes, Ariel Herbert-Voss, William Hebgen Guss, Alex Nichol, Alex Paino, Nikolas Tezak, Jie Tang, Igor Babuschkin, Suchir Balaji, Shantanu Jain, William Saunders, Christopher Hesse, Andrew N. Carr, Jan Leike, Josh Achiam, Vedant Misra, Evan Morikawa, Alec Radford, Matthew Knight, Miles Brundage, Mira Murati, Katie Mayer, Peter Welinder, Bob McGrew, Dario Amodei, Sam McCandlish, Ilya Sutskever, Wojciech Zaremba","sourceUrl":"https://arxiv.org/abs/2107.03374","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:medqa:overall","benchmarkName":"MedQA","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":90.217,"unit":"percent","family":"vals-ai:medqa:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medqa","measuredAt":"2026-04-16","stderr":0.532,"costPerTest":0.004296,"latency":5.885},{"benchmarkId":"math500","benchmarkName":"MATH-500 Problem Set","category":"math","score":90,"unit":"percent","family":"math-500","index":39.23502237506648,"aboveFit":false,"run":null,"source":"Dan Hendrycks, Collin Burns, Saurav Kadavath, Akul Arora, Steven Basart, Eric Tang, Dawn Song, Jacob Steinhardt","sourceUrl":"https://arxiv.org/abs/2103.03874","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mgsm","benchmarkName":"Multilingual Grade School Math","category":"multilingual","score":89.9,"unit":"percent","family":"mgsm","index":33.4244331650364,"aboveFit":false,"run":null,"source":"Freda Shi, Mirac Suzgun, Markus Freitag, Xuezhi Wang, Suraj Srivats, Soroush Vosoughi, Hyung Won Chung, Yi Tay, Sebastian Ruder, Denny Zhou, Dipanjan Das, Jason Wei","sourceUrl":"https://arxiv.org/abs/2210.03057","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-math-level-5:overall","benchmarkName":"MATH level 5","category":"math","domain":null,"eligible":true,"ranked":true,"score":89.18051359516616,"unit":"percent","family":"epoch-ai:epoch-math-level-5:overall","index":42.31782519827549,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0.6802820166065648,"costPerTest":null,"latency":null},{"benchmarkId":"mmlu","benchmarkName":"Massive Multitask Language Understanding","category":"knowledge","score":85.2,"unit":"percent","family":"mmlu","index":33.26044357098425,"aboveFit":false,"run":null,"source":"Dan Hendrycks, Collin Burns, Steven Basart, Andy Zou, Mantas Mazeika, Dawn Song, Jacob Steinhardt","sourceUrl":"https://arxiv.org/abs/2009.03300","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"drop","benchmarkName":"Discrete Reasoning Over Paragraphs","category":"reasoning","score":83.9,"unit":"percent","family":"drop","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":62.37373737373737,"unit":"percent","family":"gpqa-diamond","index":34.4165882896644,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.7427723511230755,"costPerTest":null,"latency":null},{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":60.3,"unit":"percent","family":"gpqa-diamond","index":32.51461303551257,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gpqa","benchmarkName":"Graduate-Level Google-Proof Q&A","category":"knowledge","score":60,"unit":"percent","family":"gpqa-diamond","index":32.239461233036636,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":46.94444444444444,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":36.231089134590164,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":6.456873681038247,"costPerTest":null,"latency":null},{"benchmarkId":"arc-prize:v1-semi-private:overall","benchmarkName":"ARC-AGI-1 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":14.000000000000002,"unit":"percent","family":"arc-prize:v1-semi-private:overall","index":33.42366430183236,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.135,"latency":null},{"benchmarkId":"arc-prize:v1-public-eval:overall","benchmarkName":"ARC-AGI-1 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":13.16,"unit":"percent","family":"arc-prize:v1-public-eval:overall","index":null,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.1239,"latency":null},{"benchmarkId":"simpleQa","benchmarkName":"Measuring Short-Form Factuality in Large Language Models","category":"knowledge","score":7.6,"unit":"percent","family":"simpleQa","index":18.86447099910048,"aboveFit":false,"run":null,"source":"Jason Wei, Najoung Kim, Hyung Won Chung, Yu-An Chung, Siddhartha Papay, Yifeng Lu, Hannaneh Hajishirzi, Luke Zettlemoyer","sourceUrl":"https://arxiv.org/abs/2411.04368","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":3.6,"unit":"percent","family":"hle","index":27.998498195420737,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-2025-02-28-private:overall","benchmarkName":"FrontierMath-2025-02-28-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":1.3793103448275863,"unit":"percent","family":"epoch-ai:epoch-frontiermath-2025-02-28-private:overall","index":31.131797669206875,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0.6860663093423026,"costPerTest":null,"latency":null},{"benchmarkId":"arc-prize:v2-semi-private:overall","benchmarkName":"ARC-AGI-2 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":0.83,"unit":"percent","family":"arc-prize:v2-semi-private:overall","index":39.569597204052755,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.1907,"latency":null},{"benchmarkId":"arc-prize:v2-public-eval:overall","benchmarkName":"ARC-AGI-2 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":0,"unit":"percent","family":"arc-prize:v2-public-eval:overall","index":null,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.1838,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-2025-02-28-public:overall","benchmarkName":"FrontierMath-2025-02-28-Public","category":"math","domain":null,"eligible":false,"ranked":true,"score":0,"unit":"percent","family":"epoch-ai:epoch-frontiermath-2025-02-28-public:overall","index":null,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":1.1,"outputPerMillion":4.4,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":{"elo":1317.0828044768368,"eloUpper":1320.7199346479733,"eloLower":1313.4456743057005,"votes":51981,"rank":243,"rankUpper":null,"rankLower":null,"arenaName":"o1-mini","organization":"openai","updatedAt":"2026-11-09"},"quality":{"coveragePercent":65,"benchmarkCount":12,"categoryCount":5,"confidence":"low","state":"available"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:arc-prize-foundation","name":"ARC Prize Foundation","url":"https://arcprize.org","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"grok-4-1-fast","providerId":"xai","slug":"grok-4-1-fast","name":"Grok 4.1 Fast","provider":"xAI","description":"Grok 4.1 Fast is a standard model from xAI. 9 benchmarks count toward its index, in 5 categories.","logoUrl":null,"releaseDate":"2025-11-19","contextWindowTokens":1000000,"modalities":["text"],"openWeights":false,"details":null,"reasoning":false,"overallScore":33.30629677046964,"overallScoreRange":{"low":24.416296770469643,"high":42.196296770469644},"badges":[],"capabilityScores":{"agentic":41.073237166256476,"coding":null,"reasoning":32.81074296138739,"multimodal":18.141168659603274,"knowledge":32.500930278551294,"multilingual":null,"instruction":23.133266833341754,"math":null},"capabilityCounts":{"agentic":2,"knowledge":3,"multimodal":1,"instruction":1,"reasoning":2},"benchmarks":[{"benchmarkId":"tau2Bench","benchmarkName":"τ²-Bench Tool-Agent-User Evaluation","category":"agentic","score":63.7,"unit":"percent","family":"tau2-bench","index":45.05904737474517,"aboveFit":false,"run":null,"source":"Victor Barres, Honghua Dong, Soham Ray, Xujie Si, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2506.07982","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":63.7,"unit":"percent","family":"gpqa-diamond","index":35.633000130239836,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaMmmuPro","benchmarkName":"Artificial Analysis MMMU-Pro","category":"multimodal","score":48.4,"unit":"percent","family":"mmmu-pro","index":18.141168659603274,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmmu-pro","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gertLabs","benchmarkName":"Gert Labs Composite Game Benchmark","category":"agentic","score":47.32,"unit":"percent","family":"gertLabs","index":53.862555227145975,"aboveFit":true,"run":null,"source":"Gert Labs","sourceUrl":"https://gertlabs.com/rankings","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaIfBench","benchmarkName":"Artificial Analysis IFBench","category":"instruction","score":36.5,"unit":"percent","family":"ifbench-family","index":23.133266833341754,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":31.3,"unit":"percent","family":"lcr","index":28.77236991000187,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"omniscienceAccuracy","benchmarkName":"Artificial Analysis Omniscience Accuracy","category":"knowledge","score":17.2,"unit":"percent","family":"omniscienceAccuracy","index":32.3663533924486,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"tau2-bench:banking-knowledge:overall","benchmarkName":"τ²-bench Banking","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":13.14,"unit":"percent","family":"tau2-bench","index":11.508790835988776,"aboveFit":false,"run":"high effort · Sierra","source":"Sierra Research","sourceUrl":"https://github.com/sierra-research/tau2-bench","measuredAt":"2026-10-20","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":5.1,"unit":"percent","family":"hle","index":29.503437312965428,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":0,"unit":"percent","family":"critpt","index":36.8491160127729,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":0.2,"outputPerMillion":0.5,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":138,"timeToFirstToken":0.54,"source":"BenchLM","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":65,"benchmarkCount":9,"categoryCount":5,"confidence":"low","state":"available"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:sierra-research","name":"Sierra Research","url":"https://github.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"granite-4-2-3b","providerId":"ibm","slug":"granite-4-2-3b","name":"Granite 4.2 3B","provider":"IBM","description":"Granite 4.2 3B is a reasoning model from IBM in the Granite 4.2 family. 14 benchmarks count toward its index, in 6 categories.","logoUrl":null,"releaseDate":"2026-08-25","contextWindowTokens":128000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":true,"overallScore":33.204566476945814,"overallScoreRange":{"low":27.38295219522191,"high":39.02618075866972},"badges":[],"capabilityScores":{"agentic":33.32701133288225,"coding":34.850226522791395,"reasoning":30.409461121837573,"multimodal":null,"knowledge":27.804574608206238,"multilingual":null,"instruction":58.485803392341865,"math":34.33947390373857},"capabilityCounts":{"math":2,"instruction":1,"coding":2,"knowledge":4,"agentic":3,"reasoning":2},"benchmarks":[{"benchmarkId":"aime2025","benchmarkName":"American Invitational Mathematics Examination 2025","category":"math","score":78.33,"unit":"percent","family":"aime2025","index":41.51110195330684,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"ifBench","benchmarkName":"Instruction Following Benchmark","category":"instruction","score":74.33,"unit":"percent","family":"ifbench-family","index":58.485803392341865,"aboveFit":true,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaOpennessIndex","benchmarkName":"Artificial Analysis Openness Index","category":"knowledge","score":72.2,"unit":"percent","family":"aaOpennessIndex","index":null,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-openness-index","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"liveCodeBenchV6","benchmarkName":"LiveCodeBench v6","category":"coding","score":69.71,"unit":"percent","family":"liveCodeBenchV6","index":41.652964830895534,"aboveFit":false,"run":null,"source":"LiveCodeBench maintainers","sourceUrl":"https://github.com/LiveCodeBench/LiveCodeBench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmluPro","benchmarkName":"Massive Multitask Language Understanding Professional","category":"knowledge","score":67.84,"unit":"percent","family":"mmlu-pro","index":29.951008187682888,"aboveFit":false,"run":null,"source":"Yubo Wang, Xueguang Ma, Ge Zhang, Yuansheng Ni, Abhranil Chandra, Shiguang Guo, Weiming Ren, Aaran Arulraj, Xuan He, Ziyan Jiang, Tianle Li, Max Ku, Kai Wang, Alex Zhuang, Rongqi Fan, Xiang Yue, Wenhu Chen","sourceUrl":"https://arxiv.org/abs/2406.01574","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hmmtFeb2025","benchmarkName":"Harvard-MIT Mathematics Tournament February 2025","category":"math","score":66.67,"unit":"percent","family":"hmmtFeb2025","index":27.167845854170306,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":55.9,"unit":"percent","family":"gpqa-diamond","index":28.47905326586551,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gpqa","benchmarkName":"Graduate-Level Google-Proof Q&A","category":"knowledge","score":54.8,"unit":"percent","family":"gpqa-diamond","index":27.470163323453747,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"bfclV4","benchmarkName":"Berkeley Function Calling Leaderboard v4","category":"agentic","score":52.41,"unit":"percent","family":"bfclV4","index":33.81345989005027,"aboveFit":false,"run":null,"source":"Shishir G. Patil et al.","sourceUrl":"https://gorilla.cs.berkeley.edu/leaderboard.html","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"tau3Bench","benchmarkName":"τ³-Bench Tool-Agent-User Evaluation","category":"agentic","score":45.78,"unit":"percent","family":"tau3Bench","index":32.13379919140369,"aboveFit":false,"run":null,"source":"Sierra Research","sourceUrl":"https://github.com/sierra-research/tau2-bench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaSciCode","benchmarkName":"Artificial Analysis SciCode","category":"coding","score":25.3,"unit":"percent","family":"scicode-family","index":28.823184446804024,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/scicode","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":24.3,"unit":"percent","family":"lcr","index":23.969806230902247,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"sciCode","benchmarkName":"Scientific Code Benchmark","category":"coding","score":24.11,"unit":"percent","family":"scicode-family","index":27.27179198257049,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"omniscienceAccuracy","benchmarkName":"Artificial Analysis Omniscience Accuracy","category":"knowledge","score":9.2,"unit":"percent","family":"omniscienceAccuracy","index":22.284305519972317,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":6.6,"unit":"percent","family":"hle","index":31.00837643051012,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gdpvalAaNormalized","benchmarkName":"GDPval-AA normalized","category":"agentic","score":0,"unit":"percent","family":"gdpvalAaNormalized","index":34.03377491719279,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":0,"unit":"percent","family":"critpt","index":36.8491160127729,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":{"elo":1292.5274780087416,"eloUpper":1304.1537449218306,"eloLower":1280.9012110956528,"votes":3185,"rank":258,"rankUpper":null,"rankLower":null,"arenaName":"Granite 4.2 3B","organization":"ibm","updatedAt":"2026-11-09"},"quality":{"coveragePercent":85,"benchmarkCount":14,"categoryCount":6,"confidence":"medium","state":"available"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"nemotron-3-nano-30b","providerId":"nvidia","slug":"nemotron-3-nano-30b","name":"Nemotron 3 Nano 30B","provider":"NVIDIA","description":"Nemotron 3 Nano 30B is a standard model from NVIDIA. 10 benchmarks count toward its index, in 6 categories.","logoUrl":null,"releaseDate":"2025-12-15","contextWindowTokens":32000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":32.92616659429418,"overallScoreRange":{"low":26.31276361780191,"high":39.53956957078645},"badges":[],"capabilityScores":{"agentic":26.838986052670226,"coding":40.495274253549766,"reasoning":25.74411354785509,"multimodal":null,"knowledge":34.70817452349276,"multilingual":null,"instruction":24.067777448171256,"math":47.35518199528268},"capabilityCounts":{"math":1,"knowledge":3,"coding":1,"instruction":1,"agentic":2,"reasoning":2},"benchmarks":[{"benchmarkId":"aime2025","benchmarkName":"American Invitational Mathematics Examination 2025","category":"math","score":89.1,"unit":"percent","family":"aime2025","index":47.35518199528268,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaOpennessIndex","benchmarkName":"Artificial Analysis Openness Index","category":"knowledge","score":83.3,"unit":"percent","family":"aaOpennessIndex","index":null,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-openness-index","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmluPro","benchmarkName":"Massive Multitask Language Understanding Professional","category":"knowledge","score":78.3,"unit":"percent","family":"mmlu-pro","index":43.129356157866965,"aboveFit":false,"run":null,"source":"Yubo Wang, Xueguang Ma, Ge Zhang, Yuansheng Ni, Abhranil Chandra, Shiguang Guo, Weiming Ren, Aaran Arulraj, Xuan He, Ziyan Jiang, Tianle Li, Max Ku, Kai Wang, Alex Zhuang, Rongqi Fan, Xiang Yue, Wenhu Chen","sourceUrl":"https://arxiv.org/abs/2406.01574","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gpqa","benchmarkName":"Graduate-Level Google-Proof Q&A","category":"knowledge","score":73,"unit":"percent","family":"gpqa-diamond","index":44.16270600699384,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"liveCodeBenchV6","benchmarkName":"LiveCodeBench v6","category":"coding","score":68.3,"unit":"percent","family":"liveCodeBenchV6","index":40.495274253549766,"aboveFit":false,"run":null,"source":"LiveCodeBench maintainers","sourceUrl":"https://github.com/LiveCodeBench/LiveCodeBench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":39.9,"unit":"percent","family":"gpqa-diamond","index":13.804290467148952,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaIfBench","benchmarkName":"Artificial Analysis IFBench","category":"instruction","score":37.5,"unit":"percent","family":"ifbench-family","index":24.067777448171256,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"tau2Bench","benchmarkName":"τ²-Bench Tool-Agent-User Evaluation","category":"agentic","score":25.4,"unit":"percent","family":"tau2-bench","index":19.644197188147665,"aboveFit":false,"run":null,"source":"Victor Barres, Honghua Dong, Soham Ray, Xujie Si, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2506.07982","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":10.7,"unit":"percent","family":"lcr","index":14.639111082937276,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hle","benchmarkName":"Humanity's Last Exam","category":"knowledge","score":10.6,"unit":"percent","family":"hle","index":35.021547410629296,"aboveFit":false,"run":null,"source":"Center for AI Safety, Scale AI, and thousands of expert contributors","sourceUrl":"https://lastexam.ai/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":4.6,"unit":"percent","family":"hle","index":29.001790940450533,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gdpvalAaNormalized","benchmarkName":"GDPval-AA normalized","category":"agentic","score":0,"unit":"percent","family":"gdpvalAaNormalized","index":34.03377491719279,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":0,"unit":"percent","family":"critpt","index":36.8491160127729,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":270,"timeToFirstToken":8.33,"source":"BenchLM","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":85,"benchmarkCount":10,"categoryCount":6,"confidence":"low","state":"available"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"exaone-4-0-32b","providerId":"lg-ai-research","slug":"exaone-4-0-32b","name":"Exaone 4.0 32B","provider":"LG AI Research","description":"Exaone 4.0 32B is a reasoning model from LG AI Research in the Exaone 4.0 family. 8 benchmarks count toward its index, in 6 categories.","logoUrl":null,"releaseDate":"2025-07-15","contextWindowTokens":128000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":true,"overallScore":32.39911384121525,"overallScoreRange":{"low":23.68215551532201,"high":41.116072167108484},"badges":[],"capabilityScores":{"agentic":5.510089904217452,"coding":null,"reasoning":36.8491160127729,"multimodal":null,"knowledge":37.24986332663683,"multilingual":33.318976275460045,"instruction":20.329734988853243,"math":45.29320389319278},"capabilityCounts":{"math":1,"knowledge":3,"multilingual":1,"instruction":1,"agentic":1,"reasoning":1},"benchmarks":[{"benchmarkId":"aime2025","benchmarkName":"American Invitational Mathematics Examination 2025","category":"math","score":85.3,"unit":"percent","family":"aime2025","index":45.29320389319278,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmluPro","benchmarkName":"Massive Multitask Language Understanding Professional","category":"knowledge","score":81.8,"unit":"percent","family":"mmlu-pro","index":47.538937218636015,"aboveFit":false,"run":null,"source":"Yubo Wang, Xueguang Ma, Ge Zhang, Yuansheng Ni, Abhranil Chandra, Shiguang Guo, Weiming Ren, Aaran Arulraj, Xuan He, Ziyan Jiang, Tianle Li, Max Ku, Kai Wang, Alex Zhuang, Rongqi Fan, Xiang Yue, Wenhu Chen","sourceUrl":"https://arxiv.org/abs/2406.01574","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":62.8,"unit":"percent","family":"gpqa-diamond","index":34.807544722812025,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-french:overall","benchmarkName":"EuroEval French","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":53.8825,"unit":"percent","family":"euroeval","index":45.32638994847433,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/french","measuredAt":null,"stderr":0.5890020691814248,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-portuguese:overall","benchmarkName":"EuroEval Portuguese","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":52.3975,"unit":"percent","family":"euroeval","index":43.31158663723973,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/portuguese","measuredAt":null,"stderr":0.5697806595524281,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-italian:overall","benchmarkName":"EuroEval Italian","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":49.67875,"unit":"percent","family":"euroeval","index":39.62286845379255,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/italian","measuredAt":null,"stderr":0.7055538604529069,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-spanish:overall","benchmarkName":"EuroEval Spanish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":48.3125,"unit":"percent","family":"euroeval","index":37.769181568961386,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/spanish","measuredAt":null,"stderr":0.6750451373797163,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":41.752500000000005,"unit":"percent","family":"euroeval","index":28.86877098195871,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.6433761050116797,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":41.31428571428571,"unit":"percent","family":"euroeval","index":28.274215026478938,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.6402231753738021,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-dutch:overall","benchmarkName":"EuroEval Dutch","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":41.21111111111111,"unit":"percent","family":"euroeval","index":28.134230829770853,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/dutch","measuredAt":null,"stderr":0.5810409985322109,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":40.982499999999995,"unit":"percent","family":"euroeval","index":27.82405815391113,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":1.031845282733802,"costPerTest":null,"latency":null},{"benchmarkId":"aaIfBench","benchmarkName":"Artificial Analysis IFBench","category":"instruction","score":33.5,"unit":"percent","family":"ifbench-family","index":20.329734988853243,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":5,"unit":"percent","family":"hle","index":29.403108038462445,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"tau2Bench","benchmarkName":"τ²-Bench Tool-Agent-User Evaluation","category":"agentic","score":4.1,"unit":"percent","family":"tau2-bench","index":5.510089904217452,"aboveFit":false,"run":null,"source":"Victor Barres, Honghua Dong, Soham Ray, Xujie Si, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2506.07982","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":0,"unit":"percent","family":"critpt","index":36.8491160127729,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":70,"benchmarkCount":8,"categoryCount":6,"confidence":"low","state":"available"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"claude-3-5-sonnet","providerId":"anthropic","slug":"claude-3-5-sonnet","name":"Claude 3.5 Sonnet","provider":"Anthropic","description":"Claude 3.5 Sonnet is a standard model from Anthropic. 12 benchmarks count toward its index, in 4 categories.","logoUrl":null,"releaseDate":"2024-06-01","contextWindowTokens":200000,"modalities":["text"],"openWeights":false,"details":null,"reasoning":false,"overallScore":32.23648234579235,"overallScoreRange":{"low":23.068605419394228,"high":41.404359272190476},"badges":[],"capabilityScores":{"agentic":null,"coding":34.01457134001381,"reasoning":null,"multimodal":44.695184517094724,"knowledge":37.4490898338381,"multilingual":null,"instruction":null,"math":25.226293730769367},"capabilityCounts":{"knowledge":2,"math":6,"multimodal":1,"coding":3},"benchmarks":[{"benchmarkId":"vals-ai:mgsm:overall","benchmarkName":"MGSM","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":92.582,"unit":"percent","family":"vals-ai:mgsm:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mgsm","measuredAt":"2026-01-09","stderr":1.616,"costPerTest":0.004012,"latency":4.1},{"benchmarkId":"vals-ai:medqa:overall","benchmarkName":"MedQA","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":83.191,"unit":"percent","family":"vals-ai:medqa:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medqa","measuredAt":"2026-04-16","stderr":0.669,"costPerTest":0.005964,"latency":5.924},{"benchmarkId":"vals-ai:mmlu_pro:overall","benchmarkName":"MMLU Pro","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":78.404,"unit":"percent","family":"mmlu-pro","index":43.26038370938696,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","measuredAt":"2026-09-01","stderr":0.397,"costPerTest":0.00859,"latency":6.288},{"benchmarkId":"vals-ai:math500:overall","benchmarkName":"MATH 500","category":"math","domain":null,"eligible":true,"ranked":true,"score":72.4,"unit":"percent","family":"math-500","index":20.636077836300586,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/math500","measuredAt":"2026-01-09","stderr":3.907,"costPerTest":0.004988,"latency":4.628},{"benchmarkId":"vals-ai:tax_eval_v2:overall","benchmarkName":"TaxEval v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":70.156,"unit":"percent","family":"vals-ai:tax_eval_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","measuredAt":"2026-09-01","stderr":0.902,"costPerTest":0.006006,"latency":5.943},{"benchmarkId":"vals-ai:mmmu:overall","benchmarkName":"MMMU Pro","category":"multimodal","domain":null,"eligible":true,"ranked":true,"score":68.804,"unit":"percent","family":"mmmu-pro","index":44.695184517094724,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmmu","measuredAt":"2026-09-01","stderr":1.114,"costPerTest":0.010237,"latency":46.011},{"benchmarkId":"vals-ai:mortgage_tax:overall","benchmarkName":"MortgageTax","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":64.07,"unit":"percent","family":"vals-ai:mortgage_tax:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mortgage_tax","measuredAt":"2026-09-01","stderr":1.805,"costPerTest":0.015076,"latency":4.224},{"benchmarkId":"gpqa","benchmarkName":"Graduate-Level Google-Proof Q&A","category":"knowledge","score":59.4,"unit":"percent","family":"gpqa-diamond","index":31.68915762808476,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:gpqa:overall","benchmarkName":"GPQA Diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":59.344,"unit":"percent","family":"gpqa-diamond","index":31.637795958289253,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","measuredAt":"2026-09-01","stderr":2.468,"costPerTest":0.009078,"latency":7.241},{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":54.04040404040404,"unit":"percent","family":"gpqa-diamond","index":26.773482665332864,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.7528555453348655,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:corp_fin_v2:overall","benchmarkName":"CorpFin v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":53.613,"unit":"percent","family":"vals-ai:corp_fin_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","measuredAt":"2026-08-12","stderr":0.981,"costPerTest":0.23772,"latency":0.049},{"benchmarkId":"swebench-experiments:swe-bench-verified:overall","benchmarkName":"SWE-bench Verified","category":"coding","domain":null,"eligible":true,"ranked":true,"score":53,"unit":"percent","family":"swe-bench-verified","index":40.31847915200732,"aboveFit":false,"run":"Engine Labs","source":"SWE-bench team","sourceUrl":"https://github.com/SWE-bench/experiments/tree/main/evaluation/verified","measuredAt":"2026-09-01","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-math-level-5:overall","benchmarkName":"MATH level 5","category":"math","domain":null,"eligible":true,"ranked":true,"score":51.68051359516617,"unit":"percent","family":"epoch-ai:epoch-math-level-5:overall","index":24.101998515916627,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.16250120233614,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:lcb:overall","benchmarkName":"LiveCodeBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":49.628,"unit":"percent","family":"livecodebench","index":30.52216421644671,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","measuredAt":"2026-09-01","stderr":1.14,"costPerTest":0.011661,"latency":11.035},{"benchmarkId":"sweVerified","benchmarkName":"Software Engineering Benchmark Verified","category":"coding","score":49,"unit":"percent","family":"swe-bench-verified","index":37.36824884087255,"aboveFit":false,"run":null,"source":"Carlos E. Jimenez, John Yang, Alexander Wettig, Shunyu Yao, Kexin Pei, Ofir Press, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2310.06770","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"swebench-experiments:swe-bench-lite:overall","benchmarkName":"SWE-bench Lite","category":"coding","domain":null,"eligible":true,"ranked":true,"score":40.83333333333333,"unit":"percent","family":"swe-bench-lite","index":37.54174223953212,"aboveFit":false,"run":"Agentless-1.5","source":"SWE-bench team","sourceUrl":"https://github.com/SWE-bench/experiments/tree/main/evaluation/lite","measuredAt":"2025-09-11","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"swe-bench:swe-bench-verified:overall","benchmarkName":"SWE-bench Verified","category":"coding","domain":null,"eligible":true,"ranked":true,"score":33.6,"unit":"percent","family":"swe-bench-verified","index":26.009862143003698,"aboveFit":false,"run":"SWE-agent","source":"SWE-bench team","sourceUrl":"https://www.swebench.com/","measuredAt":"2026-02-26","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"swe-bench:swe-bench-lite:overall","benchmarkName":"SWE-bench Lite","category":"coding","domain":null,"eligible":true,"ranked":true,"score":32.5,"unit":"percent","family":"swe-bench-lite","index":30.764859685912207,"aboveFit":false,"run":"Moatless Tools","source":"SWE-bench team","sourceUrl":"https://www.swebench.com/","measuredAt":"2025-09-11","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"swebench-experiments:swe-bench-test:overall","benchmarkName":"SWE-bench (full test split)","category":"coding","domain":null,"eligible":true,"ranked":true,"score":21.512641673931995,"unit":"percent","family":"swebench-experiments:swe-bench-test:overall","index":null,"aboveFit":false,"run":"SWE-agent","source":"SWE-bench team","sourceUrl":"https://github.com/SWE-bench/experiments/tree/main/evaluation/test","measuredAt":"2025-12-19","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:aime:overall","benchmarkName":"AIME","category":"math","domain":null,"eligible":true,"ranked":true,"score":10,"unit":"percent","family":"vals-ai:aime:overall","index":19.873023594543326,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/aime","measuredAt":"2026-04-16","stderr":0.938,"costPerTest":0.008821,"latency":9.188},{"benchmarkId":"swebench-experiments:swe-bench-multimodal:overall","benchmarkName":"SWE-bench Multimodal","category":"multimodal","domain":null,"eligible":true,"ranked":true,"score":8.800773694390715,"unit":"percent","family":"swe-bench-multimodal","index":null,"aboveFit":false,"run":"Agentless","source":"SWE-bench team","sourceUrl":"https://github.com/SWE-bench/experiments/tree/main/evaluation/multimodal","measuredAt":"2025-11-17","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"swe-bench:swe-bench-multimodal:overall","benchmarkName":"SWE-bench Multimodal","category":"multimodal","domain":null,"eligible":true,"ranked":true,"score":8.8,"unit":"percent","family":"swe-bench-multimodal","index":null,"aboveFit":false,"run":"SWE-agent Multimodal","source":"SWE-bench team","sourceUrl":"https://www.swebench.com/","measuredAt":"2025-11-17","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":6.527777777777779,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":12.7169131195446,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.2293131815660017,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-2025-02-28-private:overall","benchmarkName":"FrontierMath-2025-02-28-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":1.0344827586206897,"unit":"percent","family":"epoch-ai:epoch-frontiermath-2025-02-28-private:overall","index":30.82739159114545,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0.5951886714450795,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-private:overall","benchmarkName":"FrontierMath-Tier-4-2025-07-01-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":0,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-private:overall","index":43.20235772716562,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-public:overall","benchmarkName":"FrontierMath-Tier-4-2025-07-01-Public","category":"math","domain":null,"eligible":false,"ranked":true,"score":0,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-public:overall","index":null,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-2025-02-28-public:overall","benchmarkName":"FrontierMath-2025-02-28-Public","category":"math","domain":null,"eligible":false,"ranked":true,"score":0,"unit":"percent","family":"epoch-ai:epoch-frontiermath-2025-02-28-public:overall","index":null,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":3,"outputPerMillion":15,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":{"elo":1297.8644172871832,"eloUpper":1301.0554220377278,"eloLower":1294.6734125366384,"votes":87743,"rank":256,"rankUpper":null,"rankLower":null,"arenaName":"Claude 3.5 Sonnet-20241022","organization":"anthropic","updatedAt":"2026-11-09"},"quality":{"coveragePercent":55.00000000000001,"benchmarkCount":12,"categoryCount":4,"confidence":"low","state":"available"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:swe-bench-team","name":"SWE-bench team","url":"https://github.com","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"magistral-small","providerId":"mistral","slug":"magistral-small","name":"Magistral Small","provider":"Mistral","description":"Magistral Small is a reasoning model from Mistral in the Magistral family. 18 benchmarks count toward its index, in 7 categories.","logoUrl":null,"releaseDate":"2025-06-10","contextWindowTokens":128000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":true,"overallScore":31.388558572528556,"overallScoreRange":{"low":27.077253021933064,"high":35.699864123124044},"badges":[],"capabilityScores":{"agentic":24.756813872855787,"coding":36.26450548641098,"reasoning":28.155430062312238,"multimodal":40.00749753813548,"knowledge":31.037371903518306,"multilingual":null,"instruction":12.199492639836564,"math":41.25970985524741},"capabilityCounts":{"math":4,"coding":2,"knowledge":4,"multimodal":1,"agentic":2,"instruction":1,"reasoning":4},"benchmarks":[{"benchmarkId":"math500","benchmarkName":"MATH-500 Problem Set","category":"math","score":95.9,"unit":"percent","family":"math-500","index":45.469895828402784,"aboveFit":false,"run":null,"source":"Dan Hendrycks, Collin Burns, Saurav Kadavath, Akul Arora, Steven Basart, Eric Tang, Dawn Song, Jacob Steinhardt","sourceUrl":"https://arxiv.org/abs/2103.03874","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:mgsm:overall","benchmarkName":"MGSM","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":86.254,"unit":"percent","family":"vals-ai:mgsm:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mgsm","measuredAt":"2026-01-09","stderr":2.065,"costPerTest":0.001465,"latency":8.038},{"benchmarkId":"vals-ai:medqa:overall","benchmarkName":"MedQA","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":82.358,"unit":"percent","family":"vals-ai:medqa:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medqa","measuredAt":"2026-04-16","stderr":0.348,"costPerTest":0.001729,"latency":7.973},{"benchmarkId":"vals-ai:aime:overall","benchmarkName":"AIME","category":"math","domain":null,"eligible":true,"ranked":true,"score":80.679,"unit":"percent","family":"vals-ai:aime:overall","index":50.76298089631024,"aboveFit":true,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/aime","measuredAt":"2026-04-16","stderr":1.208,"costPerTest":0.025112,"latency":267.982},{"benchmarkId":"vals-ai:lcb:overall","benchmarkName":"LiveCodeBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":72.131,"unit":"percent","family":"livecodebench","index":49.193831092695355,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","measuredAt":"2026-09-01","stderr":1.123,"costPerTest":0.015218,"latency":100.782},{"benchmarkId":"aime2024","benchmarkName":"American Invitational Mathematics Examination 2024","category":"math","score":70.7,"unit":"percent","family":"aime2024","index":35.72183967072139,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gpqa","benchmarkName":"Graduate-Level Google-Proof Q&A","category":"knowledge","score":68.2,"unit":"percent","family":"gpqa-diamond","index":39.760277167378874,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:mmmu:overall","benchmarkName":"MMMU Pro","category":"multimodal","domain":null,"eligible":true,"ranked":true,"score":65.202,"unit":"percent","family":"mmmu-pro","index":40.00749753813548,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmmu","measuredAt":"2026-09-01","stderr":1.145,"costPerTest":0.004413,"latency":19.095},{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":64.1,"unit":"percent","family":"gpqa-diamond","index":35.99986920020775,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aime2025","benchmarkName":"American Invitational Mathematics Examination 2025","category":"math","score":62.8,"unit":"percent","family":"aime2025","index":33.08412302555524,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:mmlu_pro:overall","benchmarkName":"MMLU Pro","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":62.135,"unit":"percent","family":"mmlu-pro","index":22.763391058629328,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","measuredAt":"2026-09-01","stderr":0.452,"costPerTest":0.00355,"latency":16.124},{"benchmarkId":"vals-ai:mortgage_tax:overall","benchmarkName":"MortgageTax","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":62.122,"unit":"percent","family":"vals-ai:mortgage_tax:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mortgage_tax","measuredAt":"2026-09-01","stderr":0.952,"costPerTest":0.004484,"latency":10.648},{"benchmarkId":"vals-ai:tax_eval_v2:overall","benchmarkName":"TaxEval v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":60.302,"unit":"percent","family":"vals-ai:tax_eval_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","measuredAt":"2026-09-01","stderr":0.967,"costPerTest":0.003909,"latency":16.441},{"benchmarkId":"vals-ai:gpqa:overall","benchmarkName":"GPQA Diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":58.334,"unit":"percent","family":"gpqa-diamond","index":30.711451556620272,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","measuredAt":"2026-09-01","stderr":2.487,"costPerTest":0.007058,"latency":46.252},{"benchmarkId":"liveCodeBenchV5","benchmarkName":"LiveCodeBench v5","category":"coding","score":55.8,"unit":"percent","family":"liveCodeBenchV5","index":null,"aboveFit":false,"run":null,"source":"LiveCodeBench maintainers","sourceUrl":"https://github.com/LiveCodeBench/LiveCodeBench","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"liveCodeBenchV6","benchmarkName":"LiveCodeBench v6","category":"coding","score":47.4,"unit":"percent","family":"liveCodeBenchV6","index":23.335179880126613,"aboveFit":false,"run":null,"source":"LiveCodeBench maintainers","sourceUrl":"https://github.com/LiveCodeBench/LiveCodeBench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:corp_fin_v2:overall","benchmarkName":"CorpFin v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":44.017,"unit":"percent","family":"vals-ai:corp_fin_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","measuredAt":"2026-08-12","stderr":0.965,"costPerTest":0.030437,"latency":14.053},{"benchmarkId":"vals-ai:legal_bench:overall","benchmarkName":"LegalBench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":40.023,"unit":"percent","family":"vals-ai:legal_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","measuredAt":"2026-10-13","stderr":0.472,"costPerTest":null,"latency":6.67},{"benchmarkId":"tau2Bench","benchmarkName":"τ²-Bench Tool-Agent-User Evaluation","category":"agentic","score":26.6,"unit":"percent","family":"tau2-bench","index":20.44048492245359,"aboveFit":false,"run":null,"source":"Victor Barres, Honghua Dong, Soham Ray, Xujie Si, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2506.07982","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaIfBench","benchmarkName":"Artificial Analysis IFBench","category":"instruction","score":24.8,"unit":"percent","family":"ifbench-family","index":12.199492639836564,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"omniscienceAccuracy","benchmarkName":"Artificial Analysis Omniscience Accuracy","category":"knowledge","score":18,"unit":"percent","family":"omniscienceAccuracy","index":33.37455817969624,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:terminal-bench:overall","benchmarkName":"Terminal-Bench 1.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":15,"unit":"percent","family":"vals-ai:terminal-bench:overall","index":29.07314282325798,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench","measuredAt":"2026-01-12","stderr":3.992,"costPerTest":0.22067,"latency":607.926},{"benchmarkId":"arc-prize:v1-public-eval:overall","benchmarkName":"ARC-AGI-1 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":8.62,"unit":"percent","family":"arc-prize:v1-public-eval:overall","index":null,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.0289,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":7.6,"unit":"percent","family":"hle","index":32.011669175539915,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"arc-prize:v1-semi-private:overall","benchmarkName":"ARC-AGI-1 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":5,"unit":"percent","family":"arc-prize:v1-semi-private:overall","index":29.286039375062796,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.0399,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":0,"unit":"percent","family":"lcr","index":7.298049459170706,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":0,"unit":"percent","family":"critpt","index":36.8491160127729,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"arc-prize:v2-public-eval:overall","benchmarkName":"ARC-AGI-2 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":0,"unit":"percent","family":"arc-prize:v2-public-eval:overall","index":null,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.0495,"latency":null},{"benchmarkId":"arc-prize:v2-semi-private:overall","benchmarkName":"ARC-AGI-2 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":0,"unit":"percent","family":"arc-prize:v2-semi-private:overall","index":39.188515402242544,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.0488,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":95,"benchmarkCount":18,"categoryCount":7,"confidence":"medium","state":"available"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:arc-prize-foundation","name":"ARC Prize Foundation","url":"https://arcprize.org","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"lfm2-5-8b-a1b","providerId":"liquidai","slug":"lfm2-5-8b-a1b","name":"LFM2.5-8B-A1B","provider":"LiquidAI","description":"LFM2.5-8B-A1B is a reasoning model from LiquidAI. 13 benchmarks count toward its index, in 6 categories.","logoUrl":null,"releaseDate":"2026-05-28","contextWindowTokens":128000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":true,"overallScore":30.609719329383516,"overallScoreRange":{"low":23.12496824290254,"high":38.09447041586449},"badges":[],"capabilityScores":{"agentic":46.27682294516874,"coding":null,"reasoning":22.073582735971804,"multimodal":null,"knowledge":26.035259977345927,"multilingual":5,"instruction":44.76464198436857,"math":27.09924752852446},"capabilityCounts":{"instruction":2,"math":3,"agentic":2,"knowledge":3,"multilingual":1,"reasoning":2},"benchmarks":[{"benchmarkId":"ifeval","benchmarkName":"Instruction-Following Eval","category":"instruction","score":91.84,"unit":"percent","family":"ifeval","index":48.14035227470105,"aboveFit":true,"run":null,"source":"Jeffrey Zhou, Tianjian Lu, Swaroop Mishra, Siddhartha Brahma, Sujoy Basu, Yi Luan, Denny Zhou, Le Hou","sourceUrl":"https://arxiv.org/abs/2311.07911","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"math500","benchmarkName":"MATH-500 Problem Set","category":"math","score":88.76,"unit":"percent","family":"math-500","index":37.92464219165343,"aboveFit":false,"run":null,"source":"Dan Hendrycks, Collin Burns, Saurav Kadavath, Akul Arora, Steven Basart, Eric Tang, Dawn Song, Jacob Steinhardt","sourceUrl":"https://arxiv.org/abs/2103.03874","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"tau2Bench","benchmarkName":"τ²-Bench Tool-Agent-User Evaluation","category":"agentic","score":88.07,"unit":"percent","family":"tau2-bench","index":61.230324112274694,"aboveFit":true,"run":null,"source":"Victor Barres, Honghua Dong, Soham Ray, Xujie Si, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2506.07982","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"ifBench","benchmarkName":"Instruction Following Benchmark","category":"instruction","score":56.47,"unit":"percent","family":"ifbench-family","index":41.79544381148693,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaIfBench","benchmarkName":"Artificial Analysis IFBench","category":"instruction","score":55.6,"unit":"percent","family":"ifbench-family","index":40.98241957658527,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":51.3,"unit":"percent","family":"gpqa-diamond","index":24.2600589612345,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aime2026","benchmarkName":"AIME 2026","category":"math","score":50,"unit":"percent","family":"aime2026","index":21.28800266556528,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"bfclV4","benchmarkName":"Berkeley Function Calling Leaderboard v4","category":"agentic","score":49.73,"unit":"percent","family":"bfclV4","index":31.323321778062784,"aboveFit":false,"run":null,"source":"Shishir G. Patil et al.","sourceUrl":"https://gorilla.cs.berkeley.edu/leaderboard.html","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aime2025","benchmarkName":"American Invitational Mathematics Examination 2025","category":"math","score":42.53,"unit":"percent","family":"aime2025","index":22.085097728354672,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-french:overall","benchmarkName":"EuroEval French","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":29.605,"unit":"percent","family":"euroeval","index":12.387408542078362,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/french","measuredAt":null,"stderr":0.8139880604161218,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-spanish:overall","benchmarkName":"EuroEval Spanish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":24.12875,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/spanish","measuredAt":null,"stderr":1.0529964684176296,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-portuguese:overall","benchmarkName":"EuroEval Portuguese","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":21.970000000000002,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/portuguese","measuredAt":null,"stderr":0.9778739693846032,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-dutch:overall","benchmarkName":"EuroEval Dutch","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":21.060000000000002,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/dutch","measuredAt":null,"stderr":0.7128009435916771,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-italian:overall","benchmarkName":"EuroEval Italian","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":20.819999999999997,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/italian","measuredAt":null,"stderr":1.1332958020746393,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":20.5625,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.8687778772505663,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":14.275,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":1.1995389739395714,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":10.788571428571428,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.8047030127789446,"costPerTest":null,"latency":null},{"benchmarkId":"omniscienceAccuracy","benchmarkName":"Artificial Analysis Omniscience Accuracy","category":"knowledge","score":9.4,"unit":"percent","family":"omniscienceAccuracy","index":22.536356716784226,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":6.9,"unit":"percent","family":"hle","index":31.309364254019055,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":0,"unit":"percent","family":"lcr","index":7.298049459170706,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":0,"unit":"percent","family":"critpt","index":36.8491160127729,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":335,"timeToFirstToken":7.62,"source":"BenchLM","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":70,"benchmarkCount":13,"categoryCount":6,"confidence":"low","state":"available"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"sarvam-30b","providerId":"sarvam","slug":"sarvam-30b","name":"Sarvam 30B","provider":"Sarvam","description":"Sarvam 30B is a reasoning model from Sarvam. 14 benchmarks count toward its index, in 6 categories.","logoUrl":null,"releaseDate":"2026-03-06","contextWindowTokens":64000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":true,"overallScore":30.349139829797103,"overallScoreRange":{"low":24.5275255480732,"high":36.170754111521006},"badges":[],"capabilityScores":{"agentic":26.818704835103805,"coding":29.37869146688079,"reasoning":22.36396523347664,"multimodal":null,"knowledge":36.76428616082452,"multilingual":null,"instruction":13.78816068504672,"math":37.93470684093113},"capabilityCounts":{"math":3,"coding":2,"knowledge":4,"agentic":2,"instruction":1,"reasoning":2},"benchmarks":[{"benchmarkId":"math500","benchmarkName":"MATH-500 Problem Set","category":"math","score":97,"unit":"percent","family":"math-500","index":46.63232986207564,"aboveFit":false,"run":null,"source":"Dan Hendrycks, Collin Burns, Saurav Kadavath, Akul Arora, Steven Basart, Eric Tang, Dawn Song, Jacob Steinhardt","sourceUrl":"https://arxiv.org/abs/2103.03874","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"humaneval","benchmarkName":"Evaluating Large Language Models Trained on Code","category":"coding","score":92.1,"unit":"percent","family":"humaneval","index":32.452497759644416,"aboveFit":false,"run":null,"source":"Mark Chen, Jerry Tworek, Heewoo Jun, Qiming Yuan, Henrique Ponde de Oliveira Pinto, Jared Kaplan, Harri Edwards, Yuri Burda, Nicholas Joseph, Greg Brockman, Alex Ray, Raul Puri, Gretchen Krueger, Michael Petrov, Heidy Khlaaf, Girish Sastry, Pamela Mishkin, Brooke Chan, Scott Gray, Nick Ryder, Mikhail Pavlov, Alethea Power, Lukasz Kaiser, Mohammad Bavarian, Clemens Winter, Philippe Tillet, Felipe Petroski Such, Dave Cummings, Matthias Plappert, Fotios Chantzis, Elizabeth Barnes, Ariel Herbert-Voss, William Hebgen Guss, Alex Nichol, Alex Paino, Nikolas Tezak, Jie Tang, Igor Babuschkin, Suchir Balaji, Shantanu Jain, William Saunders, Christopher Hesse, Andrew N. Carr, Jan Leike, Josh Achiam, Vedant Misra, Evan Morikawa, Alec Radford, Matthew Knight, Miles Brundage, Mira Murati, Katie Mayer, Peter Welinder, Bob McGrew, Dario Amodei, Sam McCandlish, Ilya Sutskever, Wojciech Zaremba","sourceUrl":"https://arxiv.org/abs/2107.03374","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmlu","benchmarkName":"Massive Multitask Language Understanding","category":"knowledge","score":85.1,"unit":"percent","family":"mmlu","index":33.14104472901991,"aboveFit":false,"run":null,"source":"Dan Hendrycks, Collin Burns, Steven Basart, Andy Zou, Mantas Mazeika, Dawn Song, Jacob Steinhardt","sourceUrl":"https://arxiv.org/abs/2009.03300","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmluPro","benchmarkName":"Massive Multitask Language Understanding Professional","category":"knowledge","score":80,"unit":"percent","family":"mmlu-pro","index":45.271152673097646,"aboveFit":false,"run":null,"source":"Yubo Wang, Xueguang Ma, Ge Zhang, Yuansheng Ni, Abhranil Chandra, Shiguang Guo, Weiming Ren, Aaran Arulraj, Xuan He, Ziyan Jiang, Tianle Li, Max Ku, Kai Wang, Alex Zhuang, Rongqi Fan, Xiang Yue, Wenhu Chen","sourceUrl":"https://arxiv.org/abs/2406.01574","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hmmtNov2025","benchmarkName":"Harvard-MIT Mathematics Tournament November 2025","category":"math","score":74.2,"unit":"percent","family":"hmmtNov2025","index":33.97909623484528,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hmmtFeb2025","benchmarkName":"Harvard-MIT Mathematics Tournament February 2025","category":"math","score":73.3,"unit":"percent","family":"hmmtFeb2025","index":33.19269442587246,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gpqaDiamond","benchmarkName":"GPQA Diamond","category":"reasoning","score":66.5,"unit":"percent","family":"gpqa-diamond","index":38.20108362001523,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":63.3,"unit":"percent","family":"gpqa-diamond","index":35.26613106027192,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"browseComp","benchmarkName":"BrowseComp","category":"agentic","score":35.5,"unit":"percent","family":"browseComp","index":27.954697163573336,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"tau2Bench","benchmarkName":"τ²-Bench Tool-Agent-User Evaluation","category":"agentic","score":34.5,"unit":"percent","family":"tau2-bench","index":25.682712506634278,"aboveFit":false,"run":null,"source":"Victor Barres, Honghua Dong, Soham Ray, Xujie Si, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2506.07982","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"sweVerified","benchmarkName":"Software Engineering Benchmark Verified","category":"coding","score":34,"unit":"percent","family":"swe-bench-verified","index":26.304885174117175,"aboveFit":false,"run":null,"source":"Carlos E. Jimenez, John Yang, Alexander Wettig, Shunyu Yao, Kexin Pei, Ofir Press, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2310.06770","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaIfBench","benchmarkName":"Artificial Analysis IFBench","category":"instruction","score":26.5,"unit":"percent","family":"ifbench-family","index":13.78816068504672,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":7.5,"unit":"percent","family":"hle","index":31.911339901036932,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":0.3,"unit":"percent","family":"critpt","index":37.42988100778257,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":0,"unit":"percent","family":"lcr","index":7.298049459170706,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":85,"benchmarkCount":14,"categoryCount":6,"confidence":"medium","state":"available"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"deepseek-v3","providerId":"deepseek","slug":"deepseek-v3","name":"DeepSeek V3","provider":"DeepSeek","description":"DeepSeek V3 is a standard model from DeepSeek in the DeepSeek family. 22 benchmarks count toward its index, in 6 categories.","logoUrl":null,"releaseDate":"2024-12-26","contextWindowTokens":128000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":29.93517805464482,"overallScoreRange":{"low":24.9877224711746,"high":34.88263363811504},"badges":[],"capabilityScores":{"agentic":25.976341007172138,"coding":31.133995067921116,"reasoning":34.11643583783529,"multimodal":null,"knowledge":34.25026689326069,"multilingual":null,"instruction":29.06258573993692,"math":27.15886974149815},"capabilityCounts":{"math":6,"instruction":2,"knowledge":5,"reasoning":3,"coding":4,"agentic":2},"benchmarks":[{"benchmarkId":"vals-ai:mgsm:overall","benchmarkName":"MGSM","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":92.146,"unit":"percent","family":"vals-ai:mgsm:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mgsm","measuredAt":"2026-01-09","stderr":1.663,"costPerTest":0.000364,"latency":15.681},{"benchmarkId":"math500","benchmarkName":"MATH-500 Problem Set","category":"math","score":90.2,"unit":"percent","family":"math-500","index":39.44637401755246,"aboveFit":false,"run":null,"source":"Dan Hendrycks, Collin Burns, Saurav Kadavath, Akul Arora, Steven Basart, Eric Tang, Dawn Song, Jacob Steinhardt","sourceUrl":"https://arxiv.org/abs/2103.03874","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"ifeval","benchmarkName":"Instruction-Following Eval","category":"instruction","score":86.1,"unit":"percent","family":"ifeval","index":36.58057269174225,"aboveFit":false,"run":null,"source":"Jeffrey Zhou, Tianjian Lu, Swaroop Mishra, Siddhartha Brahma, Sujoy Basu, Yi Luan, Denny Zhou, Le Hou","sourceUrl":"https://arxiv.org/abs/2311.07911","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:medqa:overall","benchmarkName":"MedQA","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":80.9,"unit":"percent","family":"vals-ai:medqa:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medqa","measuredAt":"2026-04-16","stderr":0.359,"costPerTest":0.000578,"latency":6.816},{"benchmarkId":"vals-ai:legal_bench:overall","benchmarkName":"LegalBench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":80.762,"unit":"percent","family":"vals-ai:legal_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","measuredAt":"2026-10-13","stderr":0.429,"costPerTest":null,"latency":4.395},{"benchmarkId":"vals-ai:math500:overall","benchmarkName":"MATH 500","category":"math","domain":null,"eligible":true,"ranked":true,"score":80.4,"unit":"percent","family":"math-500","index":29.090143535739635,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/math500","measuredAt":"2026-01-09","stderr":1.775,"costPerTest":0.000563,"latency":7.94},{"benchmarkId":"mmluPro","benchmarkName":"Massive Multitask Language Understanding Professional","category":"knowledge","score":75.9,"unit":"percent","family":"mmlu-pro","index":40.10564343048248,"aboveFit":false,"run":null,"source":"Yubo Wang, Xueguang Ma, Ge Zhang, Yuansheng Ni, Abhranil Chandra, Shiguang Guo, Weiming Ren, Aaran Arulraj, Xuan He, Ziyan Jiang, Tianle Li, Max Ku, Kai Wang, Alex Zhuang, Rongqi Fan, Xiang Yue, Wenhu Chen","sourceUrl":"https://arxiv.org/abs/2406.01574","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:mmlu_pro:overall","benchmarkName":"MMLU Pro","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":73.82,"unit":"percent","family":"mmlu-pro","index":37.48509240008257,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","measuredAt":"2026-09-01","stderr":0.424,"costPerTest":0.001338,"latency":11.36},{"benchmarkId":"vals-ai:tax_eval_v2:overall","benchmarkName":"TaxEval v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":67.907,"unit":"percent","family":"vals-ai:tax_eval_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","measuredAt":"2026-09-01","stderr":0.918,"costPerTest":0.00056,"latency":27.696},{"benchmarkId":"epoch-ai:epoch-math-level-5:overall","benchmarkName":"MATH level 5","category":"math","domain":null,"eligible":true,"ranked":true,"score":64.85083081570997,"unit":"percent","family":"epoch-ai:epoch-math-level-5:overall","index":30.49955093834629,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.0485766934209062,"costPerTest":null,"latency":null},{"benchmarkId":"gpqa","benchmarkName":"Graduate-Level Google-Proof Q&A","category":"knowledge","score":59.1,"unit":"percent","family":"gpqa-diamond","index":31.41400582560883,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":56.53409090909091,"unit":"percent","family":"gpqa-diamond","index":29.060624121098737,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.763990029950893,"costPerTest":null,"latency":null},{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":55.7,"unit":"percent","family":"gpqa-diamond","index":28.295618730881557,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:gpqa:overall","benchmarkName":"GPQA Diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":54.546,"unit":"percent","family":"gpqa-diamond","index":27.23720146402412,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","measuredAt":"2026-09-01","stderr":2.504,"costPerTest":0.001468,"latency":21.282},{"benchmarkId":"vals-ai:corp_fin_v2:overall","benchmarkName":"CorpFin v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":52.486,"unit":"percent","family":"vals-ai:corp_fin_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","measuredAt":"2026-08-12","stderr":0.98,"costPerTest":0.052026,"latency":28.322},{"benchmarkId":"longBenchV2","benchmarkName":"LongBench v2","category":"reasoning","score":48.7,"unit":"percent","family":"longBenchV2","index":38.09998264190242,"aboveFit":false,"run":null,"source":"LongBench v2 authors","sourceUrl":"https://arxiv.org/abs/2412.15204","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"sweVerified","benchmarkName":"Software Engineering Benchmark Verified","category":"coding","score":42,"unit":"percent","family":"swe-bench-verified","index":32.20534579638671,"aboveFit":false,"run":null,"source":"Carlos E. Jimenez, John Yang, Alexander Wettig, Shunyu Yao, Kexin Pei, Ofir Press, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2310.06770","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aime2024","benchmarkName":"American Invitational Mathematics Examination 2024","category":"math","score":39.2,"unit":"percent","family":"aime2024","index":21.097050650571745,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"liveCodeBench","benchmarkName":"LiveCodeBench: Holistic and Contamination Free Evaluation of Large Language Models for Code","category":"coding","score":37.6,"unit":"percent","family":"livecodebench","index":20.542036714001767,"aboveFit":false,"run":null,"source":"Naman Jain, King Han, Alex Gu, Wen-Ding Li, Fanjia Yan, Tianjun Zhang, Sida Wang, Armando Solar-Lezama, Koushik Sen, Ion Stoica","sourceUrl":"https://arxiv.org/abs/2403.07974","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"swebench-experiments:swe-bench-lite:overall","benchmarkName":"SWE-bench Lite","category":"coding","domain":null,"eligible":true,"ranked":true,"score":36.666666666666664,"unit":"percent","family":"swe-bench-lite","index":34.15330096272216,"aboveFit":false,"run":"KGCompass","source":"SWE-bench team","sourceUrl":"https://github.com/SWE-bench/experiments/tree/main/evaluation/lite","measuredAt":"2025-09-11","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaSciCode","benchmarkName":"Artificial Analysis SciCode","category":"coding","score":35.8,"unit":"percent","family":"scicode-family","index":42.51194148415871,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/scicode","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaIfBench","benchmarkName":"Artificial Analysis IFBench","category":"instruction","score":34.8,"unit":"percent","family":"ifbench-family","index":21.544598788131594,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"swe-bench:swe-bench-lite:overall","benchmarkName":"SWE-bench Lite","category":"coding","domain":null,"eligible":true,"ranked":true,"score":30.67,"unit":"percent","family":"swe-bench-lite","index":29.276656277137274,"aboveFit":false,"run":"Moatless Tools","source":"SWE-bench team","sourceUrl":"https://www.swebench.com/","measuredAt":"2025-09-11","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"swebench-experiments:swe-bench-lite:overall","benchmarkName":"SWE-bench Lite","category":"coding","domain":null,"eligible":true,"ranked":true,"score":30.666666666666664,"unit":"percent","family":"swe-bench-lite","index":29.27394552411582,"aboveFit":false,"run":"Moatless Tools","source":"SWE-bench team","sourceUrl":"https://github.com/SWE-bench/experiments/tree/main/evaluation/lite","measuredAt":"2025-09-11","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":29.3,"unit":"percent","family":"lcr","index":27.40020885883055,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:aime:overall","benchmarkName":"AIME","category":"math","domain":null,"eligible":true,"ranked":true,"score":27.5,"unit":"percent","family":"vals-ai:aime:overall","index":27.521324402151254,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/aime","measuredAt":"2026-04-16","stderr":0.73,"costPerTest":0.003364,"latency":58.801},{"benchmarkId":"omniscienceAccuracy","benchmarkName":"Artificial Analysis Omniscience Accuracy","category":"knowledge","score":25.5,"unit":"percent","family":"omniscienceAccuracy","index":42.82647806014276,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"simpleQa","benchmarkName":"Measuring Short-Form Factuality in Large Language Models","category":"knowledge","score":24.9,"unit":"percent","family":"simpleQa","index":33.655173790988115,"aboveFit":false,"run":null,"source":"Jason Wei, Najoung Kim, Hyung Won Chung, Yu-An Chung, Siddhartha Papay, Yifeng Lu, Hannaneh Hajishirzi, Luke Zettlemoyer","sourceUrl":"https://arxiv.org/abs/2411.04368","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"tau2Bench","benchmarkName":"τ²-Bench Tool-Agent-User Evaluation","category":"agentic","score":22.8,"unit":"percent","family":"tau2-bench","index":17.91890709715149,"aboveFit":false,"run":null,"source":"Victor Barres, Honghua Dong, Soham Ray, Xujie Si, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2506.07982","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":15.833333333333332,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":18.130829934005266,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":4.3365008819311806,"costPerTest":null,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":2.9,"unit":"percent","family":"hle","index":27.296193273899878,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-2025-02-28-private:overall","benchmarkName":"FrontierMath-2025-02-28-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":1.7241379310344827,"unit":"percent","family":"epoch-ai:epoch-frontiermath-2025-02-28-private:overall","index":31.436203747268305,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0.7657032895812118,"costPerTest":null,"latency":null},{"benchmarkId":"gdpvalAaNormalized","benchmarkName":"GDPval-AA normalized","category":"agentic","score":0,"unit":"percent","family":"gdpvalAaNormalized","index":34.03377491719279,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":0,"unit":"percent","family":"critpt","index":36.8491160127729,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-2025-02-28-public:overall","benchmarkName":"FrontierMath-2025-02-28-Public","category":"math","domain":null,"eligible":false,"ranked":true,"score":0,"unit":"percent","family":"epoch-ai:epoch-frontiermath-2025-02-28-public:overall","index":null,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":0.27,"outputPerMillion":1.1,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":37,"timeToFirstToken":1.25,"source":"BenchLM","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":{"elo":1332.5033093019706,"eloUpper":1337.2388748770077,"eloLower":1327.7677437269335,"votes":21770,"rank":229,"rankUpper":null,"rankLower":null,"arenaName":"DeepSeek-V3","organization":"deepseek","updatedAt":"2026-11-09"},"quality":{"coveragePercent":85,"benchmarkCount":22,"categoryCount":6,"confidence":"medium","state":"available"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:swe-bench-team","name":"SWE-bench team","url":"https://github.com","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gemini-2-0-flash","providerId":"google","slug":"gemini-2-0-flash","name":"Gemini 2.0 Flash","provider":"Google","description":"Gemini 2.0 Flash is a standard model from Google. 15 benchmarks count toward its index, in 7 categories.","logoUrl":null,"releaseDate":"2025-02-05","contextWindowTokens":1000000,"modalities":["text"],"openWeights":false,"details":null,"reasoning":false,"overallScore":29.46128267753719,"overallScoreRange":{"low":24.786206630325594,"high":34.13635872474879},"badges":[],"capabilityScores":{"agentic":22.364846947026248,"coding":11.199705981107165,"reasoning":35.13562547876211,"multimodal":37.020516377611344,"knowledge":35.74364142801314,"multilingual":null,"instruction":26.59095610821092,"math":28.122286406967007},"capabilityCounts":{"math":4,"multimodal":1,"knowledge":4,"instruction":1,"reasoning":3,"agentic":1,"coding":1},"benchmarks":[{"benchmarkId":"epoch-ai:epoch-math-level-5:overall","benchmarkName":"MATH level 5","category":"math","domain":null,"eligible":true,"ranked":true,"score":82.16578549848943,"unit":"percent","family":"epoch-ai:epoch-math-level-5:overall","index":38.910383298730515,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0.8734081505176936,"costPerTest":null,"latency":null},{"benchmarkId":"mmmu","benchmarkName":"Massive Multi-discipline Multimodal Understanding","category":"multimodal","score":69.3,"unit":"percent","family":"mmmu","index":37.020516377611344,"aboveFit":false,"run":null,"source":"MMMU authors","sourceUrl":"https://arxiv.org/abs/2401.05508","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gpqa","benchmarkName":"Graduate-Level Google-Proof Q&A","category":"knowledge","score":65.2,"unit":"percent","family":"gpqa-diamond","index":37.00875914261952,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":64.14141414141415,"unit":"percent","family":"gpqa-diamond","index":36.03785311906807,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.739561817902757,"costPerTest":null,"latency":null},{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":62.3,"unit":"percent","family":"gpqa-diamond","index":34.348958385352134,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaIfBench","benchmarkName":"Artificial Analysis IFBench","category":"instruction","score":40.2,"unit":"percent","family":"ifbench-family","index":26.59095610821092,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":31.3,"unit":"percent","family":"lcr","index":28.77236991000187,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":31.11111111111111,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":27.01935007714964,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":6.311218144454454,"costPerTest":null,"latency":null},{"benchmarkId":"simpleQa","benchmarkName":"Measuring Short-Form Factuality in Large Language Models","category":"knowledge","score":29.9,"unit":"percent","family":"simpleQa","index":37.929943384019225,"aboveFit":false,"run":null,"source":"Jason Wei, Najoung Kim, Hyung Won Chung, Yu-An Chung, Siddhartha Papay, Yifeng Lu, Hannaneh Hajishirzi, Luke Zettlemoyer","sourceUrl":"https://arxiv.org/abs/2411.04368","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aime2025","benchmarkName":"American Invitational Mathematics Examination 2025","category":"math","score":29.7,"unit":"percent","family":"aime2025","index":15.123208504719571,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"tau2Bench","benchmarkName":"τ²-Bench Tool-Agent-User Evaluation","category":"agentic","score":29.5,"unit":"percent","family":"tau2-bench","index":22.364846947026248,"aboveFit":false,"run":null,"source":"Victor Barres, Honghua Dong, Soham Ray, Xujie Si, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2506.07982","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"omniscienceAccuracy","benchmarkName":"Artificial Analysis Omniscience Accuracy","category":"knowledge","score":23.5,"unit":"percent","family":"omniscienceAccuracy","index":40.30596609202369,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"sweVerified","benchmarkName":"Software Engineering Benchmark Verified","category":"coding","score":21.4,"unit":"percent","family":"swe-bench-verified","index":17.011659694042656,"aboveFit":false,"run":null,"source":"Carlos E. Jimenez, John Yang, Alexander Wettig, Shunyu Yao, Kexin Pei, Ofir Press, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2310.06770","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"swe-bench:swe-bench-verified:overall","benchmarkName":"SWE-bench Verified","category":"coding","domain":null,"eligible":true,"ranked":true,"score":13.52,"unit":"percent","family":"swe-bench-verified","index":11.199705981107165,"aboveFit":false,"run":"mini-SWE-agent","source":"SWE-bench team","sourceUrl":"https://www.swebench.com/","measuredAt":"2026-02-26","stderr":null,"costPerTest":0,"latency":null},{"benchmarkId":"swebench-experiments:swe-bench-verified:overall","benchmarkName":"SWE-bench Verified","category":"coding","domain":null,"eligible":true,"ranked":true,"score":13.52,"unit":"percent","family":"swe-bench-verified","index":11.199705981107165,"aboveFit":false,"run":"mini-SWE-agent","source":"SWE-bench team","sourceUrl":"https://github.com/SWE-bench/experiments/tree/main/evaluation/verified","measuredAt":"2026-09-01","stderr":null,"costPerTest":0,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":4.3,"unit":"percent","family":"hle","index":28.700803116941593,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-2025-02-28-private:overall","benchmarkName":"FrontierMath-2025-02-28-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":1.7241379310344827,"unit":"percent","family":"epoch-ai:epoch-frontiermath-2025-02-28-private:overall","index":31.436203747268305,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0.7657032895812118,"costPerTest":null,"latency":null},{"benchmarkId":"arc-prize:v2-semi-private:overall","benchmarkName":"ARC-AGI-2 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":1.3,"unit":"percent","family":"arc-prize:v2-semi-private:overall","index":39.78539051351155,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.004,"latency":null},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":0,"unit":"percent","family":"critpt","index":36.8491160127729,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-2025-02-28-public:overall","benchmarkName":"FrontierMath-2025-02-28-Public","category":"math","domain":null,"eligible":false,"ranked":true,"score":0,"unit":"percent","family":"epoch-ai:epoch-frontiermath-2025-02-28-public:overall","index":null,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":95,"benchmarkCount":15,"categoryCount":7,"confidence":"medium","state":"available"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:swe-bench-team","name":"SWE-bench team","url":"https://www.swebench.com","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:arc-prize-foundation","name":"ARC Prize Foundation","url":"https://arcprize.org","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"lfm2-5-2-6b","providerId":"liquidai","slug":"lfm2-5-2-6b","name":"LFM2.5-2.6B","provider":"LiquidAI","description":"LFM2.5-2.6B is a reasoning model from LiquidAI. 13 benchmarks count toward its index, in 6 categories.","logoUrl":null,"releaseDate":"2026-08-04","contextWindowTokens":128000,"modalities":["text"],"openWeights":true,"details":{"apiId":"liquid/lfm-2.5-2.6b","maxOutputTokens":8192,"knowledgeCutoff":null,"retiresAt":"2026-11-13","tools":true,"structuredOutput":true,"cacheReadPerMillion":0.04,"weightsUrl":"https://huggingface.co/LiquidAI/LFM2.5-2.6B"},"reasoning":true,"overallScore":29.30585285798668,"overallScoreRange":{"low":23.321101771505703,"high":35.290603944467655},"badges":[],"capabilityScores":{"agentic":32.040576831939944,"coding":23.90451358356273,"reasoning":24.02891223389094,"multimodal":null,"knowledge":29.497197665435866,"multilingual":null,"instruction":44.318622471526595,"math":27.153222852965097},"capabilityCounts":{"agentic":5,"coding":2,"instruction":1,"knowledge":2,"math":1,"reasoning":2},"benchmarks":[{"benchmarkId":"pinchBench","benchmarkName":"PinchBench","category":"agentic","score":68.22,"unit":"percent","family":"pinchBench","index":27.199866243422623,"aboveFit":false,"run":null,"source":"Kilo Code","sourceUrl":"https://pinchbench.com/about","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"clawEval","benchmarkName":"Claw-Eval","category":"agentic","score":62.85,"unit":"percent","family":"clawEval","index":56.00245573568179,"aboveFit":true,"run":null,"source":"Bowen Ye, Rang Li, Qibin Yang, Yuanxin Liu, Linli Yao, Hanglong Lv, Zhihui Xie, Chenxin An, Lei Li, Lingpeng Kong, Qi Liu, Zhifang Sui, Tong Yang","sourceUrl":"https://arxiv.org/abs/2604.06132","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"liveCodeBenchV6","benchmarkName":"LiveCodeBench v6","category":"coding","score":59.41,"unit":"percent","family":"liveCodeBenchV6","index":33.19607621624202,"aboveFit":false,"run":null,"source":"LiveCodeBench maintainers","sourceUrl":"https://github.com/LiveCodeBench/LiveCodeBench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"ifBench","benchmarkName":"Instruction Following Benchmark","category":"instruction","score":59.17,"unit":"percent","family":"ifbench-family","index":44.318622471526595,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"bfclV4","benchmarkName":"Berkeley Function Calling Leaderboard v4","category":"agentic","score":56.88,"unit":"percent","family":"bfclV4","index":37.966787263402544,"aboveFit":false,"run":null,"source":"Shishir G. Patil et al.","sourceUrl":"https://gorilla.cs.berkeley.edu/leaderboard.html","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":55.8,"unit":"percent","family":"gpqa-diamond","index":28.387335998373533,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aime2025","benchmarkName":"American Invitational Mathematics Examination 2025","category":"math","score":51.87,"unit":"percent","family":"aime2025","index":27.153222852965097,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaSciCode","benchmarkName":"Artificial Analysis SciCode","category":"coding","score":14.4,"unit":"percent","family":"scicode-family","index":14.612950950883437,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/scicode","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":6.2,"unit":"percent","family":"hle","index":30.6070593324982,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":5.7,"unit":"percent","family":"lcr","index":11.208708455008974,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"tau3Bench","benchmarkName":"τ³-Bench Tool-Agent-User Evaluation","category":"agentic","score":5.67,"unit":"percent","family":"tau3Bench","index":5,"aboveFit":false,"run":null,"source":"Sierra Research","sourceUrl":"https://github.com/sierra-research/tau2-bench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gdpvalAaNormalized","benchmarkName":"GDPval-AA normalized","category":"agentic","score":0,"unit":"percent","family":"gdpvalAaNormalized","index":34.03377491719279,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":0,"unit":"percent","family":"critpt","index":36.8491160127729,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":0.35,"outputPerMillion":0.5,"batch":null,"freeTier":true,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":188.5,"timeToFirstToken":0.177,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":85,"benchmarkCount":13,"categoryCount":6,"confidence":"medium","state":"available"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"grok-2","providerId":"xai","slug":"grok-2","name":"Grok 2","provider":"xAI","description":"Grok 2 is a standard model from xAI. 11 benchmarks count toward its index, in 4 categories.","logoUrl":null,"releaseDate":"2024-08-13","contextWindowTokens":null,"modalities":["text"],"openWeights":false,"details":null,"reasoning":false,"overallScore":29.094613289140717,"overallScoreRange":{"low":19.71917484731828,"high":38.470051730963156},"badges":[],"capabilityScores":{"agentic":null,"coding":25.184005639419986,"reasoning":null,"multimodal":33.538047371049366,"knowledge":34.044391088437855,"multilingual":null,"instruction":null,"math":25.02551409890768},"capabilityCounts":{"coding":2,"knowledge":3,"math":5,"multimodal":1},"benchmarks":[{"benchmarkId":"vals-ai:medqa:overall","benchmarkName":"MedQA","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":92.317,"unit":"percent","family":"vals-ai:medqa:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medqa","measuredAt":"2026-04-16","stderr":0.477,"costPerTest":0.003276,"latency":4.093},{"benchmarkId":"humaneval","benchmarkName":"Evaluating Large Language Models Trained on Code","category":"coding","score":88.4,"unit":"percent","family":"humaneval","index":28.93068377874425,"aboveFit":false,"run":null,"source":"Mark Chen, Jerry Tworek, Heewoo Jun, Qiming Yuan, Henrique Ponde de Oliveira Pinto, Jared Kaplan, Harri Edwards, Yuri Burda, Nicholas Joseph, Greg Brockman, Alex Ray, Raul Puri, Gretchen Krueger, Michael Petrov, Heidy Khlaaf, Girish Sastry, Pamela Mishkin, Brooke Chan, Scott Gray, Nick Ryder, Mikhail Pavlov, Alethea Power, Lukasz Kaiser, Mohammad Bavarian, Clemens Winter, Philippe Tillet, Felipe Petroski Such, Dave Cummings, Matthias Plappert, Fotios Chantzis, Elizabeth Barnes, Ariel Herbert-Voss, William Hebgen Guss, Alex Nichol, Alex Paino, Nikolas Tezak, Jie Tang, Igor Babuschkin, Suchir Balaji, Shantanu Jain, William Saunders, Christopher Hesse, Andrew N. Carr, Jan Leike, Josh Achiam, Vedant Misra, Evan Morikawa, Alec Radford, Matthew Knight, Miles Brundage, Mira Murati, Katie Mayer, Peter Welinder, Bob McGrew, Dario Amodei, Sam McCandlish, Ilya Sutskever, Wojciech Zaremba","sourceUrl":"https://arxiv.org/abs/2107.03374","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmlu","benchmarkName":"Massive Multitask Language Understanding","category":"knowledge","score":87.5,"unit":"percent","family":"mmlu","index":36.00661693616388,"aboveFit":false,"run":null,"source":"Dan Hendrycks, Collin Burns, Steven Basart, Andy Zou, Mantas Mazeika, Dawn Song, Jacob Steinhardt","sourceUrl":"https://arxiv.org/abs/2009.03300","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:mgsm:overall","benchmarkName":"MGSM","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":86.146,"unit":"percent","family":"vals-ai:mgsm:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mgsm","measuredAt":"2026-01-09","stderr":2.001,"costPerTest":0.003759,"latency":6.272},{"benchmarkId":"vals-ai:math500:overall","benchmarkName":"MATH 500","category":"math","domain":null,"eligible":true,"ranked":true,"score":78.4,"unit":"percent","family":"math-500","index":26.97662711087987,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/math500","measuredAt":"2026-01-09","stderr":3.6,"costPerTest":0.006277,"latency":20.44},{"benchmarkId":"mmluPro","benchmarkName":"Massive Multitask Language Understanding Professional","category":"knowledge","score":75.5,"unit":"percent","family":"mmlu-pro","index":39.60169130925173,"aboveFit":false,"run":null,"source":"Yubo Wang, Xueguang Ma, Ge Zhang, Yuansheng Ni, Abhranil Chandra, Shiguang Guo, Weiming Ren, Aaran Arulraj, Xuan He, Ziyan Jiang, Tianle Li, Max Ku, Kai Wang, Alex Zhuang, Rongqi Fan, Xiang Yue, Wenhu Chen","sourceUrl":"https://arxiv.org/abs/2406.01574","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:mmlu_pro:overall","benchmarkName":"MMLU Pro","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":75.473,"unit":"percent","family":"mmlu-pro","index":39.56767454106865,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","measuredAt":"2026-09-01","stderr":0.418,"costPerTest":0.005966,"latency":8.747},{"benchmarkId":"vals-ai:tax_eval_v2:overall","benchmarkName":"TaxEval v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":67.048,"unit":"percent","family":"vals-ai:tax_eval_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","measuredAt":"2026-09-01","stderr":0.918,"costPerTest":0.006894,"latency":10.621},{"benchmarkId":"mmmu","benchmarkName":"Massive Multi-discipline Multimodal Understanding","category":"multimodal","score":66.1,"unit":"percent","family":"mmmu","index":33.538047371049366,"aboveFit":false,"run":null,"source":"MMMU authors","sourceUrl":"https://arxiv.org/abs/2401.05508","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-math-level-5:overall","benchmarkName":"MATH level 5","category":"math","domain":null,"eligible":true,"ranked":true,"score":63.51963746223564,"unit":"percent","family":"epoch-ai:epoch-math-level-5:overall","index":29.85291660747705,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.323438169590836,"costPerTest":null,"latency":null},{"benchmarkId":"gpqa","benchmarkName":"Graduate-Level Google-Proof Q&A","category":"knowledge","score":56,"unit":"percent","family":"gpqa-diamond","index":28.570770533357493,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":53.78787878787878,"unit":"percent","family":"gpqa-diamond","index":26.541873403989477,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.7066342316607197,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:corp_fin_v2:overall","benchmarkName":"CorpFin v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":51.126,"unit":"percent","family":"vals-ai:corp_fin_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","measuredAt":"2026-08-12","stderr":0.981,"costPerTest":0.115097,"latency":87.134},{"benchmarkId":"vals-ai:gpqa:overall","benchmarkName":"GPQA Diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":50.758,"unit":"percent","family":"gpqa-diamond","index":23.76295137142798,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","measuredAt":"2026-09-01","stderr":2.51,"costPerTest":0.0095,"latency":20.178},{"benchmarkId":"vals-ai:lcb:overall","benchmarkName":"LiveCodeBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":38.679,"unit":"percent","family":"livecodebench","index":21.43732750009572,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","measuredAt":"2026-09-01","stderr":1.048,"costPerTest":0.003004,"latency":2.378},{"benchmarkId":"vals-ai:aime:overall","benchmarkName":"AIME","category":"math","domain":null,"eligible":true,"ranked":true,"score":15.208,"unit":"percent","family":"vals-ai:aime:overall","index":22.149157914887443,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/aime","measuredAt":"2026-04-16","stderr":0.784,"costPerTest":0.015018,"latency":57.884},{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":11.527777777777779,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":15.625883348210031,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":3.382424700930314,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-2025-02-28-private:overall","benchmarkName":"FrontierMath-2025-02-28-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":0.6896551724137931,"unit":"percent","family":"epoch-ai:epoch-frontiermath-2025-02-28-private:overall","index":30.52298551308402,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0.4868154158215009,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-2025-02-28-public:overall","benchmarkName":"FrontierMath-2025-02-28-Public","category":"math","domain":null,"eligible":false,"ranked":true,"score":0,"unit":"percent","family":"epoch-ai:epoch-frontiermath-2025-02-28-public:overall","index":null,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":{"elo":1304.5824553888287,"eloUpper":1308.242537387191,"eloLower":1300.9223733904669,"votes":63498,"rank":250,"rankUpper":null,"rankLower":null,"arenaName":"Grok 2-2024-08-13","organization":"xai","updatedAt":"2026-11-09"},"quality":{"coveragePercent":55.00000000000001,"benchmarkCount":11,"categoryCount":4,"confidence":"low","state":"available"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gpt-4o","providerId":"openai","slug":"gpt-4o","name":"GPT-4o","provider":"OpenAI","description":"GPT-4o is a standard model from OpenAI. 24 benchmarks count toward its index, in 8 categories.","logoUrl":null,"releaseDate":"2024-05-13","contextWindowTokens":128000,"modalities":["text","image","file"],"openWeights":false,"details":{"apiId":"openai/gpt-4o","maxOutputTokens":16384,"knowledgeCutoff":"2023-10-31","retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":1.25,"weightsUrl":null},"reasoning":false,"overallScore":28.708382212285134,"overallScoreRange":{"low":25.4076947838848,"high":32.00906964068547},"badges":[],"capabilityScores":{"agentic":26.739450085881984,"coding":22.753930301146564,"reasoning":37.40628691812492,"multimodal":36.04990283990811,"knowledge":34.11726928597404,"multilingual":30.804713172528146,"instruction":21.077343480716845,"math":23.44456202375789},"capabilityCounts":{"math":6,"knowledge":5,"multimodal":1,"multilingual":1,"reasoning":5,"coding":3,"instruction":1,"agentic":2},"benchmarks":[{"benchmarkId":"vals-ai:mgsm:overall","benchmarkName":"MGSM","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":90.364,"unit":"percent","family":"vals-ai:mgsm:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mgsm","measuredAt":"2026-01-09","stderr":1.826,"costPerTest":0.004323,"latency":3.982},{"benchmarkId":"vals-ai:medqa:overall","benchmarkName":"MedQA","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":88.161,"unit":"percent","family":"vals-ai:medqa:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medqa","measuredAt":"2026-04-16","stderr":0.578,"costPerTest":0.002787,"latency":3.392},{"benchmarkId":"vals-ai:legal_bench:overall","benchmarkName":"LegalBench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":82.214,"unit":"percent","family":"vals-ai:legal_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","measuredAt":"2026-10-13","stderr":0.463,"costPerTest":null,"latency":0.452},{"benchmarkId":"vals-ai:tax_eval_v2:overall","benchmarkName":"TaxEval v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":74.53,"unit":"percent","family":"vals-ai:tax_eval_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","measuredAt":"2026-09-01","stderr":0.859,"costPerTest":0.005806,"latency":5.855},{"benchmarkId":"vals-ai:math500:overall","benchmarkName":"MATH 500","category":"math","domain":null,"eligible":true,"ranked":true,"score":74,"unit":"percent","family":"math-500","index":22.32689097618839,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/math500","measuredAt":"2026-01-09","stderr":3.834,"costPerTest":0.008846,"latency":12.804},{"benchmarkId":"vals-ai:mmlu_pro:overall","benchmarkName":"MMLU Pro","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":72.563,"unit":"percent","family":"mmlu-pro","index":35.90142285911496,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","measuredAt":"2026-09-01","stderr":0.441,"costPerTest":0.007465,"latency":9.143},{"benchmarkId":"vals-ai:mmmu:overall","benchmarkName":"MMMU Pro","category":"multimodal","domain":null,"eligible":true,"ranked":true,"score":62.161,"unit":"percent","family":"mmmu-pro","index":36.04990283990811,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmmu","measuredAt":"2026-09-01","stderr":1.166,"costPerTest":0.006822,"latency":45.873},{"benchmarkId":"mmluProX","benchmarkName":"MMLU-ProX","category":"multilingual","score":61.1,"unit":"percent","family":"mmluProX","index":30.804713172528146,"aboveFit":false,"run":null,"source":"MMLU-ProX authors","sourceUrl":"https://arxiv.org/abs/2503.10497","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:case_law_v2:overall","benchmarkName":"CaseLaw v2","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":59.7,"unit":"percent","family":"vals-ai:case_law_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/case_law_v2","measuredAt":"2026-05-04","stderr":0.429,"costPerTest":0.066774,"latency":40.867},{"benchmarkId":"vals-ai:mortgage_tax:overall","benchmarkName":"MortgageTax","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":57.432,"unit":"percent","family":"vals-ai:mortgage_tax:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mortgage_tax","measuredAt":"2026-09-01","stderr":1.907,"costPerTest":0.007141,"latency":5.984},{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":54.3,"unit":"percent","family":"gpqa-diamond","index":27.011576985993855,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:gpqa:overall","benchmarkName":"GPQA Diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":53.788,"unit":"percent","family":"gpqa-diamond","index":26.541984576434924,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","measuredAt":"2026-09-01","stderr":2.507,"costPerTest":0.009465,"latency":14.924},{"benchmarkId":"epoch-ai:epoch-math-level-5:overall","benchmarkName":"MATH level 5","category":"math","domain":null,"eligible":true,"ranked":true,"score":51.04796072507553,"unit":"percent","family":"epoch-ai:epoch-math-level-5:overall","index":23.794732557276635,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.120841984218505,"costPerTest":null,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":49.3,"unit":"percent","family":"lcr","index":41.12181937054375,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":48.89520202020202,"unit":"percent","family":"gpqa-diamond","index":22.054443965461495,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.6254109873371996,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:corp_fin_v2:overall","benchmarkName":"CorpFin v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":45.921,"unit":"percent","family":"vals-ai:corp_fin_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","measuredAt":"2026-08-12","stderr":0.973,"costPerTest":null,"latency":6.461},{"benchmarkId":"vals-ai:lcb:overall","benchmarkName":"LiveCodeBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":43.444,"unit":"percent","family":"livecodebench","index":25.391044456293773,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","measuredAt":"2026-09-01","stderr":1.123,"costPerTest":0.003999,"latency":4.347},{"benchmarkId":"aaIfBench","benchmarkName":"Artificial Analysis IFBench","category":"instruction","score":34.3,"unit":"percent","family":"ifbench-family","index":21.077343480716845,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-swe-bench-verified:overall","benchmarkName":"SWE-Bench verified","category":"coding","domain":null,"eligible":true,"ranked":true,"score":30.991735537190085,"unit":"percent","family":"swe-bench-verified","index":24.086116923594332,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.1042625979133334,"costPerTest":null,"latency":null},{"benchmarkId":"swebench-experiments:swe-bench-lite:overall","benchmarkName":"SWE-bench Lite","category":"coding","domain":null,"eligible":true,"ranked":true,"score":28.000000000000004,"unit":"percent","family":"swe-bench-lite","index":27.105343106957452,"aboveFit":false,"run":"reproducedRG","source":"SWE-bench team","sourceUrl":"https://github.com/SWE-bench/experiments/tree/main/evaluation/lite","measuredAt":"2025-09-11","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-simpleqa-verified:overall","benchmarkName":"SimpleQA Verified","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":26,"unit":"percent","family":"epoch-ai:epoch-simpleqa-verified:overall","index":45.579347543525984,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.387777332977422,"costPerTest":null,"latency":null},{"benchmarkId":"tau2Bench","benchmarkName":"τ²-Bench Tool-Agent-User Evaluation","category":"agentic","score":25.1,"unit":"percent","family":"tau2-bench","index":19.44512525457118,"aboveFit":false,"run":null,"source":"Victor Barres, Honghua Dong, Soham Ray, Xujie Si, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2506.07982","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"swe-bench:swe-bench-verified:overall","benchmarkName":"SWE-bench Verified","category":"coding","domain":null,"eligible":true,"ranked":true,"score":23.2,"unit":"percent","family":"swe-bench-verified","index":18.339263334053307,"aboveFit":false,"run":"SWE-agent","source":"SWE-bench team","sourceUrl":"https://www.swebench.com/","measuredAt":"2026-02-26","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"swe-bench:swe-bench-lite:overall","benchmarkName":"SWE-bench Lite","category":"coding","domain":null,"eligible":true,"ranked":true,"score":21.67,"unit":"percent","family":"swe-bench-lite","index":21.957623119227765,"aboveFit":false,"run":"AppMap Navie","source":"SWE-bench team","sourceUrl":"https://www.swebench.com/","measuredAt":"2025-09-11","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"swebench-experiments:swe-bench-verified:overall","benchmarkName":"SWE-bench Verified","category":"coding","domain":null,"eligible":true,"ranked":true,"score":21.62,"unit":"percent","family":"swe-bench-verified","index":17.17392236115507,"aboveFit":false,"run":"mini-SWE-agent","source":"SWE-bench team","sourceUrl":"https://github.com/SWE-bench/experiments/tree/main/evaluation/verified","measuredAt":"2026-09-01","stderr":null,"costPerTest":1.531148125,"latency":null},{"benchmarkId":"omniscienceAccuracy","benchmarkName":"Artificial Analysis Omniscience Accuracy","category":"knowledge","score":19.9,"unit":"percent","family":"omniscienceAccuracy","index":35.76904454940935,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"swebench-experiments:swe-bench-test:overall","benchmarkName":"SWE-bench (full test split)","category":"coding","domain":null,"eligible":true,"ranked":true,"score":14.603312990409764,"unit":"percent","family":"swebench-experiments:swe-bench-test:overall","index":null,"aboveFit":false,"run":"AppMap Navie","source":"SWE-bench team","sourceUrl":"https://github.com/SWE-bench/experiments/tree/main/evaluation/test","measuredAt":"2025-12-19","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:aime:overall","benchmarkName":"AIME","category":"math","domain":null,"eligible":true,"ranked":true,"score":11.875,"unit":"percent","family":"vals-ai:aime:overall","index":20.692484395358463,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/aime","measuredAt":"2026-04-16","stderr":0.955,"costPerTest":0.011954,"latency":15.776},{"benchmarkId":"swe-bench:swe-bench-multimodal:overall","benchmarkName":"SWE-bench Multimodal","category":"multimodal","domain":null,"eligible":true,"ranked":true,"score":10.635,"unit":"percent","family":"swe-bench-multimodal","index":null,"aboveFit":false,"run":"SWE-agent","source":"SWE-bench team","sourceUrl":"https://www.swebench.com/","measuredAt":"2025-11-17","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-chess-puzzles:overall","benchmarkName":"Chess Puzzles","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":9,"unit":"percent","family":"epoch-ai:epoch-chess-puzzles:overall","index":40.81581248148979,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.8762349126466145,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:finance_agent:overall","benchmarkName":"Finance Agent v1.1","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":8.064,"unit":"percent","family":"vals-ai:finance_agent:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/finance_agent","measuredAt":"2026-06-04","stderr":1.496,"costPerTest":0.158124,"latency":31.141},{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":6.25,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":12.555303662396526,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.3249986423369555,"costPerTest":null,"latency":null},{"benchmarkId":"swebench-experiments:swe-bench-multimodal:overall","benchmarkName":"SWE-bench Multimodal","category":"multimodal","domain":null,"eligible":true,"ranked":true,"score":5.996131528046422,"unit":"percent","family":"swe-bench-multimodal","index":null,"aboveFit":false,"run":"RAG","source":"SWE-bench team","sourceUrl":"https://github.com/SWE-bench/experiments/tree/main/evaluation/multimodal","measuredAt":"2025-11-17","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"arc-prize:v1-semi-private:overall","benchmarkName":"ARC-AGI-1 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":4.5,"unit":"percent","family":"arc-prize:v1-semi-private:overall","index":29.056171323575597,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.05,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":2.4,"unit":"percent","family":"hle","index":26.79454690138498,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tiers-1-3-v2-private:overall","benchmarkName":"FrontierMath-Tiers-1-3-v2-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":0.3508771929824561,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tiers-1-3-v2-private:overall","index":31.07938111630475,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0.3508771929824561,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-2025-02-28-private:overall","benchmarkName":"FrontierMath-2025-02-28-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":0.3448275862068966,"unit":"percent","family":"epoch-ai:epoch-frontiermath-2025-02-28-private:overall","index":30.21857943502259,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0.3448275862068965,"costPerTest":null,"latency":null},{"benchmarkId":"gdpvalAaNormalized","benchmarkName":"GDPval-AA normalized","category":"agentic","score":0,"unit":"percent","family":"gdpvalAaNormalized","index":34.03377491719279,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":0,"unit":"percent","family":"critpt","index":36.8491160127729,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"arc-prize:v2-public-eval:overall","benchmarkName":"ARC-AGI-2 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":0,"unit":"percent","family":"arc-prize:v2-public-eval:overall","index":null,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.08,"latency":null},{"benchmarkId":"arc-prize:v2-semi-private:overall","benchmarkName":"ARC-AGI-2 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":0,"unit":"percent","family":"arc-prize:v2-semi-private:overall","index":39.188515402242544,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.08,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-2025-02-28-public:overall","benchmarkName":"FrontierMath-2025-02-28-Public","category":"math","domain":null,"eligible":false,"ranked":true,"score":0,"unit":"percent","family":"epoch-ai:epoch-frontiermath-2025-02-28-public:overall","index":null,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":2.5,"outputPerMillion":10,"batch":{"inputPerMillion":1.25,"outputPerMillion":5},"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":66,"timeToFirstToken":0.623,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":{"elo":1300.443096004425,"eloUpper":1303.8781225383739,"eloLower":1297.0080694704761,"votes":112881,"rank":252,"rankUpper":null,"rankLower":null,"arenaName":"GPT-4o-2024-05-13","organization":"openai","updatedAt":"2026-11-09"},"quality":{"coveragePercent":100,"benchmarkCount":24,"categoryCount":8,"confidence":"high","state":"available"},"hosts":[{"host":"digitalocean","name":"DigitalOcean","kind":"cloud","url":"https://docs.digitalocean.com/products/gradient-ai-platform/details/models/","inputPerMillion":2.5,"outputPerMillion":10,"cacheReadPerMillion":1.25,"contextTokens":128000,"maxOutputTokens":16384,"deprecated":false},{"host":"azure","name":"Microsoft Azure","kind":"cloud","url":"https://learn.microsoft.com/en-us/azure/ai-services/openai/concepts/models","inputPerMillion":2.5,"outputPerMillion":10,"cacheReadPerMillion":1.25,"contextTokens":128000,"maxOutputTokens":16384,"deprecated":true},{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":2.5,"outputPerMillion":10,"cacheReadPerMillion":1.25,"contextTokens":128000,"maxOutputTokens":16384,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:swe-bench-team","name":"SWE-bench team","url":"https://github.com","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:arc-prize-foundation","name":"ARC Prize Foundation","url":"https://arcprize.org","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"llama-3-3-70b","providerId":"meta","slug":"llama-3-3-70b","name":"Llama 3.3 70B","provider":"Meta","description":"Llama 3.3 70B is a standard model from Meta. 14 benchmarks count toward its index, in 7 categories.","logoUrl":null,"releaseDate":"2024-12-06","contextWindowTokens":128000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":28.22030847052058,"overallScoreRange":{"low":22.898694188796675,"high":33.54192275224448},"badges":[],"capabilityScores":{"agentic":27.23712991982319,"coding":28.93068377874425,"reasoning":27.459314861819237,"multimodal":null,"knowledge":29.09622843997971,"multilingual":34.08589986199651,"instruction":40.85152266728407,"math":15.556476623908843},"capabilityCounts":{"instruction":2,"multilingual":1,"coding":1,"knowledge":4,"math":2,"agentic":2,"reasoning":2},"benchmarks":[{"benchmarkId":"ifeval","benchmarkName":"Instruction-Following Eval","category":"instruction","score":92.1,"unit":"percent","family":"ifeval","index":48.66396598403366,"aboveFit":true,"run":null,"source":"Jeffrey Zhou, Tianjian Lu, Swaroop Mishra, Siddhartha Brahma, Sujoy Basu, Yi Luan, Denny Zhou, Le Hou","sourceUrl":"https://arxiv.org/abs/2311.07911","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mgsm","benchmarkName":"Multilingual Grade School Math","category":"multilingual","score":91.1,"unit":"percent","family":"mgsm","index":34.08589986199651,"aboveFit":false,"run":null,"source":"Freda Shi, Mirac Suzgun, Markus Freitag, Xuezhi Wang, Suraj Srivats, Soroush Vosoughi, Hyung Won Chung, Yi Tay, Sebastian Ruder, Denny Zhou, Dipanjan Das, Jason Wei","sourceUrl":"https://arxiv.org/abs/2210.03057","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"humaneval","benchmarkName":"Evaluating Large Language Models Trained on Code","category":"coding","score":88.4,"unit":"percent","family":"humaneval","index":28.93068377874425,"aboveFit":false,"run":null,"source":"Mark Chen, Jerry Tworek, Heewoo Jun, Qiming Yuan, Henrique Ponde de Oliveira Pinto, Jared Kaplan, Harri Edwards, Yuri Burda, Nicholas Joseph, Greg Brockman, Alex Ray, Raul Puri, Gretchen Krueger, Michael Petrov, Heidy Khlaaf, Girish Sastry, Pamela Mishkin, Brooke Chan, Scott Gray, Nick Ryder, Mikhail Pavlov, Alethea Power, Lukasz Kaiser, Mohammad Bavarian, Clemens Winter, Philippe Tillet, Felipe Petroski Such, Dave Cummings, Matthias Plappert, Fotios Chantzis, Elizabeth Barnes, Ariel Herbert-Voss, William Hebgen Guss, Alex Nichol, Alex Paino, Nikolas Tezak, Jie Tang, Igor Babuschkin, Suchir Balaji, Shantanu Jain, William Saunders, Christopher Hesse, Andrew N. Carr, Jan Leike, Josh Achiam, Vedant Misra, Evan Morikawa, Alec Radford, Matthew Knight, Miles Brundage, Mira Murati, Katie Mayer, Peter Welinder, Bob McGrew, Dario Amodei, Sam McCandlish, Ilya Sutskever, Wojciech Zaremba","sourceUrl":"https://arxiv.org/abs/2107.03374","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmlu","benchmarkName":"Massive Multitask Language Understanding","category":"knowledge","score":86,"unit":"percent","family":"mmlu","index":34.2156343066989,"aboveFit":false,"run":null,"source":"Dan Hendrycks, Collin Burns, Steven Basart, Andy Zou, Mantas Mazeika, Dawn Song, Jacob Steinhardt","sourceUrl":"https://arxiv.org/abs/2009.03300","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmluPro","benchmarkName":"Massive Multitask Language Understanding Professional","category":"knowledge","score":68.9,"unit":"percent","family":"mmlu-pro","index":31.286481308944374,"aboveFit":false,"run":null,"source":"Yubo Wang, Xueguang Ma, Ge Zhang, Yuansheng Ni, Abhranil Chandra, Shiguang Guo, Weiming Ren, Aaran Arulraj, Xuan He, Ziyan Jiang, Tianle Li, Max Ku, Kai Wang, Alex Zhuang, Rongqi Fan, Xiang Yue, Wenhu Chen","sourceUrl":"https://arxiv.org/abs/2406.01574","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gpqa","benchmarkName":"Graduate-Level Google-Proof Q&A","category":"knowledge","score":50.5,"unit":"percent","family":"gpqa-diamond","index":23.526320821298675,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":49.8,"unit":"percent","family":"gpqa-diamond","index":22.884299948854824,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":47.44318181818182,"unit":"percent","family":"gpqa-diamond","index":20.72269071273706,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.8523027562061025,"costPerTest":null,"latency":null},{"benchmarkId":"aaIfBench","benchmarkName":"Artificial Analysis IFBench","category":"instruction","score":47.1,"unit":"percent","family":"ifbench-family","index":33.03907935053449,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-math-level-5:overall","benchmarkName":"MATH level 5","category":"math","domain":null,"eligible":true,"ranked":true,"score":41.59743202416919,"unit":"percent","family":"epoch-ai:epoch-math-level-5:overall","index":19.20408741401349,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.1816670264886684,"costPerTest":null,"latency":null},{"benchmarkId":"tau2Bench","benchmarkName":"τ²-Bench Tool-Agent-User Evaluation","category":"agentic","score":26.6,"unit":"percent","family":"tau2-bench","index":20.44048492245359,"aboveFit":false,"run":null,"source":"Victor Barres, Honghua Dong, Soham Ray, Xujie Si, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2506.07982","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":15.7,"unit":"percent","family":"lcr","index":18.06951371086557,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":5.138888888888888,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":11.908865833804207,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.3611111111111107,"costPerTest":null,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":3.6,"unit":"percent","family":"hle","index":27.998498195420737,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gdpvalAaNormalized","benchmarkName":"GDPval-AA normalized","category":"agentic","score":0,"unit":"percent","family":"gdpvalAaNormalized","index":34.03377491719279,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":0,"unit":"percent","family":"critpt","index":36.8491160127729,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":90,"benchmarkCount":14,"categoryCount":7,"confidence":"medium","state":"available"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"devstral-small","providerId":"mistral","slug":"devstral-small","name":"Devstral Small","provider":"Mistral","description":"Devstral Small is a standard model from Mistral. 8 benchmarks count toward its index, in 5 categories.","logoUrl":null,"releaseDate":"2025-05-21","contextWindowTokens":128000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":28.196174899463912,"overallScoreRange":{"low":19.979216573570675,"high":36.41313322535715},"badges":[],"capabilityScores":{"agentic":21.63491652391248,"coding":35.74562216974843,"reasoning":28.48843565019773,"multimodal":null,"knowledge":24.19830655208503,"multilingual":null,"instruction":21.3576966651657,"math":null},"capabilityCounts":{"coding":1,"knowledge":3,"instruction":1,"agentic":1,"reasoning":2},"benchmarks":[{"benchmarkId":"sweVerified","benchmarkName":"Software Engineering Benchmark Verified","category":"coding","score":46.8,"unit":"percent","family":"swe-bench-verified","index":35.74562216974843,"aboveFit":false,"run":null,"source":"Carlos E. Jimenez, John Yang, Alexander Wettig, Shunyu Yao, Kexin Pei, Ofir Press, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2310.06770","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":41.4,"unit":"percent","family":"gpqa-diamond","index":15.180049479528634,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaIfBench","benchmarkName":"Artificial Analysis IFBench","category":"instruction","score":34.6,"unit":"percent","family":"ifbench-family","index":21.3576966651657,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"tau2Bench","benchmarkName":"τ²-Bench Tool-Agent-User Evaluation","category":"agentic","score":28.4,"unit":"percent","family":"tau2-bench","index":21.63491652391248,"aboveFit":false,"run":null,"source":"Victor Barres, Honghua Dong, Soham Ray, Xujie Si, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2506.07982","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":18.7,"unit":"percent","family":"lcr","index":20.127755287622556,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"omniscienceAccuracy","benchmarkName":"Artificial Analysis Omniscience Accuracy","category":"knowledge","score":14.7,"unit":"percent","family":"omniscienceAccuracy","index":29.215713432299765,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":3.8,"unit":"percent","family":"hle","index":28.199156744426695,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":0,"unit":"percent","family":"critpt","index":36.8491160127729,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":75,"benchmarkCount":8,"categoryCount":5,"confidence":"low","state":"available"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gemma-4-e4b","providerId":"google","slug":"gemma-4-e4b","name":"Gemma 4 E4B","provider":"Google","description":"Gemma 4 E4B is a reasoning model from Google in the Gemma 4 family. 15 benchmarks count toward its index, in 7 categories.","logoUrl":null,"releaseDate":"2026-04-02","contextWindowTokens":128000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":true,"overallScore":28.19466703431994,"overallScoreRange":{"low":23.019590987108344,"high":33.36974308153154},"badges":[],"capabilityScores":{"agentic":25.31276789525053,"coding":27.380952585802287,"reasoning":30.88709218529674,"multimodal":32.756934190916965,"knowledge":30.20413711853285,"multilingual":8.830975424520311,"instruction":30.328998567528934,"math":null},"capabilityCounts":{"multimodal":3,"knowledge":3,"coding":2,"instruction":1,"multilingual":1,"reasoning":3,"agentic":2},"benchmarks":[{"benchmarkId":"extractBenchLlamaIndex","benchmarkName":"ExtractBench: schema-guided enterprise document extraction (LlamaIndex leaderboard)","category":"multimodal","score":69.64,"unit":"percent","family":"extractBenchLlamaIndex","index":44.34149251848816,"aboveFit":false,"run":null,"source":"Boyang Zhang, Adrian Lyjak, Eli Stewart, Zhaoqi Li and Simon Suo (LlamaIndex)","sourceUrl":"https://arxiv.org/abs/2607.29677","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmluPro","benchmarkName":"Massive Multitask Language Understanding Professional","category":"knowledge","score":69.4,"unit":"percent","family":"mmlu-pro","index":31.916421460482812,"aboveFit":false,"run":null,"source":"Yubo Wang, Xueguang Ma, Ge Zhang, Yuansheng Ni, Abhranil Chandra, Shiguang Guo, Weiming Ren, Aaran Arulraj, Xuan He, Ziyan Jiang, Tianle Li, Max Ku, Kai Wang, Alex Zhuang, Rongqi Fan, Xiang Yue, Wenhu Chen","sourceUrl":"https://arxiv.org/abs/2406.01574","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gpqa","benchmarkName":"Graduate-Level Google-Proof Q&A","category":"knowledge","score":58.6,"unit":"percent","family":"gpqa-diamond","index":30.955419488148934,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":57.6,"unit":"percent","family":"gpqa-diamond","index":30.038246813229147,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmmuPro","benchmarkName":"Massive Multi-discipline Multimodal Understanding Pro","category":"multimodal","score":52.6,"unit":"percent","family":"mmmu-pro","index":23.60710017310377,"aboveFit":false,"run":null,"source":"MMMU-Pro authors","sourceUrl":"https://arxiv.org/abs/2409.02813","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"liveCodeBenchV6","benchmarkName":"LiveCodeBench v6","category":"coding","score":52,"unit":"percent","family":"liveCodeBenchV6","index":27.112042756573814,"aboveFit":false,"run":null,"source":"LiveCodeBench maintainers","sourceUrl":"https://github.com/LiveCodeBench/LiveCodeBench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaMmmuPro","benchmarkName":"Artificial Analysis MMMU-Pro","category":"multimodal","score":51.4,"unit":"percent","family":"mmmu-pro","index":22.045405454960772,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmmu-pro","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaIfBench","benchmarkName":"Artificial Analysis IFBench","category":"instruction","score":44.2,"unit":"percent","family":"ifbench-family","index":30.328998567528934,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"parseBenchLlamaIndex","benchmarkName":"ParseBench: document parsing for AI agents (LlamaIndex leaderboard)","category":"multimodal","score":41.13,"unit":"percent","family":"parseBenchLlamaIndex","index":31.103057240230463,"aboveFit":false,"run":null,"source":"Boyang Zhang, Sebastián G. Acosta, Preston Carlson, Sacha Bron, Pierre-Loïc Doulcet, Daniel B. Ospina and Simon Suo (LlamaIndex)","sourceUrl":"https://arxiv.org/abs/2604.08538","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-dutch:overall","benchmarkName":"EuroEval Dutch","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":33.205555555555556,"unit":"percent","family":"euroeval","index":17.272533966736596,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/dutch","measuredAt":null,"stderr":0.8209005894612196,"costPerTest":null,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":32,"unit":"percent","family":"lcr","index":29.25262627791183,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-french:overall","benchmarkName":"EuroEval French","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":30.087499999999995,"unit":"percent","family":"euroeval","index":13.04205002199128,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/french","measuredAt":null,"stderr":0.8940209379539161,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":29.173749999999995,"unit":"percent","family":"euroeval","index":11.802301519876387,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":1.2348203310603532,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-portuguese:overall","benchmarkName":"EuroEval Portuguese","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":27.612499999999997,"unit":"percent","family":"euroeval","index":9.684044503266946,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/portuguese","measuredAt":null,"stderr":0.8663311650287089,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-italian:overall","benchmarkName":"EuroEval Italian","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":26.354999999999997,"unit":"percent","family":"euroeval","index":7.977906345773675,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/italian","measuredAt":null,"stderr":0.9062086197449238,"costPerTest":null,"latency":null},{"benchmarkId":"mrcrv2","benchmarkName":"MRCRv2","category":"reasoning","score":25.4,"unit":"percent","family":"mrcrv2","index":25.39800427518615,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":25.187142857142856,"unit":"percent","family":"euroeval","index":6.393392919189473,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":1.029032634694015,"costPerTest":null,"latency":null},{"benchmarkId":"aaSciCode","benchmarkName":"Artificial Analysis SciCode","category":"coding","score":24.4,"unit":"percent","family":"scicode-family","index":27.649862415030764,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/scicode","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-spanish:overall","benchmarkName":"EuroEval Spanish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":23.565,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/spanish","measuredAt":null,"stderr":0.8050834584066424,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":21.73125,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.9178677737016373,"costPerTest":null,"latency":null},{"benchmarkId":"tau2Bench","benchmarkName":"τ²-Bench Tool-Agent-User Evaluation","category":"agentic","score":20.8,"unit":"percent","family":"tau2-bench","index":16.591760873308274,"aboveFit":false,"run":null,"source":"Victor Barres, Honghua Dong, Soham Ray, Xujie Si, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2506.07982","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":3.8,"unit":"percent","family":"hle","index":28.199156744426695,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":0.6,"unit":"percent","family":"critpt","index":38.010646002792235,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gdpvalAaNormalized","benchmarkName":"GDPval-AA normalized","category":"agentic","score":0,"unit":"percent","family":"gdpvalAaNormalized","index":34.03377491719279,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":90,"benchmarkCount":15,"categoryCount":7,"confidence":"medium","state":"available"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"nemotron35lightning30ba3b","providerId":"nvidia","slug":"nvidia-nemotron-3-5-lightning-30b-a3b","name":"Nemotron 3.5 Lightning 30B A3B","provider":"NVIDIA","description":"Nemotron 3.5 Lightning 30B A3B is a model from NVIDIA. 8 benchmarks count toward its index, in 6 categories.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":28.133456990094825,"overallScoreRange":{"low":20.916498664201587,"high":35.35041531598806},"badges":[],"capabilityScores":{"agentic":32.9937796721549,"coding":21.86156812995326,"reasoning":17.29590917371972,"multimodal":null,"knowledge":17.85631032678851,"multilingual":null,"instruction":37.757606029797394,"math":32.52579794523776},"capabilityCounts":{"math":1,"instruction":1,"coding":2,"agentic":2,"reasoning":1,"knowledge":1},"benchmarks":[{"benchmarkId":"livebench:livebench-math:overall","benchmarkName":"LiveBench Mathematics","category":"math","domain":null,"eligible":true,"ranked":true,"score":66.54925,"unit":"percent","family":"livebench:livebench-math:overall","index":32.52579794523776,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Mathematics","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.010476781002638522,"latency":null},{"benchmarkId":"livebench:livebench-instruction-following:overall","benchmarkName":"LiveBench Instruction Following","category":"instruction","domain":null,"eligible":true,"ranked":true,"score":54.94175,"unit":"percent","family":"livebench:livebench-instruction-following:overall","index":37.757606029797394,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=IF","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.011914500000000001,"latency":null},{"benchmarkId":"livebench:livebench-coding:overall","benchmarkName":"LiveBench Coding","category":"coding","domain":null,"eligible":true,"ranked":true,"score":45.759499999999996,"unit":"percent","family":"livebench:livebench-coding:overall","index":19.18147066410254,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Coding","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.0035495726495726495,"latency":null},{"benchmarkId":"livebench:livebench-design:overall","benchmarkName":"LiveBench Design","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":43.9,"unit":"percent","family":"livebench-2-agentic","index":50.69490664937826,"aboveFit":true,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Design","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.03026,"latency":null},{"benchmarkId":"livebench:livebench-data-analysis:overall","benchmarkName":"LiveBench Data Analysis","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":43.08733333333333,"unit":"percent","family":"livebench","index":17.436517738718123,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Data%20Analysis","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.013829333333333332,"latency":null},{"benchmarkId":"livebench:livebench-reasoning:overall","benchmarkName":"LiveBench Reasoning","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":42.87025,"unit":"percent","family":"livebench","index":17.155300608721312,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Reasoning","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.02245841584158416,"latency":null},{"benchmarkId":"livebench:livebench-research:overall","benchmarkName":"LiveBench Research","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":40,"unit":"percent","family":"livebench-2-agentic","index":47.842715379188476,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Research","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.08682727272727273,"latency":null},{"benchmarkId":"livebench:livebench-language:overall","benchmarkName":"LiveBench Language","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":38.289,"unit":"percent","family":"livebench:livebench-language:overall","index":17.85631032678851,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Language","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.014108666666666669,"latency":null},{"benchmarkId":"livebench:livebench-coding-2:overall","benchmarkName":"LiveBench Coding 2.0","category":"coding","domain":null,"eligible":true,"ranked":true,"score":22.6,"unit":"percent","family":"livebench:livebench-coding-2:overall","index":24.541665595803977,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Coding","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.08473103448275862,"latency":null},{"benchmarkId":"livebench:livebench-agentic-coding:overall","benchmarkName":"LiveBench Agentic Coding","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":19.949666666666666,"unit":"percent","family":"livebench:livebench-agentic-coding:overall","index":29.73644181961057,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Agentic%20Coding","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.09796805555555557,"latency":null},{"benchmarkId":"livebench:livebench-long-horizon:overall","benchmarkName":"LiveBench Long Agentic","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":8.3,"unit":"percent","family":"livebench-2-agentic","index":24.659519670209992,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Long%20Agentic","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.5500818181818182,"latency":null},{"benchmarkId":"livebench:livebench-browser:overall","benchmarkName":"LiveBench Browser Use","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":0,"unit":"percent","family":"livebench-2-agentic","index":18.589471582370194,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Browser","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.11960000000000001,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":85,"benchmarkCount":8,"categoryCount":6,"confidence":"low","state":"available"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:livebench","name":"LiveBench","url":"https://livebench.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"mistral-large-2","providerId":"mistral","slug":"mistral-large-2","name":"Mistral Large 2","provider":"Mistral","description":"Mistral Large 2 is a standard model from Mistral. 9 benchmarks count toward its index, in 5 categories.","logoUrl":null,"releaseDate":"2024-07-24","contextWindowTokens":131072,"modalities":["text","file"],"openWeights":true,"details":{"apiId":"mistralai/mistral-large-2407","maxOutputTokens":104857,"knowledgeCutoff":"2024-03-31","retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.19999999999999998,"weightsUrl":"https://huggingface.co/mistralai/Mistral-Large-Instruct-2407"},"reasoning":false,"overallScore":27.436428415181556,"overallScoreRange":{"low":18.546428415181555,"high":36.32642841518155},"badges":[],"capabilityScores":{"agentic":23.161134681332175,"coding":null,"reasoning":36.8491160127729,"multimodal":null,"knowledge":28.65729473876762,"multilingual":null,"instruction":18.18036057474539,"math":21.986781533252326},"capabilityCounts":{"knowledge":3,"math":3,"instruction":1,"agentic":1,"reasoning":1},"benchmarks":[{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":50.173611111111114,"unit":"percent","family":"gpqa-diamond","index":23.226965851012356,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.6887818295509085,"costPerTest":null,"latency":null},{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":48.6,"unit":"percent","family":"gpqa-diamond","index":21.783692738951082,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-math-level-5:overall","benchmarkName":"MATH level 5","category":"math","domain":null,"eligible":true,"ranked":true,"score":47.55003776435045,"unit":"percent","family":"epoch-ai:epoch-math-level-5:overall","index":22.095597666588326,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.0703488686415494,"costPerTest":null,"latency":null},{"benchmarkId":"aaIfBench","benchmarkName":"Artificial Analysis IFBench","category":"instruction","score":31.2,"unit":"percent","family":"ifbench-family","index":18.18036057474539,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"tau2Bench","benchmarkName":"τ²-Bench Tool-Agent-User Evaluation","category":"agentic","score":30.7,"unit":"percent","family":"tau2-bench","index":23.161134681332175,"aboveFit":false,"run":null,"source":"Victor Barres, Honghua Dong, Soham Ray, Xujie Si, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2506.07982","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"omniscienceAccuracy","benchmarkName":"Artificial Analysis Omniscience Accuracy","category":"knowledge","score":19.9,"unit":"percent","family":"omniscienceAccuracy","index":35.76904454940935,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":8.125,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":13.646167498146063,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.4662778950157507,"costPerTest":null,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":3.3,"unit":"percent","family":"hle","index":27.697510371911797,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-2025-02-28-private:overall","benchmarkName":"FrontierMath-2025-02-28-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":0.3448275862068966,"unit":"percent","family":"epoch-ai:epoch-frontiermath-2025-02-28-private:overall","index":30.21857943502259,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0.3448275862068965,"costPerTest":null,"latency":null},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":0,"unit":"percent","family":"critpt","index":36.8491160127729,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-2025-02-28-public:overall","benchmarkName":"FrontierMath-2025-02-28-Public","category":"math","domain":null,"eligible":false,"ranked":true,"score":0,"unit":"percent","family":"epoch-ai:epoch-frontiermath-2025-02-28-public:overall","index":null,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":10,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":2,"outputPerMillion":6,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":70,"timeToFirstToken":1.7675,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":{"elo":1266.153160937377,"eloUpper":1270.059683215151,"eloLower":1262.2466386596027,"votes":45459,"rank":288,"rankUpper":null,"rankLower":null,"arenaName":"Mistral Large 2-2407","organization":"mistral","updatedAt":"2026-11-09"},"quality":{"coveragePercent":65,"benchmarkCount":9,"categoryCount":5,"confidence":"low","state":"available"},"hosts":[{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":2,"outputPerMillion":6,"cacheReadPerMillion":0.2,"contextTokens":131072,"maxOutputTokens":104857,"deprecated":false},{"host":"snowflake","name":"Snowflake Cortex","kind":"cloud","url":"https://docs.snowflake.com/en/user-guide/snowflake-cortex/cortex-rest-api","inputPerMillion":2,"outputPerMillion":6,"cacheReadPerMillion":null,"contextTokens":128000,"maxOutputTokens":8192,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"mistral-medium-3","providerId":"mistral","slug":"mistral-medium-3","name":"Mistral Medium 3","provider":"Mistral","description":"Mistral Medium 3 is a standard model from Mistral. 12 benchmarks count toward its index, in 7 categories.","logoUrl":null,"releaseDate":"2025-05-07","contextWindowTokens":131072,"modalities":["text","image","file"],"openWeights":false,"details":{"apiId":"mistralai/mistral-medium-3","maxOutputTokens":104857,"knowledgeCutoff":"2025-03-31","retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.04,"weightsUrl":null},"reasoning":false,"overallScore":27.095174363767505,"overallScoreRange":{"low":21.92729743736938,"high":32.263051290165635},"badges":[],"capabilityScores":{"agentic":18.914266765033897,"coding":9.29510266550767,"reasoning":32.81074296138739,"multimodal":24.127665079151438,"knowledge":31.089704088910654,"multilingual":null,"instruction":25.749896554864364,"math":32.17813285997515},"capabilityCounts":{"math":3,"knowledge":3,"multimodal":1,"instruction":1,"reasoning":2,"coding":1,"agentic":1},"benchmarks":[{"benchmarkId":"epoch-ai:epoch-math-level-5:overall","benchmarkName":"MATH level 5","category":"math","domain":null,"eligible":true,"ranked":true,"score":81.62764350453172,"unit":"percent","family":"epoch-ai:epoch-math-level-5:overall","index":38.64897793093232,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0.8995651651009511,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":59.53282828282829,"unit":"percent","family":"gpqa-diamond","index":31.810984099551387,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.824697229654645,"costPerTest":null,"latency":null},{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":57.8,"unit":"percent","family":"gpqa-diamond","index":30.221681348213103,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaMmmuPro","benchmarkName":"Artificial Analysis MMMU-Pro","category":"multimodal","score":53,"unit":"percent","family":"mmmu-pro","index":24.127665079151438,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmmu-pro","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaIfBench","benchmarkName":"Artificial Analysis IFBench","category":"instruction","score":39.3,"unit":"percent","family":"ifbench-family","index":25.749896554864364,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":32.22222222222222,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":27.665787905741954,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":5.965611798482264,"costPerTest":null,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":31.3,"unit":"percent","family":"lcr","index":28.77236991000187,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"liveCodeBenchV6","benchmarkName":"LiveCodeBench v6","category":"coding","score":30.3,"unit":"percent","family":"liveCodeBenchV6","index":9.29510266550767,"aboveFit":false,"run":null,"source":"LiveCodeBench maintainers","sourceUrl":"https://github.com/LiveCodeBench/LiveCodeBench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"tau2Bench","benchmarkName":"τ²-Bench Tool-Agent-User Evaluation","category":"agentic","score":24.3,"unit":"percent","family":"tau2-bench","index":18.914266765033897,"aboveFit":false,"run":null,"source":"Victor Barres, Honghua Dong, Soham Ray, Xujie Si, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2506.07982","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"omniscienceAccuracy","benchmarkName":"Artificial Analysis Omniscience Accuracy","category":"knowledge","score":18.3,"unit":"percent","family":"omniscienceAccuracy","index":33.752634974914095,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":4.1,"unit":"percent","family":"hle","index":28.50014456793563,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-2025-02-28-private:overall","benchmarkName":"FrontierMath-2025-02-28-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":0.34602076124567477,"unit":"percent","family":"epoch-ai:epoch-frontiermath-2025-02-28-private:overall","index":30.21963274325118,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0.3460207612456747,"costPerTest":null,"latency":null},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":0,"unit":"percent","family":"critpt","index":36.8491160127729,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-2025-02-28-public:overall","benchmarkName":"FrontierMath-2025-02-28-Public","category":"math","domain":null,"eligible":false,"ranked":true,"score":0,"unit":"percent","family":"epoch-ai:epoch-frontiermath-2025-02-28-public:overall","index":null,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":0.39999999999999997,"outputPerMillion":2,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":94,"timeToFirstToken":0.601,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":{"elo":1370.147415741093,"eloUpper":1374.8354101714135,"eloLower":1365.4594213107725,"votes":32751,"rank":189,"rankUpper":null,"rankLower":null,"arenaName":"Mistral Medium 3-2505","organization":"mistral","updatedAt":"2026-11-09"},"quality":{"coveragePercent":95,"benchmarkCount":12,"categoryCount":7,"confidence":"medium","state":"available"},"hosts":[{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":0.4,"outputPerMillion":2,"cacheReadPerMillion":0.04,"contextTokens":131072,"maxOutputTokens":104857,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gemini-2-5-flash-lite","providerId":"google","slug":"gemini-2-5-flash-lite","name":"Gemini 2.5 Flash-Lite","provider":"Google","description":"Gemini 2.5 Flash-Lite is a reasoning model from Google in the Gemini 2.5 Flash family. 12 benchmarks count toward its index, in 7 categories.","logoUrl":null,"releaseDate":"2025-07-22","contextWindowTokens":1048576,"modalities":["text","image","file","audio","video"],"openWeights":false,"details":{"apiId":"google/gemini-2.5-flash-lite","maxOutputTokens":65535,"knowledgeCutoff":"2025-01-31","retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.01,"weightsUrl":null},"reasoning":true,"overallScore":27.080840461363728,"overallScoreRange":{"low":21.912963534965602,"high":32.24871738776186},"badges":[],"capabilityScores":{"agentic":15.397329271849387,"coding":24.534746987436318,"reasoning":33.05087114534237,"multimodal":33.18368567713209,"knowledge":27.070579692800603,"multilingual":null,"instruction":18.46071375919424,"math":26.029987413142443},"capabilityCounts":{"multimodal":2,"knowledge":4,"math":1,"reasoning":2,"coding":1,"instruction":1,"agentic":1},"benchmarks":[{"benchmarkId":"mmmu","benchmarkName":"Massive Multi-discipline Multimodal Understanding","category":"multimodal","score":72.9,"unit":"percent","family":"mmmu","index":40.938294009993584,"aboveFit":false,"run":null,"source":"MMMU authors","sourceUrl":"https://arxiv.org/abs/2401.05508","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:medscribe:overall","benchmarkName":"MedScribe","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":72.832,"unit":"percent","family":"vals-ai:medscribe:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medscribe","measuredAt":"2026-10-29","stderr":1.982,"costPerTest":0.001211,"latency":4.847},{"benchmarkId":"gpqa","benchmarkName":"Graduate-Level Google-Proof Q&A","category":"knowledge","score":64.6,"unit":"percent","family":"gpqa-diamond","index":36.45845553766764,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaMmmuPro","benchmarkName":"Artificial Analysis MMMU-Pro","category":"multimodal","score":54,"unit":"percent","family":"mmmu-pro","index":25.4290773442706,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmmu-pro","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aime2025","benchmarkName":"American Invitational Mathematics Examination 2025","category":"math","score":49.8,"unit":"percent","family":"aime2025","index":26.029987413142443,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":47.4,"unit":"percent","family":"gpqa-diamond","index":20.68308552904734,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":32,"unit":"percent","family":"lcr","index":29.25262627791183,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"sweVerified","benchmarkName":"Software Engineering Benchmark Verified","category":"coding","score":31.6,"unit":"percent","family":"swe-bench-verified","index":24.534746987436318,"aboveFit":false,"run":null,"source":"Carlos E. Jimenez, John Yang, Alexander Wettig, Shunyu Yao, Kexin Pei, Ofir Press, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2310.06770","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaIfBench","benchmarkName":"Artificial Analysis IFBench","category":"instruction","score":31.5,"unit":"percent","family":"ifbench-family","index":18.46071375919424,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:medcode:overall","benchmarkName":"MedCode","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":27.115,"unit":"percent","family":"vals-ai:medcode:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medcode","measuredAt":"2026-10-29","stderr":1.843,"costPerTest":0.001342,"latency":5.746},{"benchmarkId":"tau2Bench","benchmarkName":"τ²-Bench Tool-Agent-User Evaluation","category":"agentic","score":19,"unit":"percent","family":"tau2-bench","index":15.397329271849387,"aboveFit":false,"run":null,"source":"Victor Barres, Honghua Dong, Soham Ray, Xujie Si, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2506.07982","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"omniscienceAccuracy","benchmarkName":"Artificial Analysis Omniscience Accuracy","category":"knowledge","score":15.4,"unit":"percent","family":"omniscienceAccuracy","index":30.09789262114144,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"simpleQa","benchmarkName":"Measuring Short-Form Factuality in Large Language Models","category":"knowledge","score":10.7,"unit":"percent","family":"simpleQa","index":21.514828146779763,"aboveFit":false,"run":null,"source":"Jason Wei, Najoung Kim, Hyung Won Chung, Yu-An Chung, Siddhartha Papay, Yifeng Lu, Hannaneh Hajishirzi, Luke Zettlemoyer","sourceUrl":"https://arxiv.org/abs/2411.04368","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":3.7,"unit":"percent","family":"hle","index":28.098827469923716,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":0,"unit":"percent","family":"critpt","index":36.8491160127729,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":0.09999999999999999,"outputPerMillion":0.39999999999999997,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":140,"timeToFirstToken":0.534,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":95,"benchmarkCount":12,"categoryCount":7,"confidence":"medium","state":"available"},"hosts":[{"host":"google-vertex","name":"Google Vertex AI","kind":"maker","url":"https://cloud.google.com/vertex-ai/generative-ai/docs/models","inputPerMillion":0.1,"outputPerMillion":0.4,"cacheReadPerMillion":0.01,"contextTokens":1048576,"maxOutputTokens":65536,"deprecated":true},{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":0.1,"outputPerMillion":0.4,"cacheReadPerMillion":0.01,"contextTokens":1048576,"maxOutputTokens":65535,"deprecated":true},{"host":"sap-ai-core","name":"SAP AI Core","kind":"cloud","url":"https://help.sap.com/docs/sap-ai-core","inputPerMillion":0.1,"outputPerMillion":0.4,"cacheReadPerMillion":0.01,"contextTokens":1048576,"maxOutputTokens":65536,"deprecated":true}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"phi-4-reasoning","providerId":"microsoft","slug":"phi-4-reasoning","name":"Phi-4-reasoning","provider":"Microsoft","description":"Phi-4-reasoning is a reasoning model from Microsoft. 11 benchmarks count toward its index, in 6 categories.","logoUrl":null,"releaseDate":"2025-04-30","contextWindowTokens":32000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":true,"overallScore":27.07217041508555,"overallScoreRange":{"low":20.696731973263113,"high":33.44760885690799},"badges":[],"capabilityScores":{"agentic":5,"coding":33.98384462384614,"reasoning":22.073582735971804,"multimodal":null,"knowledge":33.347262612213484,"multilingual":null,"instruction":21.063837275384675,"math":35.497954376934466},"capabilityCounts":{"instruction":2,"math":2,"knowledge":3,"coding":1,"agentic":1,"reasoning":2},"benchmarks":[{"benchmarkId":"ifeval","benchmarkName":"Instruction-Following Eval","category":"instruction","score":83.4,"unit":"percent","family":"ifeval","index":31.143045710211137,"aboveFit":false,"run":null,"source":"Jeffrey Zhou, Tianjian Lu, Swaroop Mishra, Siddhartha Brahma, Sujoy Basu, Yi Luan, Denny Zhou, Le Hou","sourceUrl":"https://arxiv.org/abs/2311.07911","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aime2024","benchmarkName":"American Invitational Mathematics Examination 2024","category":"math","score":75.3,"unit":"percent","family":"aime2024","index":37.85752314667974,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmluPro","benchmarkName":"Massive Multitask Language Understanding Professional","category":"knowledge","score":74.3,"unit":"percent","family":"mmlu-pro","index":38.089834945559474,"aboveFit":false,"run":null,"source":"Yubo Wang, Xueguang Ma, Ge Zhang, Yuansheng Ni, Abhranil Chandra, Shiguang Guo, Weiming Ren, Aaran Arulraj, Xuan He, Ziyan Jiang, Tianle Li, Max Ku, Kai Wang, Alex Zhuang, Rongqi Fan, Xiang Yue, Wenhu Chen","sourceUrl":"https://arxiv.org/abs/2406.01574","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gpqa","benchmarkName":"Graduate-Level Google-Proof Q&A","category":"knowledge","score":65.8,"unit":"percent","family":"gpqa-diamond","index":37.55906274757138,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aime2025","benchmarkName":"American Invitational Mathematics Examination 2025","category":"math","score":62.9,"unit":"percent","family":"aime2025","index":33.13838560718919,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":57.5,"unit":"percent","family":"gpqa-diamond","index":29.94652954573717,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"liveCodeBench","benchmarkName":"LiveCodeBench: Holistic and Contamination Free Evaluation of Large Language Models for Code","category":"coding","score":53.8,"unit":"percent","family":"livecodebench","index":33.98384462384614,"aboveFit":false,"run":null,"source":"Naman Jain, King Han, Alex Gu, Wen-Ding Li, Fanjia Yan, Tianjun Zhang, Sida Wang, Armando Solar-Lezama, Koushik Sen, Ion Stoica","sourceUrl":"https://arxiv.org/abs/2403.07974","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaIfBench","benchmarkName":"Artificial Analysis IFBench","category":"instruction","score":23.5,"unit":"percent","family":"ifbench-family","index":10.984628840558216,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":3.8,"unit":"percent","family":"hle","index":28.199156744426695,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"tau2Bench","benchmarkName":"τ²-Bench Tool-Agent-User Evaluation","category":"agentic","score":0,"unit":"percent","family":"tau2-bench","index":5,"aboveFit":false,"run":null,"source":"Victor Barres, Honghua Dong, Soham Ray, Xujie Si, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2506.07982","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":0,"unit":"percent","family":"lcr","index":7.298049459170706,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":0,"unit":"percent","family":"critpt","index":36.8491160127729,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":41,"timeToFirstToken":2.64,"source":"BenchLM","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":85,"benchmarkCount":11,"categoryCount":6,"confidence":"low","state":"available"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"llama-3-1-405b","providerId":"meta","slug":"llama-3-1-405b","name":"Llama 3.1 405B","provider":"Meta","description":"Llama 3.1 405B is a standard model from Meta. 9 benchmarks count toward its index, in 5 categories.","logoUrl":null,"releaseDate":"2024-07-23","contextWindowTokens":128000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":27.00035446333705,"overallScoreRange":{"low":18.11035446333705,"high":35.89035446333705},"badges":[],"capabilityScores":{"agentic":15.397329271849387,"coding":null,"reasoning":30.752501384630406,"multimodal":null,"knowledge":26.287610534823074,"multilingual":null,"instruction":33.542431633639595,"math":18.87551822883},"capabilityCounts":{"instruction":2,"knowledge":2,"math":2,"reasoning":2,"agentic":1},"benchmarks":[{"benchmarkId":"ifeval","benchmarkName":"Instruction-Following Eval","category":"instruction","score":88.6,"unit":"percent","family":"ifeval","index":41.61531989686367,"aboveFit":false,"run":null,"source":"Jeffrey Zhou, Tianjian Lu, Swaroop Mishra, Siddhartha Brahma, Sujoy Basu, Yi Luan, Denny Zhou, Le Hou","sourceUrl":"https://arxiv.org/abs/2311.07911","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":51.5,"unit":"percent","family":"gpqa-diamond","index":24.44349349621846,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":50.91540404040404,"unit":"percent","family":"gpqa-diamond","index":23.907318056208535,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.586199270277065,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-math-level-5:overall","benchmarkName":"MATH level 5","category":"math","domain":null,"eligible":true,"ranked":true,"score":49.77341389728097,"unit":"percent","family":"epoch-ai:epoch-math-level-5:overall","index":23.175614580912473,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.2185275936569535,"costPerTest":null,"latency":null},{"benchmarkId":"aaIfBench","benchmarkName":"Artificial Analysis IFBench","category":"instruction","score":39,"unit":"percent","family":"ifbench-family","index":25.46954337041551,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":25.3,"unit":"percent","family":"lcr","index":24.655886756487906,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"tau2Bench","benchmarkName":"τ²-Bench Tool-Agent-User Evaluation","category":"agentic","score":19,"unit":"percent","family":"tau2-bench","index":15.397329271849387,"aboveFit":false,"run":null,"source":"Victor Barres, Honghua Dong, Soham Ray, Xujie Si, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2506.07982","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":9.722222222222223,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":14.57542187674752,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":3.218644609162339,"costPerTest":null,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":4,"unit":"percent","family":"hle","index":28.399815293432653,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":0,"unit":"percent","family":"critpt","index":36.8491160127729,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":29,"timeToFirstToken":2.19,"source":"BenchLM","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":65,"benchmarkCount":9,"categoryCount":5,"confidence":"low","state":"available"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"llama-4-scout","providerId":"meta","slug":"llama-4-scout","name":"Llama 4 Scout","provider":"Meta","description":"Llama 4 Scout is a standard model from Meta. 14 benchmarks count toward its index, in 7 categories.","logoUrl":null,"releaseDate":"2025-04-05","contextWindowTokens":1310720,"modalities":["text","image"],"openWeights":true,"details":{"apiId":"meta-llama/llama-4-scout","maxOutputTokens":16384,"knowledgeCutoff":"2024-08-31","retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":null,"weightsUrl":"https://huggingface.co/meta-llama/Llama-4-Scout-17B-16E-Instruct"},"reasoning":false,"overallScore":26.787124707240437,"overallScoreRange":{"low":21.965510425516534,"high":31.60873898896434},"badges":[],"capabilityScores":{"agentic":23.554299148658277,"coding":23.608419861145094,"reasoning":32.38933458614674,"multimodal":23.997523852639517,"knowledge":28.048087968268142,"multilingual":null,"instruction":25.936798677830264,"math":24.201961947345854},"capabilityCounts":{"math":3,"knowledge":2,"multimodal":1,"instruction":1,"reasoning":4,"coding":1,"agentic":2},"benchmarks":[{"benchmarkId":"epoch-ai:epoch-math-level-5:overall","benchmarkName":"MATH level 5","category":"math","domain":null,"eligible":true,"ranked":true,"score":62.273413897280975,"unit":"percent","family":"epoch-ai:epoch-math-level-5:overall","index":29.24755680836543,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.2022405873783983,"costPerTest":null,"latency":null},{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":58.7,"unit":"percent","family":"gpqa-diamond","index":31.047136755640913,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaMmmuPro","benchmarkName":"Artificial Analysis MMMU-Pro","category":"multimodal","score":52.9,"unit":"percent","family":"mmmu-pro","index":23.997523852639517,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmmu-pro","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":51.830808080808076,"unit":"percent","family":"gpqa-diamond","index":24.746901628578282,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":3.1549020124994755,"costPerTest":null,"latency":null},{"benchmarkId":"aaIfBench","benchmarkName":"Artificial Analysis IFBench","category":"instruction","score":39.5,"unit":"percent","family":"ifbench-family","index":25.936798677830264,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":27.7,"unit":"percent","family":"lcr","index":26.302480017893497,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaSciCode","benchmarkName":"Artificial Analysis SciCode","category":"coding","score":21.3,"unit":"percent","family":"scicode-family","index":23.608419861145094,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/scicode","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"tau2Bench","benchmarkName":"τ²-Bench Tool-Agent-User Evaluation","category":"agentic","score":15.5,"unit":"percent","family":"tau2-bench","index":13.074823380123767,"aboveFit":false,"run":null,"source":"Victor Barres, Honghua Dong, Soham Ray, Xujie Si, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2506.07982","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":7.777777777777778,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":13.444155676710963,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":3.0069597533844092,"costPerTest":null,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":3.8,"unit":"percent","family":"hle","index":28.199156744426695,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"arc-prize:v1-public-eval:overall","benchmarkName":"ARC-AGI-1 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":2.3800000000000003,"unit":"percent","family":"arc-prize:v1-public-eval:overall","index":null,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.0036,"latency":null},{"benchmarkId":"arc-prize:v1-semi-private:overall","benchmarkName":"ARC-AGI-1 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":0.5,"unit":"percent","family":"arc-prize:v1-semi-private:overall","index":27.21722691167801,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.0041,"latency":null},{"benchmarkId":"gdpvalAaNormalized","benchmarkName":"GDPval-AA normalized","category":"agentic","score":0,"unit":"percent","family":"gdpvalAaNormalized","index":34.03377491719279,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":0,"unit":"percent","family":"critpt","index":36.8491160127729,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"arc-prize:v2-public-eval:overall","benchmarkName":"ARC-AGI-2 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":0,"unit":"percent","family":"arc-prize:v2-public-eval:overall","index":null,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.006,"latency":null},{"benchmarkId":"arc-prize:v2-semi-private:overall","benchmarkName":"ARC-AGI-2 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":0,"unit":"percent","family":"arc-prize:v2-semi-private:overall","index":39.188515402242544,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.0062,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-2025-02-28-private:overall","benchmarkName":"FrontierMath-2025-02-28-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":0,"unit":"percent","family":"epoch-ai:epoch-frontiermath-2025-02-28-private:overall","index":29.914173356961165,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-2025-02-28-public:overall","benchmarkName":"FrontierMath-2025-02-28-Public","category":"math","domain":null,"eligible":false,"ranked":true,"score":0,"unit":"percent","family":"epoch-ai:epoch-frontiermath-2025-02-28-public:overall","index":null,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":0.09999999999999999,"outputPerMillion":0.3,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":63,"timeToFirstToken":0.444,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":95,"benchmarkCount":14,"categoryCount":7,"confidence":"medium","state":"available"},"hosts":[{"host":"deepinfra","name":"Deep Infra","kind":"host","url":"https://deepinfra.com/models","inputPerMillion":0.1,"outputPerMillion":0.3,"cacheReadPerMillion":null,"contextTokens":327680,"maxOutputTokens":16384,"deprecated":false},{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":0.1,"outputPerMillion":0.3,"cacheReadPerMillion":null,"contextTokens":327680,"maxOutputTokens":16384,"deprecated":false},{"host":"novita-ai","name":"Novita AI","kind":"host","url":"https://novita.ai/docs/guides/introduction","inputPerMillion":0.18,"outputPerMillion":0.59,"cacheReadPerMillion":null,"contextTokens":131072,"maxOutputTokens":131072,"deprecated":false},{"host":"amazon-bedrock","name":"Amazon Bedrock","kind":"cloud","url":"https://docs.aws.amazon.com/bedrock/latest/userguide/models-supported.html","inputPerMillion":0.17,"outputPerMillion":0.66,"cacheReadPerMillion":null,"contextTokens":10000000,"maxOutputTokens":8192,"deprecated":true},{"host":"google-vertex","name":"Google Vertex AI","kind":"cloud","url":"https://cloud.google.com/vertex-ai/generative-ai/docs/models","inputPerMillion":0.25,"outputPerMillion":0.7,"cacheReadPerMillion":null,"contextTokens":1310720,"maxOutputTokens":8192,"deprecated":false},{"host":"oracle-cloud","name":"Oracle Cloud","kind":"cloud","url":"https://docs.oracle.com/en-us/iaas/Content/generative-ai/pretrained-models.htm","inputPerMillion":0.72,"outputPerMillion":0.72,"cacheReadPerMillion":null,"contextTokens":192000,"maxOutputTokens":16384,"deprecated":false},{"host":"cloudflare","name":"Cloudflare Workers AI","kind":"cloud","url":"https://developers.cloudflare.com/workers-ai/models/","inputPerMillion":0.27,"outputPerMillion":0.85,"cacheReadPerMillion":null,"contextTokens":131000,"maxOutputTokens":16384,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:arc-prize-foundation","name":"ARC Prize Foundation","url":"https://arcprize.org","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gpt-4-1-nano","providerId":"openai","slug":"gpt-4-1-nano","name":"GPT-4.1 nano","provider":"OpenAI","description":"GPT-4.1 nano is a standard model from OpenAI in the GPT-4.1 family. 22 benchmarks count toward its index, in 7 categories.","logoUrl":null,"releaseDate":"2025-04-14","contextWindowTokens":1047576,"modalities":["image","text","file"],"openWeights":false,"details":{"apiId":"openai/gpt-4.1-nano","maxOutputTokens":942818,"knowledgeCutoff":"2024-06-30","retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.03,"weightsUrl":null},"reasoning":false,"overallScore":26.275160551891563,"overallScoreRange":{"low":22.327704968421344,"high":30.222616135361783},"badges":[],"capabilityScores":{"agentic":24.15151494938772,"coding":24.788652324037784,"reasoning":31.64703002578096,"multimodal":17.070757071542765,"knowledge":26.708814635894367,"multilingual":null,"instruction":24.834117500205206,"math":29.099506752673236},"capabilityCounts":{"instruction":2,"math":5,"knowledge":6,"multimodal":1,"coding":1,"reasoning":5,"agentic":2},"benchmarks":[{"benchmarkId":"ifeval","benchmarkName":"Instruction-Following Eval","category":"instruction","score":83.2,"unit":"percent","family":"ifeval","index":30.74026593380142,"aboveFit":false,"run":null,"source":"Jeffrey Zhou, Tianjian Lu, Swaroop Mishra, Siddhartha Brahma, Sujoy Basu, Yi Luan, Denny Zhou, Le Hou","sourceUrl":"https://arxiv.org/abs/2311.07911","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:math500:overall","benchmarkName":"MATH 500","category":"math","domain":null,"eligible":true,"ranked":true,"score":80.2,"unit":"percent","family":"math-500","index":28.878791893253656,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/math500","measuredAt":"2026-01-09","stderr":1.782,"costPerTest":0.000314,"latency":3.372},{"benchmarkId":"mmlu","benchmarkName":"Massive Multitask Language Understanding","category":"knowledge","score":80.1,"unit":"percent","family":"mmlu","index":27.171102630803325,"aboveFit":false,"run":null,"source":"Dan Hendrycks, Collin Burns, Steven Basart, Andy Zou, Mantas Mazeika, Dawn Song, Jacob Steinhardt","sourceUrl":"https://arxiv.org/abs/2009.03300","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-math-level-5:overall","benchmarkName":"MATH level 5","category":"math","domain":null,"eligible":true,"ranked":true,"score":69.99622356495469,"unit":"percent","family":"epoch-ai:epoch-math-level-5:overall","index":32.99895313922383,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.0231591350302844,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:mgsm:overall","benchmarkName":"MGSM","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":69.273,"unit":"percent","family":"vals-ai:mgsm:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mgsm","measuredAt":"2026-01-09","stderr":2.799,"costPerTest":0.000129,"latency":1.458},{"benchmarkId":"vals-ai:medqa:overall","benchmarkName":"MedQA","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":68.225,"unit":"percent","family":"vals-ai:medqa:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medqa","measuredAt":"2026-04-16","stderr":0.425,"costPerTest":0.000154,"latency":1.416},{"benchmarkId":"vals-ai:mmlu_pro:overall","benchmarkName":"MMLU Pro","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":63.479,"unit":"percent","family":"mmlu-pro","index":24.456670185964647,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","measuredAt":"2026-09-01","stderr":0.462,"costPerTest":0.000337,"latency":2.403},{"benchmarkId":"vals-ai:legal_bench:overall","benchmarkName":"LegalBench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":61.056,"unit":"percent","family":"vals-ai:legal_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","measuredAt":"2026-10-13","stderr":0.595,"costPerTest":null,"latency":0.357},{"benchmarkId":"vals-ai:tax_eval_v2:overall","benchmarkName":"TaxEval v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":60.752,"unit":"percent","family":"vals-ai:tax_eval_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","measuredAt":"2026-09-01","stderr":0.956,"costPerTest":0.000291,"latency":2.998},{"benchmarkId":"vals-ai:mmmu:overall","benchmarkName":"MMMU Pro","category":"multimodal","domain":null,"eligible":true,"ranked":true,"score":55.055,"unit":"percent","family":"mmmu-pro","index":26.80206728397132,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmmu","measuredAt":"2026-09-01","stderr":1.196,"costPerTest":0.000566,"latency":7.251},{"benchmarkId":"vals-ai:mortgage_tax:overall","benchmarkName":"MortgageTax","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":52.822,"unit":"percent","family":"vals-ai:mortgage_tax:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mortgage_tax","measuredAt":"2026-09-01","stderr":0.995,"costPerTest":0.001003,"latency":3.858},{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":51.2,"unit":"percent","family":"gpqa-diamond","index":24.168341693742526,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:gpqa:overall","benchmarkName":"GPQA Diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":50.758,"unit":"percent","family":"gpqa-diamond","index":23.76295137142798,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","measuredAt":"2026-09-01","stderr":2.457,"costPerTest":0.000488,"latency":4.192},{"benchmarkId":"gpqa","benchmarkName":"Graduate-Level Google-Proof Q&A","category":"knowledge","score":50.3,"unit":"percent","family":"gpqa-diamond","index":23.342886286314712,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":48.926767676767675,"unit":"percent","family":"gpqa-diamond","index":22.083395123129417,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.484973495043537,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:lcb:overall","benchmarkName":"LiveCodeBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":42.718,"unit":"percent","family":"livecodebench","index":24.788652324037784,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","measuredAt":"2026-09-01","stderr":1.152,"costPerTest":0.000298,"latency":11.801},{"benchmarkId":"vals-ai:corp_fin_v2:overall","benchmarkName":"CorpFin v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":42.075,"unit":"percent","family":"vals-ai:corp_fin_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","measuredAt":"2026-08-12","stderr":0.968,"costPerTest":0.006454,"latency":4.954},{"benchmarkId":"aaMmmuPro","benchmarkName":"Artificial Analysis MMMU-Pro","category":"multimodal","score":40.1,"unit":"percent","family":"mmmu-pro","index":7.339446859114204,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmmu-pro","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaIfBench","benchmarkName":"Artificial Analysis IFBench","category":"instruction","score":32,"unit":"percent","family":"ifbench-family","index":18.927969066608988,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":28.888888888888886,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":25.726474419964998,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":5.970019560922745,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:aime:overall","benchmarkName":"AIME","category":"math","domain":null,"eligible":true,"ranked":true,"score":26.458,"unit":"percent","family":"vals-ai:aime:overall","index":27.065922719778257,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/aime","measuredAt":"2026-04-16","stderr":0.814,"costPerTest":0.001184,"latency":11.914},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":20.3,"unit":"percent","family":"lcr","index":21.225484128559607,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"tau2Bench","benchmarkName":"τ²-Bench Tool-Agent-User Evaluation","category":"agentic","score":17.3,"unit":"percent","family":"tau2-bench","index":14.269254981582655,"aboveFit":false,"run":null,"source":"Victor Barres, Honghua Dong, Soham Ray, Xujie Si, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2506.07982","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"omniscienceAccuracy","benchmarkName":"Artificial Analysis Omniscience Accuracy","category":"knowledge","score":13.7,"unit":"percent","family":"omniscienceAccuracy","index":27.955457448240228,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-simpleqa-verified:overall","benchmarkName":"SimpleQA Verified","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":6,"unit":"percent","family":"epoch-ai:epoch-simpleqa-verified:overall","index":28.917581977059967,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0.7513751157474972,"costPerTest":null,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":3.8,"unit":"percent","family":"hle","index":28.199156744426695,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"arc-prize:v1-public-eval:overall","benchmarkName":"ARC-AGI-1 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":1.7500000000000002,"unit":"percent","family":"arc-prize:v1-public-eval:overall","index":null,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.0018,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-2025-02-28-private:overall","benchmarkName":"FrontierMath-2025-02-28-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":1.0344827586206897,"unit":"percent","family":"epoch-ai:epoch-frontiermath-2025-02-28-private:overall","index":30.82739159114545,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0.5951886714450795,"costPerTest":null,"latency":null},{"benchmarkId":"gdpvalAaNormalized","benchmarkName":"GDPval-AA normalized","category":"agentic","score":0,"unit":"percent","family":"gdpvalAaNormalized","index":34.03377491719279,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":0,"unit":"percent","family":"critpt","index":36.8491160127729,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"arc-prize:v1-semi-private:overall","benchmarkName":"ARC-AGI-1 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":0,"unit":"percent","family":"arc-prize:v1-semi-private:overall","index":26.987358860190817,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.0021,"latency":null},{"benchmarkId":"arc-prize:v2-public-eval:overall","benchmarkName":"ARC-AGI-2 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":0,"unit":"percent","family":"arc-prize:v2-public-eval:overall","index":null,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.0036,"latency":null},{"benchmarkId":"arc-prize:v2-semi-private:overall","benchmarkName":"ARC-AGI-2 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":0,"unit":"percent","family":"arc-prize:v2-semi-private:overall","index":39.188515402242544,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.0036,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-2025-02-28-public:overall","benchmarkName":"FrontierMath-2025-02-28-Public","category":"math","domain":null,"eligible":false,"ranked":true,"score":0,"unit":"percent","family":"epoch-ai:epoch-frontiermath-2025-02-28-public:overall","index":null,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-chess-puzzles:overall","benchmarkName":"Chess Puzzles","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":0,"unit":"percent","family":"epoch-ai:epoch-chess-puzzles:overall","index":33.98467572513893,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":0.09999999999999999,"outputPerMillion":0.39999999999999997,"batch":{"inputPerMillion":0.049999999999999996,"outputPerMillion":0.19999999999999998},"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":56,"timeToFirstToken":1.3715,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":{"elo":1284.7871224037058,"eloUpper":1292.4288644780108,"eloLower":1277.145380329401,"votes":6103,"rank":269,"rankUpper":null,"rankLower":null,"arenaName":"GPT-4.1 nano-2025-04-14","organization":"openai","updatedAt":"2026-11-09"},"quality":{"coveragePercent":95,"benchmarkCount":22,"categoryCount":7,"confidence":"medium","state":"available"},"hosts":[{"host":"azure","name":"Microsoft Azure","kind":"cloud","url":"https://learn.microsoft.com/en-us/azure/ai-services/openai/concepts/models","inputPerMillion":0.1,"outputPerMillion":0.4,"cacheReadPerMillion":0.025,"contextTokens":1047576,"maxOutputTokens":32768,"deprecated":true},{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":0.1,"outputPerMillion":0.4,"cacheReadPerMillion":0.03,"contextTokens":1047576,"maxOutputTokens":942818,"deprecated":false},{"host":"sap-ai-core","name":"SAP AI Core","kind":"cloud","url":"https://help.sap.com/docs/sap-ai-core","inputPerMillion":0.1,"outputPerMillion":0.4,"cacheReadPerMillion":null,"contextTokens":1047576,"maxOutputTokens":32768,"deprecated":true}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:arc-prize-foundation","name":"ARC Prize Foundation","url":"https://arcprize.org","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gpt-4o-mini","providerId":"openai","slug":"gpt-4o-mini","name":"GPT-4o mini","provider":"OpenAI","description":"GPT-4o mini is a standard model from OpenAI in the GPT-4o family. 18 benchmarks count toward its index, in 7 categories.","logoUrl":null,"releaseDate":"2024-07-18","contextWindowTokens":128000,"modalities":["text","image","file"],"openWeights":false,"details":{"apiId":"openai/gpt-4o-mini","maxOutputTokens":16384,"knowledgeCutoff":"2023-10-31","retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.075,"weightsUrl":null},"reasoning":false,"overallScore":26.20556024179883,"overallScoreRange":{"low":21.894254691203336,"high":30.51686579239432},"badges":[],"capabilityScores":{"agentic":34.03377491719279,"coding":19.528246418215492,"reasoning":41.14140455845467,"multimodal":18.959106268230677,"knowledge":25.734763083535128,"multilingual":null,"instruction":17.99345845177949,"math":22.029249901603183},"capabilityCounts":{"coding":2,"knowledge":5,"math":5,"multimodal":1,"instruction":1,"reasoning":3,"agentic":1},"benchmarks":[{"benchmarkId":"humaneval","benchmarkName":"Evaluating Large Language Models Trained on Code","category":"coding","score":87.2,"unit":"percent","family":"humaneval","index":27.788473838992836,"aboveFit":false,"run":null,"source":"Mark Chen, Jerry Tworek, Heewoo Jun, Qiming Yuan, Henrique Ponde de Oliveira Pinto, Jared Kaplan, Harri Edwards, Yuri Burda, Nicholas Joseph, Greg Brockman, Alex Ray, Raul Puri, Gretchen Krueger, Michael Petrov, Heidy Khlaaf, Girish Sastry, Pamela Mishkin, Brooke Chan, Scott Gray, Nick Ryder, Mikhail Pavlov, Alethea Power, Lukasz Kaiser, Mohammad Bavarian, Clemens Winter, Philippe Tillet, Felipe Petroski Such, Dave Cummings, Matthias Plappert, Fotios Chantzis, Elizabeth Barnes, Ariel Herbert-Voss, William Hebgen Guss, Alex Nichol, Alex Paino, Nikolas Tezak, Jie Tang, Igor Babuschkin, Suchir Balaji, Shantanu Jain, William Saunders, Christopher Hesse, Andrew N. Carr, Jan Leike, Josh Achiam, Vedant Misra, Evan Morikawa, Alec Radford, Matthew Knight, Miles Brundage, Mira Murati, Katie Mayer, Peter Welinder, Bob McGrew, Dario Amodei, Sam McCandlish, Ilya Sutskever, Wojciech Zaremba","sourceUrl":"https://arxiv.org/abs/2107.03374","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:mgsm:overall","benchmarkName":"MGSM","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":86.182,"unit":"percent","family":"vals-ai:mgsm:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mgsm","measuredAt":"2026-01-09","stderr":2.133,"costPerTest":0.000187,"latency":4.033},{"benchmarkId":"mmlu","benchmarkName":"Massive Multitask Language Understanding","category":"knowledge","score":82,"unit":"percent","family":"mmlu","index":29.439680628125636,"aboveFit":false,"run":null,"source":"Dan Hendrycks, Collin Burns, Steven Basart, Andy Zou, Mantas Mazeika, Dawn Song, Jacob Steinhardt","sourceUrl":"https://arxiv.org/abs/2009.03300","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:math500:overall","benchmarkName":"MATH 500","category":"math","domain":null,"eligible":true,"ranked":true,"score":72.6,"unit":"percent","family":"math-500","index":20.847429478786555,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/math500","measuredAt":"2026-01-09","stderr":3.898,"costPerTest":0.00035,"latency":6.304},{"benchmarkId":"vals-ai:medqa:overall","benchmarkName":"MedQA","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":72.436,"unit":"percent","family":"vals-ai:medqa:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medqa","measuredAt":"2026-04-16","stderr":0.799,"costPerTest":0.000157,"latency":2.322},{"benchmarkId":"vals-ai:mmlu_pro:overall","benchmarkName":"MMLU Pro","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":62.735,"unit":"percent","family":"mmlu-pro","index":23.519319240475454,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","measuredAt":"2026-09-01","stderr":0.461,"costPerTest":0.000412,"latency":5.086},{"benchmarkId":"vals-ai:tax_eval_v2:overall","benchmarkName":"TaxEval v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":60.548,"unit":"percent","family":"vals-ai:tax_eval_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","measuredAt":"2026-09-01","stderr":0.963,"costPerTest":0.000323,"latency":8.899},{"benchmarkId":"vals-ai:mmmu:overall","benchmarkName":"MMMU Pro","category":"multimodal","domain":null,"eligible":true,"ranked":true,"score":56.557,"unit":"percent","family":"mmmu-pro","index":28.756788506180307,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmmu","measuredAt":"2026-09-01","stderr":1.191,"costPerTest":0.00037,"latency":45.016},{"benchmarkId":"vals-ai:mortgage_tax:overall","benchmarkName":"MortgageTax","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":54.492,"unit":"percent","family":"vals-ai:mortgage_tax:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mortgage_tax","measuredAt":"2026-09-01","stderr":1.93,"costPerTest":0.000822,"latency":6.059},{"benchmarkId":"epoch-ai:epoch-math-level-5:overall","benchmarkName":"MATH level 5","category":"math","domain":null,"eligible":true,"ranked":true,"score":52.63406344410876,"unit":"percent","family":"epoch-ai:epoch-math-level-5:overall","index":24.565190483418704,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.1275275696871734,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:corp_fin_v2:overall","benchmarkName":"CorpFin v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":45.455,"unit":"percent","family":"vals-ai:corp_fin_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","measuredAt":"2026-08-12","stderr":0.977,"costPerTest":0.008848,"latency":0.041},{"benchmarkId":"vals-ai:gpqa:overall","benchmarkName":"GPQA Diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":44.192,"unit":"percent","family":"gpqa-diamond","index":17.74079558790467,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","measuredAt":"2026-09-01","stderr":2.495,"costPerTest":0.000536,"latency":15.32},{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":42.6,"unit":"percent","family":"gpqa-diamond","index":16.280656689432377,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaMmmuPro","benchmarkName":"Artificial Analysis MMMU-Pro","category":"multimodal","score":41.5,"unit":"percent","family":"mmmu-pro","index":9.161424030281033,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmmu-pro","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":37.7209595959596,"unit":"percent","family":"gpqa-diamond","index":11.805734151016928,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.4068925424620136,"costPerTest":null,"latency":null},{"benchmarkId":"aaIfBench","benchmarkName":"Artificial Analysis IFBench","category":"instruction","score":31,"unit":"percent","family":"ifbench-family","index":17.99345845177949,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:lcb:overall","benchmarkName":"LiveCodeBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":26.423,"unit":"percent","family":"livecodebench","index":11.268018997438148,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","measuredAt":"2026-09-01","stderr":1.129,"costPerTest":0.000298,"latency":7.364},{"benchmarkId":"epoch-ai:epoch-mystery-game-puzzles:overall","benchmarkName":"Mystery Game Puzzles","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":12,"unit":"percent","family":"epoch-ai:epoch-mystery-game-puzzles:overall","index":47.973976962532255,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":3.2659863237109037,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:aime:overall","benchmarkName":"AIME","category":"math","domain":null,"eligible":true,"ranked":true,"score":11.458,"unit":"percent","family":"vals-ai:aime:overall","index":20.510236313257174,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/aime","measuredAt":"2026-04-16","stderr":1.03,"costPerTest":0.00101,"latency":28.771},{"benchmarkId":"epoch-ai:epoch-simpleqa-verified:overall","benchmarkName":"SimpleQA Verified","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":8.3,"unit":"percent","family":"epoch-ai:epoch-simpleqa-verified:overall","index":30.83368501720356,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0.8728527206074758,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":6.944444444444445,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":12.959327305266726,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.73963714244758,"costPerTest":null,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":4.2,"unit":"percent","family":"hle","index":28.60047384243861,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-chess-puzzles:overall","benchmarkName":"Chess Puzzles","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":3,"unit":"percent","family":"epoch-ai:epoch-chess-puzzles:overall","index":36.261721310589216,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.7144660799776528,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tiers-1-3-v2-private:overall","benchmarkName":"FrontierMath-Tiers-1-3-v2-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":0.7017543859649122,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tiers-1-3-v2-private:overall","index":31.26406592728675,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0.49534089619636035,"costPerTest":null,"latency":null},{"benchmarkId":"gdpvalAaNormalized","benchmarkName":"GDPval-AA normalized","category":"agentic","score":0,"unit":"percent","family":"gdpvalAaNormalized","index":34.03377491719279,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"arc-prize:v2-public-eval:overall","benchmarkName":"ARC-AGI-2 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":0,"unit":"percent","family":"arc-prize:v2-public-eval:overall","index":null,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.01,"latency":null},{"benchmarkId":"arc-prize:v2-semi-private:overall","benchmarkName":"ARC-AGI-2 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":0,"unit":"percent","family":"arc-prize:v2-semi-private:overall","index":39.188515402242544,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.01,"latency":null}],"price":{"inputPerMillion":0.15,"outputPerMillion":0.6,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":56,"timeToFirstToken":0.591,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":{"elo":1286.3843413327656,"eloUpper":1289.9082598645937,"eloLower":1282.8604228009376,"votes":68705,"rank":268,"rankUpper":null,"rankLower":null,"arenaName":"GPT-4o mini-2024-07-18","organization":"openai","updatedAt":"2026-11-09"},"quality":{"coveragePercent":95,"benchmarkCount":18,"categoryCount":7,"confidence":"medium","state":"available"},"hosts":[{"host":"digitalocean","name":"DigitalOcean","kind":"cloud","url":"https://docs.digitalocean.com/products/gradient-ai-platform/details/models/","inputPerMillion":0.15,"outputPerMillion":0.6,"cacheReadPerMillion":0.075,"contextTokens":128000,"maxOutputTokens":16384,"deprecated":false},{"host":"azure","name":"Microsoft Azure","kind":"cloud","url":"https://learn.microsoft.com/en-us/azure/ai-services/openai/concepts/models","inputPerMillion":0.15,"outputPerMillion":0.6,"cacheReadPerMillion":0.075,"contextTokens":128000,"maxOutputTokens":16384,"deprecated":true},{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":0.15,"outputPerMillion":0.6,"cacheReadPerMillion":0.075,"contextTokens":128000,"maxOutputTokens":16384,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:arc-prize-foundation","name":"ARC Prize Foundation","url":"https://arcprize.org","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"nova-pro","providerId":"amazon","slug":"nova-pro","name":"Nova Pro","provider":"Amazon","description":"Nova Pro is a standard model from Amazon. 8 benchmarks count toward its index, in 5 categories.","logoUrl":null,"releaseDate":"2024-12-03","contextWindowTokens":128000,"modalities":["text"],"openWeights":false,"details":null,"reasoning":false,"overallScore":25.593668316441114,"overallScoreRange":{"low":16.376709990547877,"high":34.810626642334356},"badges":[],"capabilityScores":{"agentic":12.079463712241356,"coding":null,"reasoning":29.277428254621235,"multimodal":12.805378372614697,"knowledge":27.520491636995455,"multilingual":null,"instruction":24.62848381706896,"math":null},"capabilityCounts":{"knowledge":3,"multimodal":1,"instruction":1,"reasoning":2,"agentic":1},"benchmarks":[{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":49.9,"unit":"percent","family":"gpqa-diamond","index":22.9760172163468,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaMmmuPro","benchmarkName":"Artificial Analysis MMMU-Pro","category":"multimodal","score":44.3,"unit":"percent","family":"mmmu-pro","index":12.805378372614697,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmmu-pro","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaIfBench","benchmarkName":"Artificial Analysis IFBench","category":"instruction","score":38.1,"unit":"percent","family":"ifbench-family","index":24.62848381706896,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":21,"unit":"percent","family":"lcr","index":21.705740496469566,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"omniscienceAccuracy","benchmarkName":"Artificial Analysis Omniscience Accuracy","category":"knowledge","score":16.9,"unit":"percent","family":"omniscienceAccuracy","index":31.988276597230744,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"tau2Bench","benchmarkName":"τ²-Bench Tool-Agent-User Evaluation","category":"agentic","score":14,"unit":"percent","family":"tau2-bench","index":12.079463712241356,"aboveFit":false,"run":null,"source":"Victor Barres, Honghua Dong, Soham Ray, Xujie Si, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2506.07982","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":3.2,"unit":"percent","family":"hle","index":27.597181097408818,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":0,"unit":"percent","family":"critpt","index":36.8491160127729,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":129,"timeToFirstToken":1.06,"source":"BenchLM","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":65,"benchmarkCount":8,"categoryCount":5,"confidence":"low","state":"available"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gemma-3-27b","providerId":"google","slug":"gemma-3-27b","name":"Gemma 3 27B","provider":"Google","description":"Gemma 3 27B is a standard model from Google. 13 benchmarks count toward its index, in 8 categories.","logoUrl":null,"releaseDate":"2025-03-12","contextWindowTokens":32000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":25.286338880034993,"overallScoreRange":{"low":20.801587793554013,"high":29.771089966515973},"badges":[],"capabilityScores":{"agentic":21.895366368854262,"coding":26.215802153974558,"reasoning":24.577776654359464,"multimodal":17.620603753555613,"knowledge":24.866027276308493,"multilingual":43.66047032749681,"instruction":18.741066943643087,"math":28.485621666405798},"capabilityCounts":{"math":2,"multilingual":1,"multimodal":1,"knowledge":3,"instruction":1,"coding":1,"agentic":2,"reasoning":2},"benchmarks":[{"benchmarkId":"epoch-ai:epoch-math-level-5:overall","benchmarkName":"MATH level 5","category":"math","domain":null,"eligible":true,"ranked":true,"score":74.0370090634441,"unit":"percent","family":"epoch-ai:epoch-math-level-5:overall","index":34.96178642725242,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.1195857525727289,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":58.05,"unit":"percent","family":"euroeval","index":50.980728534043486,"aboveFit":true,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":0.24126942926943729,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-french:overall","benchmarkName":"EuroEval French","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":57.54249999999999,"unit":"percent","family":"euroeval","index":50.292167806466665,"aboveFit":true,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/french","measuredAt":null,"stderr":0.3285645864970843,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-portuguese:overall","benchmarkName":"EuroEval Portuguese","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":53.48500000000001,"unit":"percent","family":"euroeval","index":44.787073910618616,"aboveFit":true,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/portuguese","measuredAt":null,"stderr":0.35057096285916206,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":51.824285714285715,"unit":"percent","family":"euroeval","index":42.533866744375004,"aboveFit":true,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.2913970571195927,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-spanish:overall","benchmarkName":"EuroEval Spanish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":49.183749999999996,"unit":"percent","family":"euroeval","index":38.95126735004767,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/spanish","measuredAt":null,"stderr":0.41362724765179576,"costPerTest":null,"latency":null},{"benchmarkId":"aaMmmuPro","benchmarkName":"Artificial Analysis MMMU-Pro","category":"multimodal","score":48,"unit":"percent","family":"mmmu-pro","index":17.620603753555613,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmmu-pro","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":47.72727272727273,"unit":"percent","family":"gpqa-diamond","index":20.98325113174836,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.698212089116894,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":47.176249999999996,"unit":"percent","family":"euroeval","index":36.22755176263794,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.4577577825662825,"costPerTest":null,"latency":null},{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":42.8,"unit":"percent","family":"gpqa-diamond","index":16.464091224416322,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaIfBench","benchmarkName":"Artificial Analysis IFBench","category":"instruction","score":31.8,"unit":"percent","family":"ifbench-family","index":18.741066943643087,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaSciCode","benchmarkName":"Artificial Analysis SciCode","category":"coding","score":23.3,"unit":"percent","family":"scicode-family","index":26.215802153974558,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/scicode","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":22.5,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":22.009456905559173,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":5.279438866723883,"costPerTest":null,"latency":null},{"benchmarkId":"omniscienceAccuracy","benchmarkName":"Artificial Analysis Omniscience Accuracy","category":"knowledge","score":13,"unit":"percent","family":"omniscienceAccuracy","index":27.073278259398553,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"tau2Bench","benchmarkName":"τ²-Bench Tool-Agent-User Evaluation","category":"agentic","score":10.5,"unit":"percent","family":"tau2-bench","index":9.756957820515737,"aboveFit":false,"run":null,"source":"Victor Barres, Honghua Dong, Soham Ray, Xujie Si, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2506.07982","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":7.3,"unit":"percent","family":"lcr","index":12.306437295946026,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":4.4,"unit":"percent","family":"hle","index":28.801132391444572,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gdpvalAaNormalized","benchmarkName":"GDPval-AA normalized","category":"agentic","score":0,"unit":"percent","family":"gdpvalAaNormalized","index":34.03377491719279,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":0,"unit":"percent","family":"critpt","index":36.8491160127729,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":31,"timeToFirstToken":2.04,"source":"BenchLM","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":100,"benchmarkCount":13,"categoryCount":8,"confidence":"medium","state":"available"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"llama-3-1-70b","providerId":"meta","slug":"llama-3-1-70b","name":"Llama 3.1 70B","provider":"Meta","description":"Llama 3.1 70B is a standard model from Meta. 14 benchmarks count toward its index, in 7 categories.","logoUrl":null,"releaseDate":"2024-07-23","contextWindowTokens":128000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":24.804083443810143,"overallScoreRange":{"low":19.48246916208624,"high":30.125697725534046},"badges":[],"capabilityScores":{"agentic":17.122619362845562,"coding":21.41113500871414,"reasoning":23.685871971098106,"multimodal":null,"knowledge":27.597994427493834,"multilingual":40.9914908583761,"instruction":27.76223417436537,"math":13.91738222532301},"capabilityCounts":{"instruction":2,"multilingual":2,"knowledge":4,"coding":1,"math":2,"agentic":1,"reasoning":2},"benchmarks":[{"benchmarkId":"gsm8k","benchmarkName":"Grade School Math 8K","category":"math","score":95.1,"unit":"percent","family":"gsm8k","index":null,"aboveFit":false,"run":null,"source":"Karl Cobbe and colleagues (OpenAI)","sourceUrl":"https://arxiv.org/abs/2110.14168","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"ifeval","benchmarkName":"Instruction-Following Eval","category":"instruction","score":87.5,"unit":"percent","family":"ifeval","index":39.40003112661026,"aboveFit":false,"run":null,"source":"Jeffrey Zhou, Tianjian Lu, Swaroop Mishra, Siddhartha Brahma, Sujoy Basu, Yi Luan, Denny Zhou, Le Hou","sourceUrl":"https://arxiv.org/abs/2311.07911","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mgsm","benchmarkName":"Multilingual Grade School Math","category":"multilingual","score":86.9,"unit":"percent","family":"mgsm","index":31.77076642263611,"aboveFit":false,"run":null,"source":"Freda Shi, Mirac Suzgun, Markus Freitag, Xuezhi Wang, Suraj Srivats, Soroush Vosoughi, Hyung Won Chung, Yi Tay, Sebastian Ruder, Denny Zhou, Dipanjan Das, Jason Wei","sourceUrl":"https://arxiv.org/abs/2210.03057","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmlu","benchmarkName":"Massive Multitask Language Understanding","category":"knowledge","score":86,"unit":"percent","family":"mmlu","index":34.2156343066989,"aboveFit":false,"run":null,"source":"Dan Hendrycks, Collin Burns, Steven Basart, Andy Zou, Mantas Mazeika, Dawn Song, Jacob Steinhardt","sourceUrl":"https://arxiv.org/abs/2009.03300","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"humaneval","benchmarkName":"Evaluating Large Language Models Trained on Code","category":"coding","score":80.5,"unit":"percent","family":"humaneval","index":21.41113500871414,"aboveFit":false,"run":null,"source":"Mark Chen, Jerry Tworek, Heewoo Jun, Qiming Yuan, Henrique Ponde de Oliveira Pinto, Jared Kaplan, Harri Edwards, Yuri Burda, Nicholas Joseph, Greg Brockman, Alex Ray, Raul Puri, Gretchen Krueger, Michael Petrov, Heidy Khlaaf, Girish Sastry, Pamela Mishkin, Brooke Chan, Scott Gray, Nick Ryder, Mikhail Pavlov, Alethea Power, Lukasz Kaiser, Mohammad Bavarian, Clemens Winter, Philippe Tillet, Felipe Petroski Such, Dave Cummings, Matthias Plappert, Fotios Chantzis, Elizabeth Barnes, Ariel Herbert-Voss, William Hebgen Guss, Alex Nichol, Alex Paino, Nikolas Tezak, Jie Tang, Igor Babuschkin, Suchir Balaji, Shantanu Jain, William Saunders, Christopher Hesse, Andrew N. Carr, Jan Leike, Josh Achiam, Vedant Misra, Evan Morikawa, Alec Radford, Matthew Knight, Miles Brundage, Mira Murati, Katie Mayer, Peter Welinder, Bob McGrew, Dario Amodei, Sam McCandlish, Ilya Sutskever, Wojciech Zaremba","sourceUrl":"https://arxiv.org/abs/2107.03374","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmluPro","benchmarkName":"Massive Multitask Language Understanding Professional","category":"knowledge","score":66.4,"unit":"percent","family":"mmlu-pro","index":28.136780551252198,"aboveFit":false,"run":null,"source":"Yubo Wang, Xueguang Ma, Ge Zhang, Yuansheng Ni, Abhranil Chandra, Shiguang Guo, Weiming Ren, Aaran Arulraj, Xuan He, Ziyan Jiang, Tianle Li, Max Ku, Kai Wang, Alex Zhuang, Rongqi Fan, Xiang Yue, Wenhu Chen","sourceUrl":"https://arxiv.org/abs/2406.01574","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-portuguese:overall","benchmarkName":"EuroEval Portuguese","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":59.30375,"unit":"percent","family":"euroeval","index":52.68177880438718,"aboveFit":true,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/portuguese","measuredAt":null,"stderr":0.41480982088181084,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-french:overall","benchmarkName":"EuroEval French","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":58.44,"unit":"percent","family":"euroeval","index":51.50986879760005,"aboveFit":true,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/french","measuredAt":null,"stderr":0.47823078895027243,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":56.52714285714285,"unit":"percent","family":"euroeval","index":48.91456179063214,"aboveFit":true,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.4722265972853218,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":52.07375,"unit":"percent","family":"euroeval","index":42.87233237999245,"aboveFit":true,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.573759531511242,"costPerTest":null,"latency":null},{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":49.1,"unit":"percent","family":"gpqa-diamond","index":22.242279076410977,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gpqa","benchmarkName":"Graduate-Level Google-Proof Q&A","category":"knowledge","score":46.7,"unit":"percent","family":"gpqa-diamond","index":20.041064656603492,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":44.19191919191919,"unit":"percent","family":"gpqa-diamond","index":17.74072147294104,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.4522863472149385,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-math-level-5:overall","benchmarkName":"MATH level 5","category":"math","domain":null,"eligible":true,"ranked":true,"score":36.678625377643506,"unit":"percent","family":"epoch-ai:epoch-math-level-5:overall","index":16.81475063115625,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.022960485969093,"costPerTest":null,"latency":null},{"benchmarkId":"aaIfBench","benchmarkName":"Artificial Analysis IFBench","category":"instruction","score":29,"unit":"percent","family":"ifbench-family","index":16.124437222120484,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"tau2Bench","benchmarkName":"τ²-Bench Tool-Agent-User Evaluation","category":"agentic","score":21.6,"unit":"percent","family":"tau2-bench","index":17.122619362845562,"aboveFit":false,"run":null,"source":"Victor Barres, Honghua Dong, Soham Ray, Xujie Si, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2506.07982","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":4.7,"unit":"percent","family":"lcr","index":10.522627929423308,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":3.6111111111111107,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":11.020013819489769,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.691212350465157,"costPerTest":null,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":3.6,"unit":"percent","family":"hle","index":27.998498195420737,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":0,"unit":"percent","family":"critpt","index":36.8491160127729,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":90,"benchmarkCount":14,"categoryCount":7,"confidence":"medium","state":"available"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"llada2-2-mini","providerId":"inclusionai","slug":"llada2-2-mini","name":"LLaDA2.2-mini","provider":"InclusionAI","description":"LLaDA2.2-mini is a reasoning model from InclusionAI in the LLaDA2.2 family. 9 benchmarks count toward its index, in 6 categories.","logoUrl":null,"releaseDate":"2026-07-16","contextWindowTokens":128000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":true,"overallScore":24.34594715183735,"overallScoreRange":{"low":17.45594715183735,"high":31.23594715183735},"badges":[],"capabilityScores":{"agentic":34.80177138833135,"coding":7.521619227871604,"reasoning":21.85775011167322,"multimodal":null,"knowledge":17.940739231037178,"multilingual":null,"instruction":12.320979019764394,"math":10.524891953592032},"capabilityCounts":{"agentic":4,"knowledge":1,"math":1,"reasoning":1,"coding":1,"instruction":1},"benchmarks":[{"benchmarkId":"pinchBench","benchmarkName":"PinchBench","category":"agentic","score":62.33,"unit":"percent","family":"pinchBench","index":21.07528113472792,"aboveFit":false,"run":null,"source":"Kilo Code","sourceUrl":"https://pinchbench.com/about","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"tau2Bench","benchmarkName":"τ²-Bench Tool-Agent-User Evaluation","category":"agentic","score":57.5,"unit":"percent","family":"tau2-bench","index":40.94489408083121,"aboveFit":true,"run":null,"source":"Victor Barres, Honghua Dong, Soham Ray, Xujie Si, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2506.07982","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"clawEval","benchmarkName":"Claw-Eval","category":"agentic","score":57.16,"unit":"percent","family":"clawEval","index":47.76835838417153,"aboveFit":true,"run":null,"source":"Bowen Ye, Rang Li, Qibin Yang, Yuanxin Liu, Linli Yao, Hanglong Lv, Zhihui Xie, Chenxin An, Lei Li, Lingpeng Kong, Qi Liu, Zhifang Sui, Tong Yang","sourceUrl":"https://arxiv.org/abs/2604.06132","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"bfclV4","benchmarkName":"Berkeley Function Calling Leaderboard v4","category":"agentic","score":47.68,"unit":"percent","family":"bfclV4","index":29.418551953594744,"aboveFit":false,"run":null,"source":"Shishir G. Patil et al.","sourceUrl":"https://gorilla.cs.berkeley.edu/leaderboard.html","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gpqaDiamond","benchmarkName":"GPQA Diamond","category":"knowledge","score":44.41,"unit":"percent","family":"gpqa-diamond","index":17.940739231037178,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aime2026","benchmarkName":"AIME 2026","category":"math","score":35.05,"unit":"percent","family":"aime2026","index":10.524891953592032,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"longBenchV2","benchmarkName":"LongBench v2","category":"reasoning","score":34.99,"unit":"percent","family":"longBenchV2","index":21.85775011167322,"aboveFit":false,"run":null,"source":"LongBench v2 authors","sourceUrl":"https://arxiv.org/abs/2412.15204","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"liveCodeBenchV6","benchmarkName":"LiveCodeBench v6","category":"coding","score":28.14,"unit":"percent","family":"liveCodeBenchV6","index":7.521619227871604,"aboveFit":false,"run":null,"source":"LiveCodeBench maintainers","sourceUrl":"https://github.com/LiveCodeBench/LiveCodeBench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"ifBench","benchmarkName":"Instruction Following Benchmark","category":"instruction","score":24.93,"unit":"percent","family":"ifbench-family","index":12.320979019764394,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":85,"benchmarkCount":9,"categoryCount":6,"confidence":"low","state":"available"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"mistral-small-3-1","providerId":"mistral","slug":"mistral-small-3-1","name":"Mistral Small 3.1","provider":"Mistral","description":"Mistral Small 3.1 is a standard model from Mistral. 20 benchmarks count toward its index, in 7 categories.","logoUrl":null,"releaseDate":"2025-03-17","contextWindowTokens":128000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":24.21064981654258,"overallScoreRange":{"low":20.09492789692542,"high":28.326371736159743},"badges":[],"capabilityScores":{"agentic":26.739450085881984,"coding":25.588199403990206,"reasoning":29.723380596251914,"multimodal":28.77392214610024,"knowledge":25.69128965273831,"multilingual":null,"instruction":16.96549677546703,"math":16.731032807259133},"capabilityCounts":{"coding":3,"knowledge":6,"math":4,"multimodal":2,"instruction":1,"agentic":2,"reasoning":2},"benchmarks":[{"benchmarkId":"humaneval","benchmarkName":"Evaluating Large Language Models Trained on Code","category":"coding","score":88.41,"unit":"percent","family":"humaneval","index":28.94020219490884,"aboveFit":false,"run":null,"source":"Mark Chen, Jerry Tworek, Heewoo Jun, Qiming Yuan, Henrique Ponde de Oliveira Pinto, Jared Kaplan, Harri Edwards, Yuri Burda, Nicholas Joseph, Greg Brockman, Alex Ray, Raul Puri, Gretchen Krueger, Michael Petrov, Heidy Khlaaf, Girish Sastry, Pamela Mishkin, Brooke Chan, Scott Gray, Nick Ryder, Mikhail Pavlov, Alethea Power, Lukasz Kaiser, Mohammad Bavarian, Clemens Winter, Philippe Tillet, Felipe Petroski Such, Dave Cummings, Matthias Plappert, Fotios Chantzis, Elizabeth Barnes, Ariel Herbert-Voss, William Hebgen Guss, Alex Nichol, Alex Paino, Nikolas Tezak, Jie Tang, Igor Babuschkin, Suchir Balaji, Shantanu Jain, William Saunders, Christopher Hesse, Andrew N. Carr, Jan Leike, Josh Achiam, Vedant Misra, Evan Morikawa, Alec Radford, Matthew Knight, Miles Brundage, Mira Murati, Katie Mayer, Peter Welinder, Bob McGrew, Dario Amodei, Sam McCandlish, Ilya Sutskever, Wojciech Zaremba","sourceUrl":"https://arxiv.org/abs/2107.03374","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:mgsm:overall","benchmarkName":"MGSM","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":84.218,"unit":"percent","family":"vals-ai:mgsm:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mgsm","measuredAt":"2026-01-09","stderr":2.21,"costPerTest":0.000095,"latency":3.87},{"benchmarkId":"mmlu","benchmarkName":"Massive Multitask Language Understanding","category":"knowledge","score":80.62,"unit":"percent","family":"mmlu","index":27.791976609017862,"aboveFit":false,"run":null,"source":"Dan Hendrycks, Collin Burns, Steven Basart, Andy Zou, Mantas Mazeika, Dawn Song, Jacob Steinhardt","sourceUrl":"https://arxiv.org/abs/2009.03300","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:legal_bench:overall","benchmarkName":"LegalBench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":69.157,"unit":"percent","family":"vals-ai:legal_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","measuredAt":"2026-10-13","stderr":0.512,"costPerTest":null,"latency":0.583},{"benchmarkId":"vals-ai:medqa:overall","benchmarkName":"MedQA","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":69.1,"unit":"percent","family":"vals-ai:medqa:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medqa","measuredAt":"2026-04-16","stderr":0.422,"costPerTest":0.000102,"latency":3.634},{"benchmarkId":"vals-ai:math500:overall","benchmarkName":"MATH 500","category":"math","domain":null,"eligible":true,"ranked":true,"score":68.4,"unit":"percent","family":"math-500","index":16.409044986581073,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/math500","measuredAt":"2026-01-09","stderr":2.079,"costPerTest":0.000158,"latency":6.406},{"benchmarkId":"mmluPro","benchmarkName":"Massive Multitask Language Understanding Professional","category":"knowledge","score":66.76,"unit":"percent","family":"mmlu-pro","index":28.59033746035987,"aboveFit":false,"run":null,"source":"Yubo Wang, Xueguang Ma, Ge Zhang, Yuansheng Ni, Abhranil Chandra, Shiguang Guo, Weiming Ren, Aaran Arulraj, Xuan He, Ziyan Jiang, Tianle Li, Max Ku, Kai Wang, Alex Zhuang, Rongqi Fan, Xiang Yue, Wenhu Chen","sourceUrl":"https://arxiv.org/abs/2406.01574","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:mmlu_pro:overall","benchmarkName":"MMLU Pro","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":66.021,"unit":"percent","family":"mmlu-pro","index":27.659285916386057,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","measuredAt":"2026-09-01","stderr":0.455,"costPerTest":0.000203,"latency":3.602},{"benchmarkId":"mmmu","benchmarkName":"Massive Multi-discipline Multimodal Understanding","category":"multimodal","score":64,"unit":"percent","family":"mmmu","index":31.25267708549307,"aboveFit":false,"run":null,"source":"MMMU authors","sourceUrl":"https://arxiv.org/abs/2401.05508","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:mortgage_tax:overall","benchmarkName":"MortgageTax","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":61.208,"unit":"percent","family":"vals-ai:mortgage_tax:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mortgage_tax","measuredAt":"2026-09-01","stderr":0.956,"costPerTest":0.000488,"latency":8.313},{"benchmarkId":"vals-ai:mmmu:overall","benchmarkName":"MMMU Pro","category":"multimodal","domain":null,"eligible":true,"ranked":true,"score":60.081,"unit":"percent","family":"mmmu-pro","index":33.342965328460245,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmmu","measuredAt":"2026-09-01","stderr":1.177,"costPerTest":0.000265,"latency":13.401},{"benchmarkId":"vals-ai:tax_eval_v2:overall","benchmarkName":"TaxEval v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":58.3,"unit":"percent","family":"vals-ai:tax_eval_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","measuredAt":"2026-09-01","stderr":0.968,"costPerTest":0.000184,"latency":7.952},{"benchmarkId":"mmmuPro","benchmarkName":"Massive Multi-discipline Multimodal Understanding Pro","category":"multimodal","score":49.25,"unit":"percent","family":"mmmu-pro","index":19.24736908495457,"aboveFit":false,"run":null,"source":"MMMU-Pro authors","sourceUrl":"https://arxiv.org/abs/2409.02813","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-math-level-5:overall","benchmarkName":"MATH level 5","category":"math","domain":null,"eligible":true,"ranked":true,"score":46.77114803625378,"unit":"percent","family":"epoch-ai:epoch-math-level-5:overall","index":21.717247792143564,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.1995700377965737,"costPerTest":null,"latency":null},{"benchmarkId":"gpqa","benchmarkName":"Graduate-Level Google-Proof Q&A","category":"knowledge","score":45.96,"unit":"percent","family":"gpqa-diamond","index":19.362356877162853,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":45.4,"unit":"percent","family":"gpqa-diamond","index":18.848740179207773,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":44.696969696969695,"unit":"percent","family":"gpqa-diamond","index":18.2039399956278,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.348827340432996,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:gpqa:overall","benchmarkName":"GPQA Diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":44.192,"unit":"percent","family":"gpqa-diamond","index":17.74079558790467,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","measuredAt":"2026-09-01","stderr":2.473,"costPerTest":0.000278,"latency":8.824},{"benchmarkId":"vals-ai:corp_fin_v2:overall","benchmarkName":"CorpFin v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":44.173,"unit":"percent","family":"vals-ai:corp_fin_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","measuredAt":"2026-08-12","stderr":0.972,"costPerTest":0.004408,"latency":11.291},{"benchmarkId":"vals-ai:lcb:overall","benchmarkName":"LiveCodeBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":31.815,"unit":"percent","family":"livecodebench","index":15.741983704220914,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","measuredAt":"2026-09-01","stderr":1.109,"costPerTest":0.000147,"latency":4.007},{"benchmarkId":"aaIfBench","benchmarkName":"Artificial Analysis IFBench","category":"instruction","score":29.9,"unit":"percent","family":"ifbench-family","index":16.96549677546703,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaSciCode","benchmarkName":"Artificial Analysis SciCode","category":"coding","score":27.8,"unit":"percent","family":"scicode-family","index":32.08241231284086,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/scicode","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"tau2Bench","benchmarkName":"τ²-Bench Tool-Agent-User Evaluation","category":"agentic","score":25.1,"unit":"percent","family":"tau2-bench","index":19.44512525457118,"aboveFit":false,"run":null,"source":"Victor Barres, Honghua Dong, Soham Ray, Xujie Si, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2506.07982","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":22.3,"unit":"percent","family":"lcr","index":22.597645179730925,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"omniscienceAccuracy","benchmarkName":"Artificial Analysis Omniscience Accuracy","category":"knowledge","score":15.1,"unit":"percent","family":"omniscienceAccuracy","index":29.71981582592358,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"simpleQa","benchmarkName":"Measuring Short-Form Factuality in Large Language Models","category":"knowledge","score":10.43,"unit":"percent","family":"simpleQa","index":21.283990588756087,"aboveFit":false,"run":null,"source":"Jason Wei, Najoung Kim, Hyung Won Chung, Yu-An Chung, Siddhartha Papay, Yifeng Lu, Hannaneh Hajishirzi, Luke Zettlemoyer","sourceUrl":"https://arxiv.org/abs/2411.04368","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":4.861111111111111,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":11.747256376656125,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.529498571765449,"costPerTest":null,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":4.3,"unit":"percent","family":"hle","index":28.700803116941593,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:aime:overall","benchmarkName":"AIME","category":"math","domain":null,"eligible":true,"ranked":true,"score":3.542,"unit":"percent","family":"vals-ai:aime:overall","index":17.050582073655782,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/aime","measuredAt":"2026-04-16","stderr":0.208,"costPerTest":0.000293,"latency":11.675},{"benchmarkId":"gdpvalAaNormalized","benchmarkName":"GDPval-AA normalized","category":"agentic","score":0,"unit":"percent","family":"gdpvalAaNormalized","index":34.03377491719279,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":0,"unit":"percent","family":"critpt","index":36.8491160127729,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":28,"timeToFirstToken":0.98,"source":"BenchLM","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":95,"benchmarkCount":20,"categoryCount":7,"confidence":"medium","state":"available"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"claude-3-5-haiku","providerId":"anthropic","slug":"claude-3-5-haiku","name":"Claude 3.5 Haiku","provider":"Anthropic","description":"Claude 3.5 Haiku is a standard model from Anthropic. 22 benchmarks count toward its index, in 8 categories.","logoUrl":null,"releaseDate":"2024-10-22","contextWindowTokens":200000,"modalities":["text"],"openWeights":false,"details":null,"reasoning":false,"overallScore":24.050022729025976,"overallScoreRange":{"low":20.602567145555756,"high":27.497478312496195},"badges":[],"capabilityScores":{"agentic":26.573556807901586,"coding":27.980918607377177,"reasoning":31.438581910216065,"multimodal":14.497214317269616,"knowledge":24.391272295427534,"multilingual":31.054177500929317,"instruction":32.59923831105009,"math":16.240995014119065},"capabilityCounts":{"coding":3,"instruction":2,"multilingual":1,"knowledge":5,"math":6,"multimodal":1,"reasoning":2,"agentic":2},"benchmarks":[{"benchmarkId":"humaneval","benchmarkName":"Evaluating Large Language Models Trained on Code","category":"coding","score":88.1,"unit":"percent","family":"humaneval","index":28.645131293806386,"aboveFit":false,"run":null,"source":"Mark Chen, Jerry Tworek, Heewoo Jun, Qiming Yuan, Henrique Ponde de Oliveira Pinto, Jared Kaplan, Harri Edwards, Yuri Burda, Nicholas Joseph, Greg Brockman, Alex Ray, Raul Puri, Gretchen Krueger, Michael Petrov, Heidy Khlaaf, Girish Sastry, Pamela Mishkin, Brooke Chan, Scott Gray, Nick Ryder, Mikhail Pavlov, Alethea Power, Lukasz Kaiser, Mohammad Bavarian, Clemens Winter, Philippe Tillet, Felipe Petroski Such, Dave Cummings, Matthias Plappert, Fotios Chantzis, Elizabeth Barnes, Ariel Herbert-Voss, William Hebgen Guss, Alex Nichol, Alex Paino, Nikolas Tezak, Jie Tang, Igor Babuschkin, Suchir Balaji, Shantanu Jain, William Saunders, Christopher Hesse, Andrew N. Carr, Jan Leike, Josh Achiam, Vedant Misra, Evan Morikawa, Alec Radford, Matthew Knight, Miles Brundage, Mira Murati, Katie Mayer, Peter Welinder, Bob McGrew, Dario Amodei, Sam McCandlish, Ilya Sutskever, Wojciech Zaremba","sourceUrl":"https://arxiv.org/abs/2107.03374","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"bbh","benchmarkName":"BIG-Bench Hard","category":"reasoning","score":86.6,"unit":"percent","family":"bbh","index":null,"aboveFit":false,"run":null,"source":"Mirac Suzgun, Nathan Scales, Nathanael Schärli, Sebastian Gehrmann, Yi Tay, Hyung Won Chung, Aakanksha Chowdhery, Quoc V. Le, Ed H. Chi, Denny Zhou, Jason Wei","sourceUrl":"https://arxiv.org/abs/2210.09261","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"ifeval","benchmarkName":"Instruction-Following Eval","category":"instruction","score":85.9,"unit":"percent","family":"ifeval","index":36.17779291533256,"aboveFit":false,"run":null,"source":"Jeffrey Zhou, Tianjian Lu, Swaroop Mishra, Siddhartha Brahma, Sujoy Basu, Yi Luan, Denny Zhou, Le Hou","sourceUrl":"https://arxiv.org/abs/2311.07911","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mgsm","benchmarkName":"Multilingual Grade School Math","category":"multilingual","score":85.6,"unit":"percent","family":"mgsm","index":31.054177500929317,"aboveFit":false,"run":null,"source":"Freda Shi, Mirac Suzgun, Markus Freitag, Xuezhi Wang, Suraj Srivats, Soroush Vosoughi, Hyung Won Chung, Yi Tay, Sebastian Ruder, Denny Zhou, Dipanjan Das, Jason Wei","sourceUrl":"https://arxiv.org/abs/2210.03057","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:mgsm:overall","benchmarkName":"MGSM","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":84.618,"unit":"percent","family":"vals-ai:mgsm:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mgsm","measuredAt":"2026-01-09","stderr":2.203,"costPerTest":0.001116,"latency":3.738},{"benchmarkId":"drop","benchmarkName":"Discrete Reasoning Over Paragraphs","category":"reasoning","score":83.1,"unit":"percent","family":"drop","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmlu","benchmarkName":"Massive Multitask Language Understanding","category":"knowledge","score":80.3,"unit":"percent","family":"mmlu","index":27.409900314731992,"aboveFit":false,"run":null,"source":"Dan Hendrycks, Collin Burns, Steven Basart, Andy Zou, Mantas Mazeika, Dawn Song, Jacob Steinhardt","sourceUrl":"https://arxiv.org/abs/2009.03300","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:legal_bench:overall","benchmarkName":"LegalBench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":70.331,"unit":"percent","family":"vals-ai:legal_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","measuredAt":"2026-10-13","stderr":0.64,"costPerTest":null,"latency":1.066},{"benchmarkId":"mmluPro","benchmarkName":"Massive Multitask Language Understanding Professional","category":"knowledge","score":65,"unit":"percent","family":"mmlu-pro","index":26.37294812694457,"aboveFit":false,"run":null,"source":"Yubo Wang, Xueguang Ma, Ge Zhang, Yuansheng Ni, Abhranil Chandra, Shiguang Guo, Weiming Ren, Aaran Arulraj, Xuan He, Ziyan Jiang, Tianle Li, Max Ku, Kai Wang, Alex Zhuang, Rongqi Fan, Xiang Yue, Wenhu Chen","sourceUrl":"https://arxiv.org/abs/2406.01574","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:math500:overall","benchmarkName":"MATH 500","category":"math","domain":null,"eligible":true,"ranked":true,"score":64.2,"unit":"percent","family":"math-500","index":11.970660494375565,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/math500","measuredAt":"2026-01-09","stderr":4.188,"costPerTest":0.001498,"latency":5.128},{"benchmarkId":"vals-ai:mmlu_pro:overall","benchmarkName":"MMLU Pro","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":64.117,"unit":"percent","family":"mmlu-pro","index":25.260473819327697,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","measuredAt":"2026-09-01","stderr":0.46,"costPerTest":0.002224,"latency":5.792},{"benchmarkId":"vals-ai:tax_eval_v2:overall","benchmarkName":"TaxEval v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":57.359,"unit":"percent","family":"vals-ai:tax_eval_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","measuredAt":"2026-09-01","stderr":0.976,"costPerTest":0.001317,"latency":4.987},{"benchmarkId":"vals-ai:corp_fin_v2:overall","benchmarkName":"CorpFin v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":50.816,"unit":"percent","family":"vals-ai:corp_fin_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","measuredAt":"2026-08-12","stderr":0.981,"costPerTest":0.063315,"latency":0.039},{"benchmarkId":"epoch-ai:epoch-math-level-5:overall","benchmarkName":"MATH level 5","category":"math","domain":null,"eligible":true,"ranked":true,"score":46.355740181268885,"unit":"percent","family":"epoch-ai:epoch-math-level-5:overall","index":21.51546119243969,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.161172810552623,"costPerTest":null,"latency":null},{"benchmarkId":"aaMmmuPro","benchmarkName":"Artificial Analysis MMMU-Pro","category":"multimodal","score":45.6,"unit":"percent","family":"mmmu-pro","index":14.497214317269616,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmmu-pro","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaIfBench","benchmarkName":"Artificial Analysis IFBench","category":"instruction","score":42.8,"unit":"percent","family":"ifbench-family","index":29.020683706767624,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:lcb:overall","benchmarkName":"LiveCodeBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":41.918,"unit":"percent","family":"livecodebench","index":24.124859340835588,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","measuredAt":"2026-09-01","stderr":1.129,"costPerTest":0.003064,"latency":10.581},{"benchmarkId":"gpqa","benchmarkName":"Graduate-Level Google-Proof Q&A","category":"knowledge","score":41.6,"unit":"percent","family":"gpqa-diamond","index":15.363484014512586,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":40.8,"unit":"percent","family":"gpqa-diamond","index":14.629745874576756,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"sweVerified","benchmarkName":"Software Engineering Benchmark Verified","category":"coding","score":40.6,"unit":"percent","family":"swe-bench-verified","index":31.172765187489542,"aboveFit":false,"run":null,"source":"Carlos E. Jimenez, John Yang, Alexander Wettig, Shunyu Yao, Kexin Pei, Ofir Press, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2310.06770","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":38.131313131313135,"unit":"percent","family":"gpqa-diamond","index":12.182099200699923,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.6351515390511557,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:gpqa:overall","benchmarkName":"GPQA Diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":37.879,"unit":"percent","family":"gpqa-diamond","index":11.950684491136066,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","measuredAt":"2026-09-01","stderr":2.436,"costPerTest":0.00222,"latency":6.56},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":27.3,"unit":"percent","family":"lcr","index":26.028047807659227,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"tau2Bench","benchmarkName":"τ²-Bench Tool-Agent-User Evaluation","category":"agentic","score":24.6,"unit":"percent","family":"tau2-bench","index":19.113338698610384,"aboveFit":false,"run":null,"source":"Victor Barres, Honghua Dong, Soham Ray, Xujie Si, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2506.07982","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"omniscienceAccuracy","benchmarkName":"Artificial Analysis Omniscience Accuracy","category":"knowledge","score":13.2,"unit":"percent","family":"omniscienceAccuracy","index":27.325329456210458,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aime2024","benchmarkName":"American Invitational Mathematics Examination 2024","category":"math","score":5.3,"unit":"percent","family":"aime2024","index":5.357991990791653,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":4.305555555555555,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":11.424037462359976,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.6237638827815495,"costPerTest":null,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":3.6,"unit":"percent","family":"hle","index":27.998498195420737,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:aime:overall","benchmarkName":"AIME","category":"math","domain":null,"eligible":true,"ranked":true,"score":3.333,"unit":"percent","family":"vals-ai:aime:overall","index":16.959239509724924,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/aime","measuredAt":"2026-04-16","stderr":0.315,"costPerTest":0.002432,"latency":9.055},{"benchmarkId":"epoch-ai:epoch-frontiermath-2025-02-28-private:overall","benchmarkName":"FrontierMath-2025-02-28-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":0.3448275862068966,"unit":"percent","family":"epoch-ai:epoch-frontiermath-2025-02-28-private:overall","index":30.21857943502259,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0.3448275862068965,"costPerTest":null,"latency":null},{"benchmarkId":"gdpvalAaNormalized","benchmarkName":"GDPval-AA normalized","category":"agentic","score":0,"unit":"percent","family":"gdpvalAaNormalized","index":34.03377491719279,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":0,"unit":"percent","family":"critpt","index":36.8491160127729,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-2025-02-28-public:overall","benchmarkName":"FrontierMath-2025-02-28-Public","category":"math","domain":null,"eligible":false,"ranked":true,"score":0,"unit":"percent","family":"epoch-ai:epoch-frontiermath-2025-02-28-public:overall","index":null,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":{"elo":1255.3919863941037,"eloUpper":1258.6247449626612,"eloLower":1252.1592278255464,"votes":69014,"rank":298,"rankUpper":null,"rankLower":null,"arenaName":"Claude 3.5 Haiku-20241022","organization":"anthropic","updatedAt":"2026-11-09"},"quality":{"coveragePercent":100,"benchmarkCount":22,"categoryCount":8,"confidence":"high","state":"available"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"commanda032025","providerId":"cohere","slug":"cohere-command-a","name":"Command A","provider":"Cohere","description":"Command A is a model from Cohere. 8 benchmarks count toward its index, in 4 categories.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":23.94692082782911,"overallScoreRange":{"low":14.729962501935873,"high":33.16387915372235},"badges":[],"capabilityScores":{"agentic":25.322561401565608,"coding":12.712248891019136,"reasoning":null,"multimodal":null,"knowledge":26.653693316358293,"multilingual":null,"instruction":null,"math":22.990728078803222},"capabilityCounts":{"math":2,"knowledge":2,"coding":2,"agentic":2},"benchmarks":[{"benchmarkId":"vals-ai:mgsm:overall","benchmarkName":"MGSM","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":85.709,"unit":"percent","family":"vals-ai:mgsm:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mgsm","measuredAt":"2026-01-09","stderr":2.016,"costPerTest":0.003732,"latency":8.36},{"benchmarkId":"vals-ai:medqa:overall","benchmarkName":"MedQA","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":80.55,"unit":"percent","family":"vals-ai:medqa:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medqa","measuredAt":"2026-04-16","stderr":0.361,"costPerTest":0.003459,"latency":2.908},{"benchmarkId":"vals-ai:legal_bench:overall","benchmarkName":"LegalBench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":79.697,"unit":"percent","family":"vals-ai:legal_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","measuredAt":"2026-10-13","stderr":0.422,"costPerTest":null,"latency":0.797},{"benchmarkId":"vals-ai:math500:overall","benchmarkName":"MATH 500","category":"math","domain":null,"eligible":true,"ranked":true,"score":76.2,"unit":"percent","family":"math-500","index":24.651759043534135,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/math500","measuredAt":"2026-01-09","stderr":1.904,"costPerTest":0.005983,"latency":8.665},{"benchmarkId":"vals-ai:mmlu_pro:overall","benchmarkName":"MMLU Pro","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":69.172,"unit":"percent","family":"mmlu-pro","index":31.629168751381275,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","measuredAt":"2026-09-01","stderr":0.463,"costPerTest":0.007902,"latency":9.853},{"benchmarkId":"vals-ai:case_law_v2:overall","benchmarkName":"CaseLaw v2","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":64.523,"unit":"percent","family":"vals-ai:case_law_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/case_law_v2","measuredAt":"2026-05-04","stderr":1.088,"costPerTest":0.098244,"latency":41.555},{"benchmarkId":"vals-ai:tax_eval_v2:overall","benchmarkName":"TaxEval v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":61.366,"unit":"percent","family":"vals-ai:tax_eval_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","measuredAt":"2026-09-01","stderr":0.952,"costPerTest":0.006484,"latency":11.214},{"benchmarkId":"vals-ai:gpqa:overall","benchmarkName":"GPQA Diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":48.485,"unit":"percent","family":"gpqa-diamond","index":21.678217881335307,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","measuredAt":"2026-09-01","stderr":2.276,"costPerTest":0.010094,"latency":11.49},{"benchmarkId":"vals-ai:corp_fin_v2:overall","benchmarkName":"CorpFin v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":45.96,"unit":"percent","family":"vals-ai:corp_fin_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","measuredAt":"2026-08-12","stderr":0.972,"costPerTest":0.206138,"latency":14.146},{"benchmarkId":"vals-ai:lcb:overall","benchmarkName":"LiveCodeBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":35.071,"unit":"percent","family":"livecodebench","index":18.44362114585383,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","measuredAt":"2026-09-01","stderr":1.086,"costPerTest":0.006568,"latency":11.746},{"benchmarkId":"vals-ai:aime:overall","benchmarkName":"AIME","category":"math","domain":null,"eligible":true,"ranked":true,"score":13.333,"unit":"percent","family":"vals-ai:aime:overall","index":21.329697114072307,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/aime","measuredAt":"2026-04-16","stderr":0.945,"costPerTest":0.011732,"latency":23.345},{"benchmarkId":"vals-ai:swebench:overall","benchmarkName":"SWE-bench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":7.8,"unit":"percent","family":"swe-bench-verified","index":6.980876636184448,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/swebench","measuredAt":"2026-09-01","stderr":1.201,"costPerTest":0.21024,"latency":227.531},{"benchmarkId":"vals-ai:terminal-bench:overall","benchmarkName":"Terminal-Bench 1.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":6.25,"unit":"percent","family":"vals-ai:terminal-bench:overall","index":22.74091429596053,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench","measuredAt":"2026-01-12","stderr":2.706,"costPerTest":2.5047,"latency":287.604},{"benchmarkId":"vals-ai:finance_agent:overall","benchmarkName":"Finance Agent v1.1","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":4.226,"unit":"percent","family":"vals-ai:finance_agent:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/finance_agent","measuredAt":"2026-06-04","stderr":0.966,"costPerTest":2.339409,"latency":573.215},{"benchmarkId":"vals-ai:terminal-bench-2:overall","benchmarkName":"Terminal-Bench 2.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":2.247,"unit":"percent","family":"vals-ai:terminal-bench-2:overall","index":27.904208507170686,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2","measuredAt":"2026-06-04","stderr":1.58,"costPerTest":2.436871,"latency":760.674}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":{"elo":1330.9530528351497,"eloUpper":1334.4067776628503,"eloLower":1327.4993280074493,"votes":55679,"rank":230,"rankUpper":null,"rankLower":null,"arenaName":"Command A-03-2025","organization":"cohere","updatedAt":"2026-11-09"},"quality":{"coveragePercent":65,"benchmarkCount":8,"categoryCount":4,"confidence":"low","state":"available"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"claude-3-opus","providerId":"anthropic","slug":"claude-3-opus","name":"Claude 3 Opus","provider":"Anthropic","description":"Claude 3 Opus is a standard model from Anthropic. 8 benchmarks count toward its index, in 4 categories.","logoUrl":null,"releaseDate":"2024-03-01","contextWindowTokens":200000,"modalities":["text"],"openWeights":false,"details":null,"reasoning":false,"overallScore":23.297143305922194,"overallScoreRange":{"low":13.580184980028957,"high":33.014101631815436},"badges":[],"capabilityScores":{"agentic":null,"coding":9.492997366339502,"reasoning":37.02073650573931,"multimodal":null,"knowledge":27.62410534365568,"multilingual":null,"instruction":null,"math":14.435508650696775},"capabilityCounts":{"knowledge":3,"math":2,"coding":2,"reasoning":1},"benchmarks":[{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":48.9,"unit":"percent","family":"gpqa-diamond","index":22.058844541427018,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":47.159090909090914,"unit":"percent","family":"gpqa-diamond","index":20.46213029372576,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.6395393956533515,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-math-level-5:overall","benchmarkName":"MATH level 5","category":"math","domain":null,"eligible":true,"ranked":true,"score":37.48111782477341,"unit":"percent","family":"epoch-ai:epoch-math-level-5:overall","index":17.204565653311462,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.058728426376393,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-simpleqa-verified:overall","benchmarkName":"SimpleQA Verified","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":12.6,"unit":"percent","family":"epoch-ai:epoch-simpleqa-verified:overall","index":34.41596461399376,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.0499249222407965,"costPerTest":null,"latency":null},{"benchmarkId":"swe-bench:swe-bench-verified:overall","benchmarkName":"SWE-bench Verified","category":"coding","domain":null,"eligible":true,"ranked":true,"score":11.4,"unit":"percent","family":"swe-bench-verified","index":9.636083916205742,"aboveFit":false,"run":"SWE-agent","source":"SWE-bench team","sourceUrl":"https://www.swebench.com/","measuredAt":"2026-02-26","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"swebench-experiments:swe-bench-verified:overall","benchmarkName":"SWE-bench Verified","category":"coding","domain":null,"eligible":true,"ranked":true,"score":11.4,"unit":"percent","family":"swe-bench-verified","index":9.636083916205742,"aboveFit":false,"run":"RAG baseline","source":"SWE-bench team","sourceUrl":"https://github.com/SWE-bench/experiments/tree/main/evaluation/verified","measuredAt":"2026-09-01","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"swe-bench:swe-bench-lite:overall","benchmarkName":"SWE-bench Lite","category":"coding","domain":null,"eligible":true,"ranked":true,"score":8,"unit":"percent","family":"swe-bench-lite","index":10.840824978269652,"aboveFit":false,"run":"SWE-agent","source":"SWE-bench team","sourceUrl":"https://www.swebench.com/","measuredAt":"2025-09-11","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"swebench-experiments:swe-bench-test:overall","benchmarkName":"SWE-bench (full test split)","category":"coding","domain":null,"eligible":true,"ranked":true,"score":7.149084568439408,"unit":"percent","family":"swebench-experiments:swe-bench-test:overall","index":null,"aboveFit":false,"run":"SWE-agent","source":"SWE-bench team","sourceUrl":"https://github.com/SWE-bench/experiments/tree/main/evaluation/test","measuredAt":"2025-12-19","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":4.722222222222222,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":11.666451648082095,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.9579258477123922,"costPerTest":null,"latency":null},{"benchmarkId":"swebench-experiments:swe-bench-lite:overall","benchmarkName":"SWE-bench Lite","category":"coding","domain":null,"eligible":true,"ranked":true,"score":4.333333333333334,"unit":"percent","family":"swe-bench-lite","index":7.8589966546768935,"aboveFit":false,"run":"RAG baseline","source":"SWE-bench team","sourceUrl":"https://github.com/SWE-bench/experiments/tree/main/evaluation/lite","measuredAt":"2025-09-11","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-chess-puzzles:overall","benchmarkName":"Chess Puzzles","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":4,"unit":"percent","family":"epoch-ai:epoch-chess-puzzles:overall","index":37.02073650573931,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.969463855669324,"costPerTest":null,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":2.8,"unit":"percent","family":"hle","index":27.195863999396902,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":15,"outputPerMillion":75,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":{"elo":1262.011954709697,"eloUpper":1265.0516830758052,"eloLower":1258.9722263435885,"votes":194909,"rank":294,"rankUpper":null,"rankLower":null,"arenaName":"Claude 3 Opus-20240229","organization":"anthropic","updatedAt":"2026-11-09"},"quality":{"coveragePercent":60,"benchmarkCount":8,"categoryCount":4,"confidence":"low","state":"available"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:swe-bench-team","name":"SWE-bench team","url":"https://www.swebench.com","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"phi-4","providerId":"microsoft","slug":"phi-4","name":"Phi-4","provider":"Microsoft","description":"Phi-4 is a standard model from Microsoft. 12 benchmarks count toward its index, in 7 categories.","logoUrl":null,"releaseDate":"2024-12-12","contextWindowTokens":16384,"modalities":["text"],"openWeights":true,"details":{"apiId":"microsoft/phi-4","maxOutputTokens":14745,"knowledgeCutoff":"2024-06-30","retiresAt":null,"tools":false,"structuredOutput":true,"cacheReadPerMillion":null,"weightsUrl":"https://huggingface.co/microsoft/phi-4"},"reasoning":false,"overallScore":22.37023484405334,"overallScoreRange":{"low":16.702357917655213,"high":28.038111770451465},"badges":[],"capabilityScores":{"agentic":5,"coding":23.410002403279098,"reasoning":22.073582735971804,"multimodal":null,"knowledge":29.881497582801025,"multilingual":24.03492218186418,"instruction":10.984628840558216,"math":23.729792237749287},"capabilityCounts":{"knowledge":3,"coding":1,"multilingual":2,"math":2,"instruction":1,"agentic":1,"reasoning":2},"benchmarks":[{"benchmarkId":"mmlu","benchmarkName":"Massive Multitask Language Understanding","category":"knowledge","score":84.8,"unit":"percent","family":"mmlu","index":32.782848203126925,"aboveFit":false,"run":null,"source":"Dan Hendrycks, Collin Burns, Steven Basart, Andy Zou, Mantas Mazeika, Dawn Song, Jacob Steinhardt","sourceUrl":"https://arxiv.org/abs/2009.03300","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"humaneval","benchmarkName":"Evaluating Large Language Models Trained on Code","category":"coding","score":82.6,"unit":"percent","family":"humaneval","index":23.410002403279098,"aboveFit":false,"run":null,"source":"Mark Chen, Jerry Tworek, Heewoo Jun, Qiming Yuan, Henrique Ponde de Oliveira Pinto, Jared Kaplan, Harri Edwards, Yuri Burda, Nicholas Joseph, Greg Brockman, Alex Ray, Raul Puri, Gretchen Krueger, Michael Petrov, Heidy Khlaaf, Girish Sastry, Pamela Mishkin, Brooke Chan, Scott Gray, Nick Ryder, Mikhail Pavlov, Alethea Power, Lukasz Kaiser, Mohammad Bavarian, Clemens Winter, Philippe Tillet, Felipe Petroski Such, Dave Cummings, Matthias Plappert, Fotios Chantzis, Elizabeth Barnes, Ariel Herbert-Voss, William Hebgen Guss, Alex Nichol, Alex Paino, Nikolas Tezak, Jie Tang, Igor Babuschkin, Suchir Balaji, Shantanu Jain, William Saunders, Christopher Hesse, Andrew N. Carr, Jan Leike, Josh Achiam, Vedant Misra, Evan Morikawa, Alec Radford, Matthew Knight, Miles Brundage, Mira Murati, Katie Mayer, Peter Welinder, Bob McGrew, Dario Amodei, Sam McCandlish, Ilya Sutskever, Wojciech Zaremba","sourceUrl":"https://arxiv.org/abs/2107.03374","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mgsm","benchmarkName":"Multilingual Grade School Math","category":"multilingual","score":80.6,"unit":"percent","family":"mgsm","index":28.298066263595505,"aboveFit":false,"run":null,"source":"Freda Shi, Mirac Suzgun, Markus Freitag, Xuezhi Wang, Suraj Srivats, Soroush Vosoughi, Hyung Won Chung, Yi Tay, Sebastian Ruder, Denny Zhou, Dipanjan Das, Jason Wei","sourceUrl":"https://arxiv.org/abs/2210.03057","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-math-level-5:overall","benchmarkName":"MATH level 5","category":"math","domain":null,"eligible":true,"ranked":true,"score":64.93580060422961,"unit":"percent","family":"epoch-ai:epoch-math-level-5:overall","index":30.540825470103904,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.0656871384084827,"costPerTest":null,"latency":null},{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":57.5,"unit":"percent","family":"gpqa-diamond","index":29.94652954573717,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gpqa","benchmarkName":"Graduate-Level Google-Proof Q&A","category":"knowledge","score":56.1,"unit":"percent","family":"gpqa-diamond","index":28.662487800849473,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":56.060606060606055,"unit":"percent","family":"gpqa-diamond","index":28.626356756079897,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.590274664132679,"costPerTest":null,"latency":null},{"benchmarkId":"mmluProX","benchmarkName":"MMLU-ProX","category":"multilingual","score":49.9,"unit":"percent","family":"mmluProX","index":19.77177810013285,"aboveFit":false,"run":null,"source":"MMLU-ProX authors","sourceUrl":"https://arxiv.org/abs/2503.10497","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaIfBench","benchmarkName":"Artificial Analysis IFBench","category":"instruction","score":23.5,"unit":"percent","family":"ifbench-family","index":10.984628840558216,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":13.750000000000002,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":16.91875900539467,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":3.6980833829256348,"costPerTest":null,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":3.8,"unit":"percent","family":"hle","index":28.199156744426695,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"tau2Bench","benchmarkName":"τ²-Bench Tool-Agent-User Evaluation","category":"agentic","score":0,"unit":"percent","family":"tau2-bench","index":5,"aboveFit":false,"run":null,"source":"Victor Barres, Honghua Dong, Soham Ray, Xujie Si, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2506.07982","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":0,"unit":"percent","family":"lcr","index":7.298049459170706,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":0,"unit":"percent","family":"critpt","index":36.8491160127729,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":0.07,"outputPerMillion":0.14,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":93,"timeToFirstToken":0.278,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":{"elo":1216.694364221165,"eloUpper":1221.2786152169654,"eloLower":1212.1101132253643,"votes":24126,"rank":319,"rankUpper":null,"rankLower":null,"arenaName":"Phi-4","organization":"microsoft","updatedAt":"2026-11-09"},"quality":{"coveragePercent":90,"benchmarkCount":12,"categoryCount":7,"confidence":"medium","state":"available"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"claude-3-haiku","providerId":"anthropic","slug":"claude-3-haiku","name":"Claude 3 Haiku","provider":"Anthropic","description":"Claude 3 Haiku is a standard model from Anthropic. 11 benchmarks count toward its index, in 7 categories.","logoUrl":null,"releaseDate":"2024-03-01","contextWindowTokens":200000,"modalities":["text","image"],"openWeights":false,"details":{"apiId":"anthropic/claude-3-haiku","maxOutputTokens":4096,"knowledgeCutoff":"2023-08-31","retiresAt":null,"tools":true,"structuredOutput":false,"cacheReadPerMillion":0.03,"weightsUrl":null},"reasoning":false,"overallScore":21.966069123490676,"overallScoreRange":{"low":16.590630681668237,"high":27.341507565313115},"badges":[],"capabilityScores":{"agentic":16.790832806884758,"coding":31.172765187489542,"reasoning":31.5757980153332,"multimodal":5,"knowledge":24.12591525730434,"multilingual":null,"instruction":22.759462587409953,"math":8.097546276907131},"capabilityCounts":{"coding":1,"knowledge":3,"instruction":1,"multimodal":1,"reasoning":2,"agentic":1,"math":2},"benchmarks":[{"benchmarkId":"swebench-experiments:swe-bench-verified:overall","benchmarkName":"SWE-bench Verified","category":"coding","domain":null,"eligible":true,"ranked":true,"score":40.6,"unit":"percent","family":"swe-bench-verified","index":31.172765187489542,"aboveFit":false,"run":"Tools","source":"SWE-bench team","sourceUrl":"https://github.com/SWE-bench/experiments/tree/main/evaluation/verified","measuredAt":"2026-09-01","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":37.4,"unit":"percent","family":"gpqa-diamond","index":11.511358779849495,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":36.30050505050505,"unit":"percent","family":"gpqa-diamond","index":10.502932055960414,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.4180910672074973,"costPerTest":null,"latency":null},{"benchmarkId":"aaIfBench","benchmarkName":"Artificial Analysis IFBench","category":"instruction","score":36.1,"unit":"percent","family":"ifbench-family","index":22.759462587409953,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaMmmuPro","benchmarkName":"Artificial Analysis MMMU-Pro","category":"multimodal","score":30.8,"unit":"percent","family":"mmmu-pro","index":5,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmmu-pro","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":27.7,"unit":"percent","family":"lcr","index":26.302480017893497,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"tau2Bench","benchmarkName":"τ²-Bench Tool-Agent-User Evaluation","category":"agentic","score":21.1,"unit":"percent","family":"tau2-bench","index":16.790832806884758,"aboveFit":false,"run":null,"source":"Victor Barres, Honghua Dong, Soham Ray, Xujie Si, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2506.07982","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"omniscienceAccuracy","benchmarkName":"Artificial Analysis Omniscience Accuracy","category":"knowledge","score":17.6,"unit":"percent","family":"omniscienceAccuracy","index":32.87045578607243,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-math-level-5:overall","benchmarkName":"MATH level 5","category":"math","domain":null,"eligible":true,"ranked":true,"score":14.879154078549849,"unit":"percent","family":"epoch-ai:epoch-math-level-5:overall","index":6.225540205786999,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0.7229965913719499,"costPerTest":null,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":4.1,"unit":"percent","family":"hle","index":28.50014456793563,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":1.8055555555555554,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":9.969552348027257,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0.8098544298396251,"costPerTest":null,"latency":null},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":0,"unit":"percent","family":"critpt","index":36.8491160127729,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":0.25,"outputPerMillion":1.25,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":73,"timeToFirstToken":0.413,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":{"elo":1194.6880421675603,"eloUpper":1198.4226613552491,"eloLower":1190.9534229798714,"votes":117701,"rank":326,"rankUpper":null,"rankLower":null,"arenaName":"Claude 3 Haiku-20240307","organization":"anthropic","updatedAt":"2026-11-09"},"quality":{"coveragePercent":95,"benchmarkCount":11,"categoryCount":7,"confidence":"medium","state":"available"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:swe-bench-team","name":"SWE-bench team","url":"https://github.com","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gemma-4-e2b","providerId":"google","slug":"gemma-4-e2b","name":"Gemma 4 E2B","provider":"Google","description":"Gemma 4 E2B is a reasoning model from Google in the Gemma 4 family. 13 benchmarks count toward its index, in 7 categories.","logoUrl":null,"releaseDate":"2026-04-02","contextWindowTokens":128000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":true,"overallScore":21.93140634255123,"overallScoreRange":{"low":16.44665525607025,"high":27.41615742903221},"badges":[],"capabilityScores":{"agentic":25.31276789525053,"coding":20.5435855801439,"reasoning":25.584556988625796,"multimodal":16.77609761530813,"knowledge":22.0815107655463,"multilingual":5,"instruction":24.53503275558601,"math":null},"capabilityCounts":{"knowledge":3,"multimodal":2,"coding":1,"instruction":1,"multilingual":1,"agentic":2,"reasoning":3},"benchmarks":[{"benchmarkId":"mmluPro","benchmarkName":"Massive Multitask Language Understanding Professional","category":"knowledge","score":60,"unit":"percent","family":"mmlu-pro","index":20.07354661156021,"aboveFit":false,"run":null,"source":"Yubo Wang, Xueguang Ma, Ge Zhang, Yuansheng Ni, Abhranil Chandra, Shiguang Guo, Weiming Ren, Aaran Arulraj, Xuan He, Ziyan Jiang, Tianle Li, Max Ku, Kai Wang, Alex Zhuang, Rongqi Fan, Xiang Yue, Wenhu Chen","sourceUrl":"https://arxiv.org/abs/2406.01574","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"extractBenchLlamaIndex","benchmarkName":"ExtractBench: schema-guided enterprise document extraction (LlamaIndex leaderboard)","category":"multimodal","score":51.85,"unit":"percent","family":"extractBenchLlamaIndex","index":20.616675631489638,"aboveFit":false,"run":null,"source":"Boyang Zhang, Adrian Lyjak, Eli Stewart, Zhaoqi Li and Simon Suo (LlamaIndex)","sourceUrl":"https://arxiv.org/abs/2607.29677","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaMmmuPro","benchmarkName":"Artificial Analysis MMMU-Pro","category":"multimodal","score":44.6,"unit":"percent","family":"mmmu-pro","index":13.195802052150455,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmmu-pro","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmmuPro","benchmarkName":"Massive Multi-discipline Multimodal Understanding Pro","category":"multimodal","score":44.2,"unit":"percent","family":"mmmu-pro","index":12.675237146102788,"aboveFit":false,"run":null,"source":"MMMU-Pro authors","sourceUrl":"https://arxiv.org/abs/2409.02813","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"liveCodeBenchV6","benchmarkName":"LiveCodeBench v6","category":"coding","score":44,"unit":"percent","family":"liveCodeBenchV6","index":20.5435855801439,"aboveFit":false,"run":null,"source":"LiveCodeBench maintainers","sourceUrl":"https://github.com/LiveCodeBench/LiveCodeBench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gpqa","benchmarkName":"Graduate-Level Google-Proof Q&A","category":"knowledge","score":43.4,"unit":"percent","family":"gpqa-diamond","index":17.0143948293682,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":43.3,"unit":"percent","family":"gpqa-diamond","index":16.92267756187622,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaIfBench","benchmarkName":"Artificial Analysis IFBench","category":"instruction","score":38,"unit":"percent","family":"ifbench-family","index":24.53503275558601,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-french:overall","benchmarkName":"EuroEval French","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":24.323750000000004,"unit":"percent","family":"euroeval","index":5.221967473083261,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/french","measuredAt":null,"stderr":0.8250501878673806,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-dutch:overall","benchmarkName":"EuroEval Dutch","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":24.109999999999996,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/dutch","measuredAt":null,"stderr":0.7159514087932828,"costPerTest":null,"latency":null},{"benchmarkId":"tau2Bench","benchmarkName":"τ²-Bench Tool-Agent-User Evaluation","category":"agentic","score":20.8,"unit":"percent","family":"tau2-bench","index":16.591760873308274,"aboveFit":false,"run":null,"source":"Victor Barres, Honghua Dong, Soham Ray, Xujie Si, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2506.07982","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":20.17428571428572,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.9323396070408803,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-portuguese:overall","benchmarkName":"EuroEval Portuguese","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":19.525,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/portuguese","measuredAt":null,"stderr":0.8104753620561207,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":19.25375,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":0.9583930235034059,"costPerTest":null,"latency":null},{"benchmarkId":"mrcrv2","benchmarkName":"MRCRv2","category":"reasoning","score":19.1,"unit":"percent","family":"mrcrv2","index":21.423392926887534,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-italian:overall","benchmarkName":"EuroEval Italian","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":16.59125,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/italian","measuredAt":null,"stderr":0.8568118594534042,"costPerTest":null,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":16.3,"unit":"percent","family":"lcr","index":18.481162026216964,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-spanish:overall","benchmarkName":"EuroEval Spanish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":14.598749999999999,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/spanish","measuredAt":null,"stderr":0.8706077259592864,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":12.919999999999998,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.9348529296097863,"costPerTest":null,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":4.8,"unit":"percent","family":"hle","index":29.202449489456487,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gdpvalAaNormalized","benchmarkName":"GDPval-AA normalized","category":"agentic","score":0,"unit":"percent","family":"gdpvalAaNormalized","index":34.03377491719279,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":0,"unit":"percent","family":"critpt","index":36.8491160127729,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":90,"benchmarkCount":13,"categoryCount":7,"confidence":"medium","state":"available"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gemini-1-5-flash","providerId":"google","slug":"gemini-1-5-flash","name":"Gemini 1.5 Flash","provider":"Google","description":"Gemini 1.5 Flash is a standard model from Google. 10 benchmarks count toward its index, in 4 categories.","logoUrl":null,"releaseDate":"2024-05-14","contextWindowTokens":1000000,"modalities":["text"],"openWeights":false,"details":null,"reasoning":false,"overallScore":20.190960410770433,"overallScoreRange":{"low":10.577557434278162,"high":29.804363387262704},"badges":[],"capabilityScores":{"agentic":null,"coding":8.308480276195098,"reasoning":null,"multimodal":21.595348913578906,"knowledge":22.330267200583698,"multilingual":null,"instruction":null,"math":17.950328401176286},"capabilityCounts":{"multimodal":2,"knowledge":3,"math":4,"coding":1},"benchmarks":[{"benchmarkId":"mmmu","benchmarkName":"Massive Multi-discipline Multimodal Understanding","category":"multimodal","score":58.3,"unit":"percent","family":"mmmu","index":25.049529167554542,"aboveFit":false,"run":null,"source":"MMMU authors","sourceUrl":"https://arxiv.org/abs/2401.05508","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gpqa","benchmarkName":"Graduate-Level Google-Proof Q&A","category":"knowledge","score":50,"unit":"percent","family":"gpqa-diamond","index":23.06773448383878,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaMmmuPro","benchmarkName":"Artificial Analysis MMMU-Pro","category":"multimodal","score":48.4,"unit":"percent","family":"mmmu-pro","index":18.141168659603274,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmmu-pro","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":46.3,"unit":"percent","family":"gpqa-diamond","index":19.674195586635577,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":43.84469696969697,"unit":"percent","family":"gpqa-diamond","index":17.422258738593897,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.3852475604024477,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-math-level-5:overall","benchmarkName":"MATH level 5","category":"math","domain":null,"eligible":true,"ranked":true,"score":43.49509063444109,"unit":"percent","family":"epoch-ai:epoch-math-level-5:overall","index":20.125885289933464,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.120122127580226,"costPerTest":null,"latency":null},{"benchmarkId":"aime2025","benchmarkName":"American Invitational Mathematics Examination 2025","category":"math","score":14.7,"unit":"percent","family":"aime2025","index":6.983821259627888,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":10.069444444444445,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":14.77743369818262,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":4.111375571619442,"costPerTest":null,"latency":null},{"benchmarkId":"sweVerified","benchmarkName":"Software Engineering Benchmark Verified","category":"coding","score":9.6,"unit":"percent","family":"swe-bench-verified","index":8.308480276195098,"aboveFit":false,"run":null,"source":"Carlos E. Jimenez, John Yang, Alexander Wettig, Shunyu Yao, Kexin Pei, Ofir Press, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2310.06770","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"simpleQa","benchmarkName":"Measuring Short-Form Factuality in Large Language Models","category":"knowledge","score":8.6,"unit":"percent","family":"simpleQa","index":19.719424917706696,"aboveFit":false,"run":null,"source":"Jason Wei, Najoung Kim, Hyung Won Chung, Yu-An Chung, Siddhartha Papay, Yifeng Lu, Hannaneh Hajishirzi, Luke Zettlemoyer","sourceUrl":"https://arxiv.org/abs/2411.04368","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":3.2,"unit":"percent","family":"hle","index":27.597181097408818,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-2025-02-28-private:overall","benchmarkName":"FrontierMath-2025-02-28-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":0,"unit":"percent","family":"epoch-ai:epoch-frontiermath-2025-02-28-private:overall","index":29.914173356961165,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-2025-02-28-public:overall","benchmarkName":"FrontierMath-2025-02-28-Public","category":"math","domain":null,"eligible":false,"ranked":true,"score":0,"unit":"percent","family":"epoch-ai:epoch-frontiermath-2025-02-28-public:overall","index":null,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":55.00000000000001,"benchmarkCount":10,"categoryCount":4,"confidence":"low","state":"available"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"llama-3-1-8b","providerId":"meta","slug":"llama-3-1-8b","name":"Llama 3.1 8B","provider":"Meta","description":"Llama 3.1 8B is a standard model from Meta. 10 benchmarks count toward its index, in 5 categories.","logoUrl":null,"releaseDate":"2024-07-23","contextWindowTokens":128000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":17.933608617186014,"overallScoreRange":{"low":8.320205640693743,"high":27.547011593678285},"badges":[],"capabilityScores":{"agentic":null,"coding":13.891586238684027,"reasoning":null,"multimodal":null,"knowledge":14.431883758459541,"multilingual":24.760517744424828,"instruction":25.101349064065435,"math":9.999339934769907},"capabilityCounts":{"instruction":1,"knowledge":4,"coding":1,"multilingual":2,"math":2},"benchmarks":[{"benchmarkId":"gsm8k","benchmarkName":"Grade School Math 8K","category":"math","score":84.5,"unit":"percent","family":"gsm8k","index":null,"aboveFit":false,"run":null,"source":"Karl Cobbe and colleagues (OpenAI)","sourceUrl":"https://arxiv.org/abs/2110.14168","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"ifeval","benchmarkName":"Instruction-Following Eval","category":"instruction","score":80.4,"unit":"percent","family":"ifeval","index":25.101349064065435,"aboveFit":false,"run":null,"source":"Jeffrey Zhou, Tianjian Lu, Swaroop Mishra, Siddhartha Brahma, Sujoy Basu, Yi Luan, Denny Zhou, Le Hou","sourceUrl":"https://arxiv.org/abs/2311.07911","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmlu","benchmarkName":"Massive Multitask Language Understanding","category":"knowledge","score":73,"unit":"percent","family":"mmlu","index":18.693784851335774,"aboveFit":false,"run":null,"source":"Dan Hendrycks, Collin Burns, Steven Basart, Andy Zou, Mantas Mazeika, Dawn Song, Jacob Steinhardt","sourceUrl":"https://arxiv.org/abs/2009.03300","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"humaneval","benchmarkName":"Evaluating Large Language Models Trained on Code","category":"coding","score":72.6,"unit":"percent","family":"humaneval","index":13.891586238684027,"aboveFit":false,"run":null,"source":"Mark Chen, Jerry Tworek, Heewoo Jun, Qiming Yuan, Henrique Ponde de Oliveira Pinto, Jared Kaplan, Harri Edwards, Yuri Burda, Nicholas Joseph, Greg Brockman, Alex Ray, Raul Puri, Gretchen Krueger, Michael Petrov, Heidy Khlaaf, Girish Sastry, Pamela Mishkin, Brooke Chan, Scott Gray, Nick Ryder, Mikhail Pavlov, Alethea Power, Lukasz Kaiser, Mohammad Bavarian, Clemens Winter, Philippe Tillet, Felipe Petroski Such, Dave Cummings, Matthias Plappert, Fotios Chantzis, Elizabeth Barnes, Ariel Herbert-Voss, William Hebgen Guss, Alex Nichol, Alex Paino, Nikolas Tezak, Jie Tang, Igor Babuschkin, Suchir Balaji, Shantanu Jain, William Saunders, Christopher Hesse, Andrew N. Carr, Jan Leike, Josh Achiam, Vedant Misra, Evan Morikawa, Alec Radford, Matthew Knight, Miles Brundage, Mira Murati, Katie Mayer, Peter Welinder, Bob McGrew, Dario Amodei, Sam McCandlish, Ilya Sutskever, Wojciech Zaremba","sourceUrl":"https://arxiv.org/abs/2107.03374","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mgsm","benchmarkName":"Multilingual Grade School Math","category":"multilingual","score":68.9,"unit":"percent","family":"mgsm","index":21.8487659682344,"aboveFit":false,"run":null,"source":"Freda Shi, Mirac Suzgun, Markus Freitag, Xuezhi Wang, Suraj Srivats, Soroush Vosoughi, Hyung Won Chung, Yi Tay, Sebastian Ruder, Denny Zhou, Dipanjan Das, Jason Wei","sourceUrl":"https://arxiv.org/abs/2210.03057","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmluPro","benchmarkName":"Massive Multitask Language Understanding Professional","category":"knowledge","score":48.3,"unit":"percent","family":"mmlu-pro","index":5.332947065560802,"aboveFit":false,"run":null,"source":"Yubo Wang, Xueguang Ma, Ge Zhang, Yuansheng Ni, Abhranil Chandra, Shiguang Guo, Weiming Ren, Aaran Arulraj, Xuan He, Ziyan Jiang, Tianle Li, Max Ku, Kai Wang, Alex Zhuang, Rongqi Fan, Xiang Yue, Wenhu Chen","sourceUrl":"https://arxiv.org/abs/2406.01574","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-french:overall","benchmarkName":"EuroEval French","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":45.87125,"unit":"percent","family":"euroeval","index":34.4569670345833,"aboveFit":true,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/french","measuredAt":null,"stderr":0.5214478041952041,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-dutch:overall","benchmarkName":"EuroEval Dutch","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":43.795555555555566,"unit":"percent","family":"euroeval","index":31.64072727715709,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/dutch","measuredAt":null,"stderr":0.4099939776733252,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":41.387142857142855,"unit":"percent","family":"euroeval","index":28.373065405385113,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.37682074989445635,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-portuguese:overall","benchmarkName":"EuroEval Portuguese","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":41.14,"unit":"percent","family":"euroeval","index":28.03774941419359,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/portuguese","measuredAt":null,"stderr":0.5251220096320474,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":40.60124999999999,"unit":"percent","family":"euroeval","index":27.30678962703692,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":0.38407356066253767,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-italian:overall","benchmarkName":"EuroEval Italian","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":38.45125,"unit":"percent","family":"euroeval","index":24.389734327943067,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/italian","measuredAt":null,"stderr":0.5845898669152587,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":36.6175,"unit":"percent","family":"euroeval","index":21.90175751179731,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.7865381030185378,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-spanish:overall","benchmarkName":"EuroEval Spanish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":31.299999999999997,"unit":"percent","family":"euroeval","index":14.687133533689568,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/spanish","measuredAt":null,"stderr":0.5003982788739385,"costPerTest":null,"latency":null},{"benchmarkId":"gpqa","benchmarkName":"Graduate-Level Google-Proof Q&A","category":"knowledge","score":30.4,"unit":"percent","family":"gpqa-diamond","index":5.091150055410992,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":27,"unit":"percent","family":"gpqa-diamond","index":5,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":26.957070707070706,"unit":"percent","family":"gpqa-diamond","index":5,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.8583813167150034,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-math-level-5:overall","benchmarkName":"MATH level 5","category":"math","domain":null,"eligible":true,"ranked":true,"score":22.875755287009063,"unit":"percent","family":"epoch-ai:epoch-math-level-5:overall","index":10.109932250086594,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0.8519457631995396,"costPerTest":null,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":4.3,"unit":"percent","family":"hle","index":28.700803116941593,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":1.6666666666666667,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":9.88874761945322,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0.9401267913629434,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":55.00000000000001,"benchmarkCount":10,"categoryCount":5,"confidence":"low","state":"available"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"granite-4-0-h-1b","providerId":"ibm","slug":"granite-4-0-h-1b","name":"Granite-4.0-H-1B","provider":"IBM","description":"Granite-4.0-H-1B is a standard model from IBM in the Granite 4.0 1B family. 11 benchmarks count toward its index, in 6 categories.","logoUrl":null,"releaseDate":"2025-10-28","contextWindowTokens":128000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":16.25049874347284,"overallScoreRange":{"low":9.375060301650402,"high":23.125937185295278},"badges":[],"capabilityScores":{"agentic":15.795473139002347,"coding":14.272322885267833,"reasoning":24.474864575521615,"multimodal":null,"knowledge":11.100777009615612,"multilingual":5,"instruction":17.421582827616234,"math":null},"capabilityCounts":{"instruction":2,"coding":1,"knowledge":4,"multilingual":1,"agentic":1,"reasoning":2},"benchmarks":[{"benchmarkId":"ifeval","benchmarkName":"Instruction-Following Eval","category":"instruction","score":78.53,"unit":"percent","family":"ifeval","index":21.335358154634598,"aboveFit":false,"run":null,"source":"Jeffrey Zhou, Tianjian Lu, Swaroop Mishra, Siddhartha Brahma, Sujoy Basu, Yi Luan, Denny Zhou, Le Hou","sourceUrl":"https://arxiv.org/abs/2311.07911","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"humaneval","benchmarkName":"Evaluating Large Language Models Trained on Code","category":"coding","score":73,"unit":"percent","family":"humaneval","index":14.272322885267833,"aboveFit":false,"run":null,"source":"Mark Chen, Jerry Tworek, Heewoo Jun, Qiming Yuan, Henrique Ponde de Oliveira Pinto, Jared Kaplan, Harri Edwards, Yuri Burda, Nicholas Joseph, Greg Brockman, Alex Ray, Raul Puri, Gretchen Krueger, Michael Petrov, Heidy Khlaaf, Girish Sastry, Pamela Mishkin, Brooke Chan, Scott Gray, Nick Ryder, Mikhail Pavlov, Alethea Power, Lukasz Kaiser, Mohammad Bavarian, Clemens Winter, Philippe Tillet, Felipe Petroski Such, Dave Cummings, Matthias Plappert, Fotios Chantzis, Elizabeth Barnes, Ariel Herbert-Voss, William Hebgen Guss, Alex Nichol, Alex Paino, Nikolas Tezak, Jie Tang, Igor Babuschkin, Suchir Balaji, Shantanu Jain, William Saunders, Christopher Hesse, Andrew N. Carr, Jan Leike, Josh Achiam, Vedant Misra, Evan Morikawa, Alec Radford, Matthew Knight, Miles Brundage, Mira Murati, Katie Mayer, Peter Welinder, Bob McGrew, Dario Amodei, Sam McCandlish, Ilya Sutskever, Wojciech Zaremba","sourceUrl":"https://arxiv.org/abs/2107.03374","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmlu","benchmarkName":"Massive Multitask Language Understanding","category":"knowledge","score":59.74,"unit":"percent","family":"mmlu","index":5,"aboveFit":false,"run":null,"source":"Dan Hendrycks, Collin Burns, Steven Basart, Andy Zou, Mantas Mazeika, Dawn Song, Jacob Steinhardt","sourceUrl":"https://arxiv.org/abs/2009.03300","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"bbh","benchmarkName":"BIG-Bench Hard","category":"reasoning","score":59.68,"unit":"percent","family":"bbh","index":null,"aboveFit":false,"run":null,"source":"Mirac Suzgun, Nathan Scales, Nathanael Schärli, Sebastian Gehrmann, Yi Tay, Hyung Won Chung, Aakanksha Chowdhery, Quoc V. Le, Ed H. Chi, Denny Zhou, Jason Wei","sourceUrl":"https://arxiv.org/abs/2210.09261","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmluPro","benchmarkName":"Massive Multitask Language Understanding Professional","category":"knowledge","score":32.86,"unit":"percent","family":"mmlu-pro","index":5,"aboveFit":false,"run":null,"source":"Yubo Wang, Xueguang Ma, Ge Zhang, Yuansheng Ni, Abhranil Chandra, Shiguang Guo, Weiming Ren, Aaran Arulraj, Xuan He, Ziyan Jiang, Tianle Li, Max Ku, Kai Wang, Alex Zhuang, Rongqi Fan, Xiang Yue, Wenhu Chen","sourceUrl":"https://arxiv.org/abs/2406.01574","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gpqa","benchmarkName":"Graduate-Level Google-Proof Q&A","category":"knowledge","score":29.69,"unit":"percent","family":"gpqa-diamond","index":5,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mgsm","benchmarkName":"Multilingual Grade School Math","category":"multilingual","score":27.52,"unit":"percent","family":"mgsm","index":5,"aboveFit":false,"run":null,"source":"Freda Shi, Mirac Suzgun, Markus Freitag, Xuezhi Wang, Suraj Srivats, Soroush Vosoughi, Hyung Won Chung, Yi Tay, Sebastian Ruder, Denny Zhou, Dipanjan Das, Jason Wei","sourceUrl":"https://arxiv.org/abs/2210.03057","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":26.3,"unit":"percent","family":"gpqa-diamond","index":5,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaIfBench","benchmarkName":"Artificial Analysis IFBench","category":"instruction","score":26.2,"unit":"percent","family":"ifbench-family","index":13.50780750059787,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"tau2Bench","benchmarkName":"τ²-Bench Tool-Agent-User Evaluation","category":"agentic","score":19.6,"unit":"percent","family":"tau2-bench","index":15.795473139002347,"aboveFit":false,"run":null,"source":"Victor Barres, Honghua Dong, Soham Ray, Xujie Si, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2506.07982","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":7,"unit":"percent","family":"lcr","index":12.100613138270326,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":5,"unit":"percent","family":"hle","index":29.403108038462445,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":0,"unit":"percent","family":"critpt","index":36.8491160127729,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":80,"benchmarkCount":11,"categoryCount":6,"confidence":"low","state":"available"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"minicpm5-1b","providerId":"openbmb","slug":"minicpm5-1b","name":"MiniCPM5-1B","provider":"OpenBMB","description":"MiniCPM5-1B is a reasoning model from OpenBMB in the MiniCPM5 family. 13 benchmarks count toward its index, in 6 categories.","logoUrl":null,"releaseDate":"2026-05-25","contextWindowTokens":131000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":true,"overallScore":15.459733365514774,"overallScoreRange":{"low":8.474982279033796,"high":22.444484451995752},"badges":[],"capabilityScores":{"agentic":8.484667004685,"coding":11.938906679020718,"reasoning":null,"multimodal":null,"knowledge":9.208527564545655,"multilingual":5,"instruction":28.87936391952185,"math":21.92123869172682},"capabilityCounts":{"math":4,"instruction":2,"knowledge":4,"coding":1,"agentic":1,"multilingual":1},"benchmarks":[{"benchmarkId":"math500","benchmarkName":"MATH-500 Problem Set","category":"math","score":91.6,"unit":"percent","family":"math-500","index":40.92583551495429,"aboveFit":true,"run":null,"source":"Dan Hendrycks, Collin Burns, Saurav Kadavath, Akul Arora, Steven Basart, Eric Tang, Dawn Song, Jacob Steinhardt","sourceUrl":"https://arxiv.org/abs/2103.03874","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaOpennessIndex","benchmarkName":"Artificial Analysis Openness Index","category":"knowledge","score":83.3,"unit":"percent","family":"aaOpennessIndex","index":null,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-openness-index","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"ifeval","benchmarkName":"Instruction-Following Eval","category":"instruction","score":80.41,"unit":"percent","family":"ifeval","index":25.1214880528859,"aboveFit":false,"run":null,"source":"Jeffrey Zhou, Tianjian Lu, Swaroop Mishra, Siddhartha Brahma, Sujoy Basu, Yi Luan, Denny Zhou, Le Hou","sourceUrl":"https://arxiv.org/abs/2311.07911","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"bbh","benchmarkName":"BIG-Bench Hard","category":"reasoning","score":71.89,"unit":"percent","family":"bbh","index":null,"aboveFit":false,"run":null,"source":"Mirac Suzgun, Nathan Scales, Nathanael Schärli, Sebastian Gehrmann, Yi Tay, Hyung Won Chung, Aakanksha Chowdhery, Quoc V. Le, Ed H. Chi, Denny Zhou, Jason Wei","sourceUrl":"https://arxiv.org/abs/2210.09261","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmluRedux","benchmarkName":"MMLU-Redux","category":"knowledge","score":70.06,"unit":"percent","family":"mmluRedux","index":9.010410353574521,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmluPro","benchmarkName":"Massive Multitask Language Understanding Professional","category":"knowledge","score":48.85,"unit":"percent","family":"mmlu-pro","index":6.0258812322530915,"aboveFit":false,"run":null,"source":"Yubo Wang, Xueguang Ma, Ge Zhang, Yuansheng Ni, Abhranil Chandra, Shiguang Guo, Weiming Ren, Aaran Arulraj, Xuan He, Ziyan Jiang, Tianle Li, Max Ku, Kai Wang, Alex Zhuang, Rongqi Fan, Xiang Yue, Wenhu Chen","sourceUrl":"https://arxiv.org/abs/2406.01574","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"ifBench","benchmarkName":"Instruction Following Benchmark","category":"instruction","score":46.67,"unit":"percent","family":"ifbench-family","index":32.63723978615781,"aboveFit":true,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aime2025","benchmarkName":"American Invitational Mathematics Examination 2025","category":"math","score":40.42,"unit":"percent","family":"aime2025","index":20.94015725587844,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aime2026","benchmarkName":"AIME 2026","category":"math","score":40.42,"unit":"percent","family":"aime2026","index":14.390972523712186,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"liveCodeBenchV6","benchmarkName":"LiveCodeBench v6","category":"coding","score":33.52,"unit":"percent","family":"liveCodeBenchV6","index":11.938906679020718,"aboveFit":false,"run":null,"source":"LiveCodeBench maintainers","sourceUrl":"https://github.com/LiveCodeBench/LiveCodeBench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gpqaDiamond","benchmarkName":"GPQA Diamond","category":"knowledge","score":26.26,"unit":"percent","family":"gpqa-diamond","index":5,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hmmtFeb2026","benchmarkName":"Harvard-MIT Mathematics Tournament February 2026","category":"math","score":25.76,"unit":"percent","family":"hmmtFeb2026","index":11.427989472362377,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"bfclV4","benchmarkName":"Berkeley Function Calling Leaderboard v4","category":"agentic","score":25.15,"unit":"percent","family":"bfclV4","index":8.484667004685,"aboveFit":false,"run":null,"source":"Shishir G. Patil et al.","sourceUrl":"https://gorilla.cs.berkeley.edu/leaderboard.html","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"superGpqa","benchmarkName":"SuperGPQA: Scaling LLM Evaluation Across 285 Graduate Disciplines","category":"knowledge","score":23.14,"unit":"percent","family":"superGpqa","index":16.797818672355007,"aboveFit":false,"run":null,"source":"Xiaoxuan Du, Yao Yao, Kexin Ma, Bowen Wang, Tianyu Zheng, Kaiyan Zhu, Yiming Zhang, Yutao Zhu, Jiawei Zhou, Jingren Zhou","sourceUrl":"https://arxiv.org/abs/2502.14739","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"liveCodeBenchPro","benchmarkName":"LiveCodeBench Pro","category":"coding","score":22.68,"unit":"percent","family":"liveCodeBenchPro","index":null,"aboveFit":false,"run":null,"source":"LiveCodeBench Pro authors","sourceUrl":"https://arxiv.org/abs/2506.11928","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-french:overall","benchmarkName":"EuroEval French","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":11.72375,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/french","measuredAt":null,"stderr":0.8556850325908476,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-spanish:overall","benchmarkName":"EuroEval Spanish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":9.735000000000001,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/spanish","measuredAt":null,"stderr":0.7830748687066901,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":7.133750000000001,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.9422463186449709,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":75,"benchmarkCount":13,"categoryCount":6,"confidence":"low","state":"available"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"granite-4-0-h-350m","providerId":"ibm","slug":"granite-4-0-h-350m","name":"Granite-4.0-H-350M","provider":"IBM","description":"Granite-4.0-H-350M is a standard model from IBM in the Granite 4.0 350M family. 11 benchmarks count toward its index, in 6 categories.","logoUrl":null,"releaseDate":"2025-10-28","contextWindowTokens":32000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":12.720791784908618,"overallScoreRange":{"low":5.845353343086181,"high":19.596230226731056},"badges":[],"capabilityScores":{"agentic":12.477607579394316,"coding":5,"reasoning":22.073582735971804,"multimodal":null,"knowledge":11.451929470376044,"multilingual":5,"instruction":5.235508106532066,"math":null},"capabilityCounts":{"instruction":2,"coding":1,"knowledge":4,"multilingual":1,"agentic":1,"reasoning":2},"benchmarks":[{"benchmarkId":"ifeval","benchmarkName":"Instruction-Following Eval","category":"instruction","score":61.63,"unit":"percent","family":"ifeval","index":5,"aboveFit":false,"run":null,"source":"Jeffrey Zhou, Tianjian Lu, Swaroop Mishra, Siddhartha Brahma, Sujoy Basu, Yi Luan, Denny Zhou, Le Hou","sourceUrl":"https://arxiv.org/abs/2311.07911","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"humaneval","benchmarkName":"Evaluating Large Language Models Trained on Code","category":"coding","score":38,"unit":"percent","family":"humaneval","index":5,"aboveFit":false,"run":null,"source":"Mark Chen, Jerry Tworek, Heewoo Jun, Qiming Yuan, Henrique Ponde de Oliveira Pinto, Jared Kaplan, Harri Edwards, Yuri Burda, Nicholas Joseph, Greg Brockman, Alex Ray, Raul Puri, Gretchen Krueger, Michael Petrov, Heidy Khlaaf, Girish Sastry, Pamela Mishkin, Brooke Chan, Scott Gray, Nick Ryder, Mikhail Pavlov, Alethea Power, Lukasz Kaiser, Mohammad Bavarian, Clemens Winter, Philippe Tillet, Felipe Petroski Such, Dave Cummings, Matthias Plappert, Fotios Chantzis, Elizabeth Barnes, Ariel Herbert-Voss, William Hebgen Guss, Alex Nichol, Alex Paino, Nikolas Tezak, Jie Tang, Igor Babuschkin, Suchir Balaji, Shantanu Jain, William Saunders, Christopher Hesse, Andrew N. Carr, Jan Leike, Josh Achiam, Vedant Misra, Evan Morikawa, Alec Radford, Matthew Knight, Miles Brundage, Mira Murati, Katie Mayer, Peter Welinder, Bob McGrew, Dario Amodei, Sam McCandlish, Ilya Sutskever, Wojciech Zaremba","sourceUrl":"https://arxiv.org/abs/2107.03374","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmlu","benchmarkName":"Massive Multitask Language Understanding","category":"knowledge","score":36.21,"unit":"percent","family":"mmlu","index":5,"aboveFit":false,"run":null,"source":"Dan Hendrycks, Collin Burns, Steven Basart, Andy Zou, Mantas Mazeika, Dawn Song, Jacob Steinhardt","sourceUrl":"https://arxiv.org/abs/2009.03300","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"bbh","benchmarkName":"BIG-Bench Hard","category":"reasoning","score":33.28,"unit":"percent","family":"bbh","index":null,"aboveFit":false,"run":null,"source":"Mirac Suzgun, Nathan Scales, Nathanael Schärli, Sebastian Gehrmann, Yi Tay, Hyung Won Chung, Aakanksha Chowdhery, Quoc V. Le, Ed H. Chi, Denny Zhou, Jason Wei","sourceUrl":"https://arxiv.org/abs/2210.09261","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gpqa","benchmarkName":"Graduate-Level Google-Proof Q&A","category":"knowledge","score":26.12,"unit":"percent","family":"gpqa-diamond","index":5,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":25.7,"unit":"percent","family":"gpqa-diamond","index":5,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaIfBench","benchmarkName":"Artificial Analysis IFBench","category":"instruction","score":17.6,"unit":"percent","family":"ifbench-family","index":5.471016213064146,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mgsm","benchmarkName":"Multilingual Grade School Math","category":"multilingual","score":16.16,"unit":"percent","family":"mgsm","index":5,"aboveFit":false,"run":null,"source":"Freda Shi, Mirac Suzgun, Markus Freitag, Xuezhi Wang, Suraj Srivats, Soroush Vosoughi, Hyung Won Chung, Yi Tay, Sebastian Ruder, Denny Zhou, Dipanjan Das, Jason Wei","sourceUrl":"https://arxiv.org/abs/2210.03057","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"tau2Bench","benchmarkName":"τ²-Bench Tool-Agent-User Evaluation","category":"agentic","score":14.6,"unit":"percent","family":"tau2-bench","index":12.477607579394316,"aboveFit":false,"run":null,"source":"Victor Barres, Honghua Dong, Soham Ray, Xujie Si, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2506.07982","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmluPro","benchmarkName":"Massive Multitask Language Understanding Professional","category":"knowledge","score":14.38,"unit":"percent","family":"mmlu-pro","index":5,"aboveFit":false,"run":null,"source":"Yubo Wang, Xueguang Ma, Ge Zhang, Yuansheng Ni, Abhranil Chandra, Shiguang Guo, Weiming Ren, Aaran Arulraj, Xuan He, Ziyan Jiang, Tianle Li, Max Ku, Kai Wang, Alex Zhuang, Rongqi Fan, Xiang Yue, Wenhu Chen","sourceUrl":"https://arxiv.org/abs/2406.01574","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":6.4,"unit":"percent","family":"hle","index":30.807717881504157,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":0,"unit":"percent","family":"lcr","index":7.298049459170706,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":0,"unit":"percent","family":"critpt","index":36.8491160127729,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":80,"benchmarkCount":11,"categoryCount":6,"confidence":"low","state":"available"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"granite-4-0-350m","providerId":"ibm","slug":"granite-4-0-350m","name":"Granite-4.0-350M","provider":"IBM","description":"Granite-4.0-350M is a standard model from IBM in the Granite 4.0 350M family. 12 benchmarks count toward its index, in 7 categories.","logoUrl":null,"releaseDate":"2025-10-28","contextWindowTokens":32000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":11.87321979146354,"overallScoreRange":{"low":6.205342865065416,"high":17.541096717861667},"badges":[],"capabilityScores":{"agentic":11.548605222704069,"coding":5,"reasoning":22.073582735971804,"multimodal":null,"knowledge":11.226188602744337,"multilingual":5,"instruction":5,"math":9.080700333712812},"capabilityCounts":{"instruction":2,"coding":1,"knowledge":4,"multilingual":1,"agentic":1,"math":1,"reasoning":2},"benchmarks":[{"benchmarkId":"ifeval","benchmarkName":"Instruction-Following Eval","category":"instruction","score":55.4,"unit":"percent","family":"ifeval","index":5,"aboveFit":false,"run":null,"source":"Jeffrey Zhou, Tianjian Lu, Swaroop Mishra, Siddhartha Brahma, Sujoy Basu, Yi Luan, Denny Zhou, Le Hou","sourceUrl":"https://arxiv.org/abs/2311.07911","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"humaneval","benchmarkName":"Evaluating Large Language Models Trained on Code","category":"coding","score":39,"unit":"percent","family":"humaneval","index":5,"aboveFit":false,"run":null,"source":"Mark Chen, Jerry Tworek, Heewoo Jun, Qiming Yuan, Henrique Ponde de Oliveira Pinto, Jared Kaplan, Harri Edwards, Yuri Burda, Nicholas Joseph, Greg Brockman, Alex Ray, Raul Puri, Gretchen Krueger, Michael Petrov, Heidy Khlaaf, Girish Sastry, Pamela Mishkin, Brooke Chan, Scott Gray, Nick Ryder, Mikhail Pavlov, Alethea Power, Lukasz Kaiser, Mohammad Bavarian, Clemens Winter, Philippe Tillet, Felipe Petroski Such, Dave Cummings, Matthias Plappert, Fotios Chantzis, Elizabeth Barnes, Ariel Herbert-Voss, William Hebgen Guss, Alex Nichol, Alex Paino, Nikolas Tezak, Jie Tang, Igor Babuschkin, Suchir Balaji, Shantanu Jain, William Saunders, Christopher Hesse, Andrew N. Carr, Jan Leike, Josh Achiam, Vedant Misra, Evan Morikawa, Alec Radford, Matthew Knight, Miles Brundage, Mira Murati, Katie Mayer, Peter Welinder, Bob McGrew, Dario Amodei, Sam McCandlish, Ilya Sutskever, Wojciech Zaremba","sourceUrl":"https://arxiv.org/abs/2107.03374","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmlu","benchmarkName":"Massive Multitask Language Understanding","category":"knowledge","score":35.01,"unit":"percent","family":"mmlu","index":5,"aboveFit":false,"run":null,"source":"Dan Hendrycks, Collin Burns, Steven Basart, Andy Zou, Mantas Mazeika, Dawn Song, Jacob Steinhardt","sourceUrl":"https://arxiv.org/abs/2009.03300","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"bbh","benchmarkName":"BIG-Bench Hard","category":"reasoning","score":33.07,"unit":"percent","family":"bbh","index":null,"aboveFit":false,"run":null,"source":"Mirac Suzgun, Nathan Scales, Nathanael Schärli, Sebastian Gehrmann, Yi Tay, Hyung Won Chung, Aakanksha Chowdhery, Quoc V. Le, Ed H. Chi, Denny Zhou, Jason Wei","sourceUrl":"https://arxiv.org/abs/2210.09261","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":26.1,"unit":"percent","family":"gpqa-diamond","index":5,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gpqa","benchmarkName":"Graduate-Level Google-Proof Q&A","category":"knowledge","score":24.11,"unit":"percent","family":"gpqa-diamond","index":5,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaIfBench","benchmarkName":"Artificial Analysis IFBench","category":"instruction","score":15.9,"unit":"percent","family":"ifbench-family","index":5,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mgsm","benchmarkName":"Multilingual Grade School Math","category":"multilingual","score":14.72,"unit":"percent","family":"mgsm","index":5,"aboveFit":false,"run":null,"source":"Freda Shi, Mirac Suzgun, Markus Freitag, Xuezhi Wang, Suraj Srivats, Soroush Vosoughi, Hyung Won Chung, Yi Tay, Sebastian Ruder, Denny Zhou, Dipanjan Das, Jason Wei","sourceUrl":"https://arxiv.org/abs/2210.03057","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"tau2Bench","benchmarkName":"τ²-Bench Tool-Agent-User Evaluation","category":"agentic","score":13.2,"unit":"percent","family":"tau2-bench","index":11.548605222704069,"aboveFit":false,"run":null,"source":"Victor Barres, Honghua Dong, Soham Ray, Xujie Si, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2506.07982","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmluPro","benchmarkName":"Massive Multitask Language Understanding Professional","category":"knowledge","score":12.13,"unit":"percent","family":"mmlu-pro","index":5,"aboveFit":false,"run":null,"source":"Yubo Wang, Xueguang Ma, Ge Zhang, Yuansheng Ni, Abhranil Chandra, Shiguang Guo, Weiming Ren, Aaran Arulraj, Xuan He, Ziyan Jiang, Tianle Li, Max Ku, Kai Wang, Alex Zhuang, Rongqi Fan, Xiang Yue, Wenhu Chen","sourceUrl":"https://arxiv.org/abs/2406.01574","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":11.237373737373737,"unit":"percent","family":"gpqa-diamond","index":5,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0.9083571075486867,"costPerTest":null,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":5.5,"unit":"percent","family":"hle","index":29.904754410977343,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":0.2777777777777778,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":9.080700333712812,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0.2777777777777778,"costPerTest":null,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":0,"unit":"percent","family":"lcr","index":7.298049459170706,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":0,"unit":"percent","family":"critpt","index":36.8491160127729,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":90,"benchmarkCount":12,"categoryCount":7,"confidence":"medium","state":"available"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"bonsai-1-7b","providerId":"prism-ml","slug":"bonsai-1-7b","name":"1-bit Bonsai 1.7B","provider":"Prism ML","description":"1-bit Bonsai 1.7B is a standard model from Prism ML in the 1-bit Bonsai family. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":"2026-03-31","contextWindowTokens":32000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":5,"math":null},"capabilityCounts":{"instruction":1},"benchmarks":[{"benchmarkId":"ifeval","benchmarkName":"Instruction-Following Eval","category":"instruction","score":63,"unit":"percent","family":"ifeval","index":5,"aboveFit":false,"run":null,"source":"Jeffrey Zhou, Tianjian Lu, Swaroop Mishra, Siddhartha Brahma, Sujoy Basu, Yi Luan, Denny Zhou, Le Hou","sourceUrl":"https://arxiv.org/abs/2311.07911","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"musr","benchmarkName":"Testing the Limits of Chain-of-thought with Multistep Soft Reasoning","category":"reasoning","score":45.1,"unit":"percent","family":"musr","index":null,"aboveFit":false,"run":null,"source":"Zayne Sprague, Xi Ye, Kaj Bostrom, Swarat Chaudhuri, Greg Durrett","sourceUrl":"https://arxiv.org/abs/2310.16049","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"bonsai-27b","providerId":"prism-ml","slug":"bonsai-27b","name":"1-bit Bonsai 27B","provider":"Prism ML","description":"1-bit Bonsai 27B is a reasoning model from Prism ML in the Bonsai family. It has no scored result.","logoUrl":null,"releaseDate":"2026-07-14","contextWindowTokens":262000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":true,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"bonsai-4b","providerId":"prism-ml","slug":"bonsai-4b","name":"1-bit Bonsai 4B","provider":"Prism ML","description":"1-bit Bonsai 4B is a standard model from Prism ML in the 1-bit Bonsai family. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":"2026-03-31","contextWindowTokens":32000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":5,"math":null},"capabilityCounts":{"instruction":1},"benchmarks":[{"benchmarkId":"ifeval","benchmarkName":"Instruction-Following Eval","category":"instruction","score":69.6,"unit":"percent","family":"ifeval","index":5,"aboveFit":false,"run":null,"source":"Jeffrey Zhou, Tianjian Lu, Swaroop Mishra, Siddhartha Brahma, Sujoy Basu, Yi Luan, Denny Zhou, Le Hou","sourceUrl":"https://arxiv.org/abs/2311.07911","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"musr","benchmarkName":"Testing the Limits of Chain-of-thought with Multistep Soft Reasoning","category":"reasoning","score":41.4,"unit":"percent","family":"musr","index":null,"aboveFit":false,"run":null,"source":"Zayne Sprague, Xi Ye, Kaj Bostrom, Swarat Chaudhuri, Greg Durrett","sourceUrl":"https://arxiv.org/abs/2310.16049","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"bonsai-8b","providerId":"prism-ml","slug":"bonsai-8b","name":"1-bit Bonsai 8B","provider":"Prism ML","description":"1-bit Bonsai 8B is a standard model from Prism ML in the 1-bit Bonsai family. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":"2026-03-31","contextWindowTokens":64000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":23.893009734836276,"math":null},"capabilityCounts":{"instruction":1},"benchmarks":[{"benchmarkId":"ifeval","benchmarkName":"Instruction-Following Eval","category":"instruction","score":79.8,"unit":"percent","family":"ifeval","index":23.893009734836276,"aboveFit":false,"run":null,"source":"Jeffrey Zhou, Tianjian Lu, Swaroop Mishra, Siddhartha Brahma, Sujoy Basu, Yi Luan, Denny Zhou, Le Hou","sourceUrl":"https://arxiv.org/abs/2311.07911","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"musr","benchmarkName":"Testing the Limits of Chain-of-thought with Multistep Soft Reasoning","category":"reasoning","score":50,"unit":"percent","family":"musr","index":null,"aboveFit":false,"run":null,"source":"Zayne Sprague, Xi Ye, Kaj Bostrom, Swarat Chaudhuri, Greg Durrett","sourceUrl":"https://arxiv.org/abs/2310.16049","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"4xscaled","providerId":"unknown","slug":"unknown-4x-scaled","name":"4x Scaled","provider":"Unknown","description":"4x Scaled is a model. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":46.07142825872012,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"coding":1},"benchmarks":[{"benchmarkId":"swe-bench:swe-bench-verified:overall","benchmarkName":"SWE-bench Verified","category":"coding","domain":null,"eligible":true,"ranked":true,"score":60.8,"unit":"percent","family":"swe-bench-verified","index":46.07142825872012,"aboveFit":false,"run":"OpenHands","source":"SWE-bench team","sourceUrl":"https://www.swebench.com/","measuredAt":"2026-02-26","stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":20,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:swe-bench-team","name":"SWE-bench team","url":"https://www.swebench.com","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"skt-ax","providerId":"sk-telecom","slug":"skt-ax","name":"A.X series","provider":"SK Telecom","description":"A.X series is a standard model from SK Telecom in the A.X family. It has no scored result.","logoUrl":null,"releaseDate":null,"contextWindowTokens":64000,"modalities":["text"],"openWeights":false,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"agents-a1","providerId":"internscience","slug":"agents-a1","name":"Agents-A1","provider":"InternScience","description":"Agents-A1 is a reasoning model from InternScience. It has 6 scored results in 4 categories. It has no index.","logoUrl":null,"releaseDate":"2026-06-26","contextWindowTokens":262000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":true,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":57.08481257547111,"coding":null,"reasoning":51.72402889264172,"multimodal":null,"knowledge":72.1433789767317,"multilingual":null,"instruction":54.14177094320576,"math":null},"capabilityCounts":{"instruction":1,"agentic":3,"reasoning":1,"knowledge":1},"benchmarks":[{"benchmarkId":"ifeval","benchmarkName":"Instruction-Following Eval","category":"instruction","score":94.82,"unit":"percent","family":"ifeval","index":54.14177094320576,"aboveFit":false,"run":null,"source":"Jeffrey Zhou, Tianjian Lu, Swaroop Mishra, Siddhartha Brahma, Sujoy Basu, Yi Luan, Denny Zhou, Le Hou","sourceUrl":"https://arxiv.org/abs/2311.07911","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"browseComp","benchmarkName":"BrowseComp","category":"agentic","score":75.51,"unit":"percent","family":"browseComp","index":59.56338960160624,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"longBenchV2","benchmarkName":"LongBench v2","category":"reasoning","score":60.2,"unit":"percent","family":"longBenchV2","index":51.72402889264172,"aboveFit":false,"run":null,"source":"LongBench v2 authors","sourceUrl":"https://arxiv.org/abs/2412.15204","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hleWithTools","benchmarkName":"Humanity's Last Exam with tools","category":"agentic","score":47.6,"unit":"percent","family":"hleWithTools","index":57.99646324316515,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hle","benchmarkName":"Humanity's Last Exam","category":"knowledge","score":47.6,"unit":"percent","family":"hle","index":72.1433789767317,"aboveFit":false,"run":null,"source":"Center for AI Safety, Scale AI, and thousands of expert contributors","sourceUrl":"https://lastexam.ai/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vitaBench","benchmarkName":"VITA-Bench","category":"agentic","score":38.75,"unit":"percent","family":"vitaBench","index":53.694584881641944,"aboveFit":false,"run":null,"source":"Meituan LongCat Team","sourceUrl":"https://vitabench.github.io/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":55.00000000000001,"benchmarkCount":6,"categoryCount":4,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"agents-a1-f16-gguf","providerId":"internscience","slug":"agents-a1-f16-gguf","name":"Agents-A1-F16-GGUF","provider":"InternScience","description":"Agents-A1-F16-GGUF is a reasoning model from InternScience in the Agents-A1 family. It has no scored result.","logoUrl":null,"releaseDate":"2026-07-02","contextWindowTokens":262000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":true,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"agents-a1-fp8","providerId":"internscience","slug":"agents-a1-fp8","name":"Agents-A1-FP8","provider":"InternScience","description":"Agents-A1-FP8 is a reasoning model from InternScience in the Agents-A1 family. It has no scored result.","logoUrl":null,"releaseDate":"2026-07-02","contextWindowTokens":262000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":true,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"agents-a1-q4-k-m-gguf","providerId":"internscience","slug":"agents-a1-q4-k-m-gguf","name":"Agents-A1-Q4_K_M-GGUF","provider":"InternScience","description":"Agents-A1-Q4_K_M-GGUF is a reasoning model from InternScience in the Agents-A1 family. It has no scored result.","logoUrl":null,"releaseDate":"2026-07-02","contextWindowTokens":262000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":true,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"agents-a1-q8-0-gguf","providerId":"internscience","slug":"agents-a1-q8-0-gguf","name":"Agents-A1-Q8_0-GGUF","provider":"InternScience","description":"Agents-A1-Q8_0-GGUF is a reasoning model from InternScience in the Agents-A1 family. It has no scored result.","logoUrl":null,"releaseDate":"2026-07-02","contextWindowTokens":262000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":true,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"aicomathematician","providerId":"unknown","slug":"unknown-ai-co-mathematician","name":"AI Co-Mathematician","provider":"Unknown","description":"AI Co-Mathematician is a model. It has 2 scored results in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":85.02066276962799},"capabilityCounts":{"math":2},"benchmarks":[{"benchmarkId":"epoch-ai:epoch-frontiermath-tier-4-v2-private:overall","benchmarkName":"FrontierMath-Tier-4-v2-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":75.6,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tier-4-v2-private:overall","index":78.18776742996235,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":6.7,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-private:overall","benchmarkName":"FrontierMath-Tier-4-2025-07-01-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":47.9,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-private:overall","index":91.85355810929364,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":7.199999999999999,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":10,"benchmarkCount":2,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"aion30","providerId":"aion-labs","slug":"aion-labs-aion-3-0","name":"Aion 3.0","provider":"Aion Labs","description":"Aion 3.0 is a model from Aion Labs. It has no scored result.","logoUrl":null,"releaseDate":"2026-07-07T16:51:35.000Z","contextWindowTokens":131072,"modalities":["text"],"openWeights":null,"details":{"apiId":"aion-labs/aion-3.0","maxOutputTokens":32768,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":false,"cacheReadPerMillion":0.75,"weightsUrl":null},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":3,"outputPerMillion":6,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":42,"timeToFirstToken":1.337,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":3,"outputPerMillion":6,"cacheReadPerMillion":0.75,"contextTokens":131072,"maxOutputTokens":32768,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"aion30mini","providerId":"aion-labs","slug":"aion-labs-aion-3-0-mini","name":"Aion 3.0 Mini","provider":"Aion Labs","description":"Aion 3.0 Mini is a model from Aion Labs. It has no scored result.","logoUrl":null,"releaseDate":"2026-07-07T16:51:36.000Z","contextWindowTokens":131072,"modalities":["text"],"openWeights":null,"details":{"apiId":"aion-labs/aion-3.0-mini","maxOutputTokens":32768,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":false,"cacheReadPerMillion":0.18,"weightsUrl":null},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":0.7,"outputPerMillion":1.4,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":54,"timeToFirstToken":0.754,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":0.7,"outputPerMillion":1.4,"cacheReadPerMillion":0.18,"contextTokens":131072,"maxOutputTokens":32768,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"aion35","providerId":"aion-labs","slug":"aion-labs-aion-3-5","name":"Aion 3.5","provider":"Aion Labs","description":"Aion 3.5 is a model from Aion Labs. It has no scored result.","logoUrl":null,"releaseDate":"2026-09-23T13:42:41.000Z","contextWindowTokens":262144,"modalities":["text"],"openWeights":null,"details":{"apiId":"aion-labs/aion-3.5","maxOutputTokens":32768,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":false,"cacheReadPerMillion":0.75,"weightsUrl":null},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":3,"outputPerMillion":6,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":50,"timeToFirstToken":1.986,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"aion35mini","providerId":"aion-labs","slug":"aion-labs-aion-3-5-mini","name":"Aion 3.5 Mini","provider":"Aion Labs","description":"Aion 3.5 Mini is a model from Aion Labs. It has no scored result.","logoUrl":null,"releaseDate":"2026-09-23T13:42:42.000Z","contextWindowTokens":262144,"modalities":["text"],"openWeights":null,"details":{"apiId":"aion-labs/aion-3.5-mini","maxOutputTokens":32768,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":false,"cacheReadPerMillion":0.18,"weightsUrl":null},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":0.7,"outputPerMillion":1.4,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":58,"timeToFirstToken":1.7245,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"aion-2-0","providerId":"aion-labs","slug":"aion-2-0","name":"Aion-2.0","provider":"Aion Labs","description":"Aion-2.0 is a standard model from Aion Labs. It has no scored result.","logoUrl":null,"releaseDate":null,"contextWindowTokens":131072,"modalities":["text"],"openWeights":false,"details":{"apiId":"aion-labs/aion-2.0","maxOutputTokens":32768,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":false,"cacheReadPerMillion":0.19999999999999998,"weightsUrl":null},"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":0.7999999999999999,"outputPerMillion":1.5999999999999999,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":53,"timeToFirstToken":0.867,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":0.8,"outputPerMillion":1.6,"cacheReadPerMillion":0.2,"contextTokens":131072,"maxOutputTokens":32768,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"aionrpllama318b","providerId":"aion-labs","slug":"aion-labs-aion-rp-1-0-8b","name":"Aion-RP 1.0 (8B)","provider":"Aion Labs","description":"Aion-RP 1.0 (8B) is a model from Aion Labs. It has no scored result.","logoUrl":null,"releaseDate":"2025-02-04T19:18:38.000Z","contextWindowTokens":32768,"modalities":["text"],"openWeights":true,"details":{"apiId":"aion-labs/aion-rp-llama-3.1-8b","maxOutputTokens":29491,"knowledgeCutoff":"2023-12-31","retiresAt":null,"tools":false,"structuredOutput":false,"cacheReadPerMillion":null,"weightsUrl":"https://huggingface.co/aion-labs/Aion-RP-Llama-3.1-8B"},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":0.7999999999999999,"outputPerMillion":1.5999999999999999,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":30,"timeToFirstToken":1.2795,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":0.8,"outputPerMillion":1.6,"cacheReadPerMillion":null,"contextTokens":32768,"maxOutputTokens":29491,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"decision-index-0-3-ajev-lora5-gemma-4-12b","providerId":"andyzhang232","slug":"decision-index-0-3-ajev-lora5-gemma-4-12b","name":"AJev lora5 (Gemma 4 12B)","provider":"andyzhang232","description":"AJev lora5 (Gemma 4 12B) is a model from andyzhang232. It has no scored result.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"alephprover","providerId":"unknown","slug":"unknown-alephprover","name":"Alephprover","provider":"Unknown","description":"Alephprover is a model. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":81.68109068983412},"capabilityCounts":{"math":1},"benchmarks":[{"benchmarkId":"vals-ai:proof_bench:overall","benchmarkName":"ProofBench v1.1","category":"math","domain":null,"eligible":true,"ranked":true,"score":100,"unit":"percent","family":"vals-ai:proof_bench:overall","index":81.68109068983412,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/proof_bench","measuredAt":"2026-11-09","stderr":0,"costPerTest":9.3487,"latency":1108.1}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":10,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"amalia9b0626dpo","providerId":"unknown","slug":"unknown-amalia-9b-dpo","name":"AMALIA 9B DPO","provider":"Unknown","description":"AMALIA 9B DPO is a model. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":24.562722491028865,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-french:overall","benchmarkName":"EuroEval French","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":43.7275,"unit":"percent","family":"euroeval","index":31.548391547405394,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/french","measuredAt":null,"stderr":0.4642298730155137,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-dutch:overall","benchmarkName":"EuroEval Dutch","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":43.32111111111111,"unit":"percent","family":"euroeval","index":30.99701533260251,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/dutch","measuredAt":null,"stderr":0.5785582335682639,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-portuguese:overall","benchmarkName":"EuroEval Portuguese","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":43.2425,"unit":"percent","family":"euroeval","index":30.890358142726082,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/portuguese","measuredAt":null,"stderr":0.5820840789782864,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-spanish:overall","benchmarkName":"EuroEval Spanish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":39.565000000000005,"unit":"percent","family":"euroeval","index":25.90083681136902,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/spanish","measuredAt":null,"stderr":0.5257776977963216,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-italian:overall","benchmarkName":"EuroEval Italian","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":37.592499999999994,"unit":"percent","family":"euroeval","index":23.224608170688704,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/italian","measuredAt":null,"stderr":0.573790848219105,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":37.3925,"unit":"percent","family":"euroeval","index":22.953254189377652,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":0.734597040900656,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":36.47714285714286,"unit":"percent","family":"euroeval","index":21.71132516419866,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.6747319951700249,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":33.20375,"unit":"percent","family":"euroeval","index":17.2700842432942,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.5399001643822681,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"amalia9b0626sft","providerId":"unknown","slug":"unknown-amalia-9b-sft","name":"AMALIA 9B SFT","provider":"Unknown","description":"AMALIA 9B SFT is a model. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":22.918486960522177,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-french:overall","benchmarkName":"EuroEval French","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":44.0075,"unit":"percent","family":"euroeval","index":31.928287121240878,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/french","measuredAt":null,"stderr":0.500015624755867,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-dutch:overall","benchmarkName":"EuroEval Dutch","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":42.97888888888889,"unit":"percent","family":"euroeval","index":30.532698520136925,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/dutch","measuredAt":null,"stderr":0.3966184222139791,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-portuguese:overall","benchmarkName":"EuroEval Portuguese","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":41.3575,"unit":"percent","family":"euroeval","index":28.332846868869368,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/portuguese","measuredAt":null,"stderr":0.5106429770397317,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-spanish:overall","benchmarkName":"EuroEval Spanish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":37.82125,"unit":"percent","family":"euroeval","index":23.534969286813233,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/spanish","measuredAt":null,"stderr":0.5726610035614438,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-italian:overall","benchmarkName":"EuroEval Italian","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":36.9125,"unit":"percent","family":"euroeval","index":22.30200463423112,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/italian","measuredAt":null,"stderr":0.5238782659168063,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":36.817142857142855,"unit":"percent","family":"euroeval","index":22.17262693242745,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.4886069327116405,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":33.25875,"unit":"percent","family":"euroeval","index":17.344706588154736,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":0.6408258831383139,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":31.04375,"unit":"percent","family":"euroeval","index":14.339461245134778,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.6252449519988147,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"nova-2-lite","providerId":"amazon","slug":"nova-2-lite","name":"Amazon Nova 2 Lite","provider":"Amazon","description":"Amazon Nova 2 Lite is a reasoning model from Amazon in the Amazon Nova 2 family. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":"2025-12-02","contextWindowTokens":1000000,"modalities":["text"],"openWeights":false,"details":null,"reasoning":true,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":30.126312673234686,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"multimodal":1},"benchmarks":[{"benchmarkId":"parseBenchLlamaIndex","benchmarkName":"ParseBench: document parsing for AI agents (LlamaIndex leaderboard)","category":"multimodal","score":40.43,"unit":"percent","family":"parseBenchLlamaIndex","index":30.126312673234686,"aboveFit":false,"run":null,"source":"Boyang Zhang, Sebastián G. Acosta, Preston Carlson, Sacha Bron, Pierre-Loïc Doulcet, Daniel B. Ospina and Simon Suo (LlamaIndex)","sourceUrl":"https://arxiv.org/abs/2604.08538","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":10,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"nova-2-pro","providerId":"amazon","slug":"nova-2-pro","name":"Amazon Nova 2 Pro","provider":"Amazon","description":"Amazon Nova 2 Pro is a reasoning model from Amazon in the Amazon Nova 2 family. It has no scored result.","logoUrl":null,"releaseDate":"2025-12-02","contextWindowTokens":1000000,"modalities":["text"],"openWeights":false,"details":null,"reasoning":true,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"amazon-nova-2-sonic","providerId":"amazon","slug":"amazon-nova-2-sonic","name":"Amazon Nova 2 Sonic","provider":"Amazon","description":"Amazon Nova 2 Sonic is a standard model from Amazon. It has no scored result.","logoUrl":null,"releaseDate":"2025-12-02","contextWindowTokens":null,"modalities":["text"],"openWeights":false,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"amazonnovapremierv10","providerId":"unknown","slug":"unknown-amazon-nova-premier-v1-0","name":"Amazon.nova Premier V1:0","provider":"Unknown","description":"Amazon.nova Premier V1:0 is a model. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":32.50036882750018,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"coding":1},"benchmarks":[{"benchmarkId":"swebench-experiments:swe-bench-verified:overall","benchmarkName":"SWE-bench Verified","category":"coding","domain":null,"eligible":true,"ranked":true,"score":42.4,"unit":"percent","family":"swe-bench-verified","index":32.50036882750018,"aboveFit":false,"run":"Amazon Nova Premier 1.0","source":"SWE-bench team","sourceUrl":"https://github.com/SWE-bench/experiments/tree/main/evaluation/verified","measuredAt":"2026-09-01","stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":20,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:swe-bench-team","name":"SWE-bench team","url":"https://github.com","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"apertusv1105b","providerId":"unknown","slug":"unknown-apertus-v1-1-0-5b","name":"Apertus V1.1 0.5B","provider":"Unknown","description":"Apertus V1.1 0.5B is a model. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":5,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-dutch:overall","benchmarkName":"EuroEval Dutch","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":23.683333333333337,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/dutch","measuredAt":null,"stderr":0.8422236880667396,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-french:overall","benchmarkName":"EuroEval French","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":21.9725,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/french","measuredAt":null,"stderr":0.6444850172812399,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-portuguese:overall","benchmarkName":"EuroEval Portuguese","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":20.06375,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/portuguese","measuredAt":null,"stderr":0.7985250074355843,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":19.62375,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":1.0240147704012867,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-spanish:overall","benchmarkName":"EuroEval Spanish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":15.936250000000001,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/spanish","measuredAt":null,"stderr":0.6982679464217156,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-italian:overall","benchmarkName":"EuroEval Italian","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":15.252499999999998,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/italian","measuredAt":null,"stderr":0.7529950614047877,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":13.98285714285714,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.9470889879100105,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":13.4825,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.690680144857227,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"apertusv1105binstruct","providerId":"unknown","slug":"unknown-apertus-v1-1-0-5b-instruct","name":"Apertus V1.1 0.5B Instruct","provider":"Unknown","description":"Apertus V1.1 0.5B Instruct is a model. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":5,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-dutch:overall","benchmarkName":"EuroEval Dutch","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":22.451111111111114,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/dutch","measuredAt":null,"stderr":0.7406411970151603,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":19.538750000000004,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":0.6959851381315552,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":17.6425,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.5581442465886395,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":17.568571428571428,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.992543629843473,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-french:overall","benchmarkName":"EuroEval French","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":17.17,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/french","measuredAt":null,"stderr":0.5098284025042151,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-italian:overall","benchmarkName":"EuroEval Italian","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":15.215,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/italian","measuredAt":null,"stderr":0.6028240829628492,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-portuguese:overall","benchmarkName":"EuroEval Portuguese","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":14.85125,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/portuguese","measuredAt":null,"stderr":0.7465388888062028,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-spanish:overall","benchmarkName":"EuroEval Spanish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":13.301250000000001,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/spanish","measuredAt":null,"stderr":0.644217063574072,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"apertusv1115b","providerId":"unknown","slug":"unknown-apertus-v1-1-1-5b","name":"Apertus V1.1 1.5B","provider":"Unknown","description":"Apertus V1.1 1.5B is a model. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":5,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-french:overall","benchmarkName":"EuroEval French","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":30.368750000000002,"unit":"percent","family":"euroeval","index":13.423641558209965,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/french","measuredAt":null,"stderr":0.6266228929747142,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-dutch:overall","benchmarkName":"EuroEval Dutch","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":27.105555555555554,"unit":"percent","family":"euroeval","index":8.99623753674934,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/dutch","measuredAt":null,"stderr":0.6734360024402587,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-portuguese:overall","benchmarkName":"EuroEval Portuguese","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":26.222499999999997,"unit":"percent","family":"euroeval","index":7.798134333155097,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/portuguese","measuredAt":null,"stderr":0.9282678762081558,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":25.368750000000002,"unit":"percent","family":"euroeval","index":6.639792025433529,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":0.9145635912827494,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":20.29571428571429,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.9123908936728504,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-spanish:overall","benchmarkName":"EuroEval Spanish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":19.0225,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/spanish","measuredAt":null,"stderr":0.7441983942471255,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-italian:overall","benchmarkName":"EuroEval Italian","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":18.222499999999997,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/italian","measuredAt":null,"stderr":0.9158261297866535,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":17.645,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.718884769973603,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"apertusv1115binstruct","providerId":"unknown","slug":"unknown-apertus-v1-1-1-5b-instruct","name":"Apertus V1.1 1.5B Instruct","provider":"Unknown","description":"Apertus V1.1 1.5B Instruct is a model. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":5,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-french:overall","benchmarkName":"EuroEval French","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":27.043750000000006,"unit":"percent","family":"euroeval","index":8.912381618913642,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/french","measuredAt":null,"stderr":0.6304338783726648,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-portuguese:overall","benchmarkName":"EuroEval Portuguese","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":26.305,"unit":"percent","family":"euroeval","index":7.9100678504459125,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/portuguese","measuredAt":null,"stderr":0.7194007923265028,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":23.38375,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":0.9119810167980471,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-dutch:overall","benchmarkName":"EuroEval Dutch","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":22.237777777777776,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/dutch","measuredAt":null,"stderr":0.82747421727019,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-italian:overall","benchmarkName":"EuroEval Italian","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":18.715000000000003,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/italian","measuredAt":null,"stderr":0.6108421440928908,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-spanish:overall","benchmarkName":"EuroEval Spanish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":17.518749999999997,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/spanish","measuredAt":null,"stderr":0.6836597289587855,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":17.36428571428571,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.7535805686649489,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":15.418749999999998,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.6749340245535115,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"apertusv114b","providerId":"unknown","slug":"unknown-apertus-v1-1-4b","name":"Apertus V1.1 4B","provider":"Unknown","description":"Apertus V1.1 4B is a model. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":19.1071749248041,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-french:overall","benchmarkName":"EuroEval French","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":38.697500000000005,"unit":"percent","family":"euroeval","index":24.72383891743231,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/french","measuredAt":null,"stderr":0.6534212844712054,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-dutch:overall","benchmarkName":"EuroEval Dutch","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":38.39222222222222,"unit":"percent","family":"euroeval","index":24.309647215403345,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/dutch","measuredAt":null,"stderr":0.5929378623784918,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-portuguese:overall","benchmarkName":"EuroEval Portuguese","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":36.00625,"unit":"percent","family":"euroeval","index":21.072431906415392,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/portuguese","measuredAt":null,"stderr":0.7162347293311042,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":35.824285714285715,"unit":"percent","family":"euroeval","index":20.82554823949042,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.8689050111303,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-italian:overall","benchmarkName":"EuroEval Italian","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":33.29125,"unit":"percent","family":"euroeval","index":17.38880161011779,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/italian","measuredAt":null,"stderr":0.9139705616156354,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":32.29,"unit":"percent","family":"euroeval","index":16.0303357411793,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":1.098256430666354,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":31.113749999999996,"unit":"percent","family":"euroeval","index":14.434435138593642,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.7543302078665549,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-spanish:overall","benchmarkName":"EuroEval Spanish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":27.45125,"unit":"percent","family":"euroeval","index":9.465265355834916,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/spanish","measuredAt":null,"stderr":0.6370022566679022,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"apertusv114binstruct","providerId":"unknown","slug":"unknown-apertus-v1-1-4b-instruct","name":"Apertus V1.1 4B Instruct","provider":"Unknown","description":"Apertus V1.1 4B Instruct is a model. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":15.6885993209657,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-dutch:overall","benchmarkName":"EuroEval Dutch","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":35.711111111111116,"unit":"percent","family":"euroeval","index":20.671996343716785,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/dutch","measuredAt":null,"stderr":0.5027320421187012,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-french:overall","benchmarkName":"EuroEval French","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":35.63999999999999,"unit":"percent","family":"euroeval","index":20.57551492813951,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/french","measuredAt":null,"stderr":0.5657572359236778,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-portuguese:overall","benchmarkName":"EuroEval Portuguese","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":34.911249999999995,"unit":"percent","family":"euroeval","index":19.586768858737344,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/portuguese","measuredAt":null,"stderr":0.556516789953367,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":32.99375,"unit":"percent","family":"euroeval","index":16.985162562917587,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":0.9909055202187542,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-italian:overall","benchmarkName":"EuroEval Italian","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":31.082500000000003,"unit":"percent","family":"euroeval","index":14.392036079013799,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/italian","measuredAt":null,"stderr":0.7047572720447799,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":29.73125,"unit":"percent","family":"euroeval","index":12.558700742780971,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.6387389920773586,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":28.09142857142857,"unit":"percent","family":"euroeval","index":10.333840376370752,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.40695735122281024,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-spanish:overall","benchmarkName":"EuroEval Spanish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":27.810000000000002,"unit":"percent","family":"euroeval","index":9.952006559811629,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/spanish","measuredAt":null,"stderr":0.6775922446427498,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"apertusv1570b","providerId":"unknown","slug":"unknown-apertus-v1-5-70b","name":"Apertus V1.5 70B","provider":"Unknown","description":"Apertus V1.5 70B is a model. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":39.00469016511829,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-french:overall","benchmarkName":"EuroEval French","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":54.6625,"unit":"percent","family":"euroeval","index":46.384670475587455,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/french","measuredAt":null,"stderr":0.45874829699956376,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-dutch:overall","benchmarkName":"EuroEval Dutch","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":52.1888888888889,"unit":"percent","family":"euroeval","index":43.028549359511125,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/dutch","measuredAt":null,"stderr":0.6248170102488378,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-portuguese:overall","benchmarkName":"EuroEval Portuguese","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":51.918749999999996,"unit":"percent","family":"euroeval","index":42.66203304447638,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/portuguese","measuredAt":null,"stderr":0.4557514399318997,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":49.75,"unit":"percent","family":"euroeval","index":39.71953830963461,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":0.8345208205910742,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-italian:overall","benchmarkName":"EuroEval Italian","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":48.69625,"unit":"percent","family":"euroeval","index":38.289842020601974,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/italian","measuredAt":null,"stderr":0.6695030806501192,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-spanish:overall","benchmarkName":"EuroEval Spanish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":47.2875,"unit":"percent","family":"euroeval","index":36.37849241474222,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/spanish","measuredAt":null,"stderr":0.5848063247776994,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":42.370000000000005,"unit":"percent","family":"euroeval","index":29.7065763992566,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.75974296640377,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":40.798571428571435,"unit":"percent","family":"euroeval","index":27.574509403241155,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.5871689253858514,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"aristotle","providerId":"unknown","slug":"unknown-aristotle","name":"Aristotle","provider":"Unknown","description":"Aristotle is a model. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":76.84766487595212},"capabilityCounts":{"math":1},"benchmarks":[{"benchmarkId":"vals-ai:proof_bench:overall","benchmarkName":"ProofBench v1.1","category":"math","domain":null,"eligible":true,"ranked":true,"score":86,"unit":"percent","family":"vals-ai:proof_bench:overall","index":76.84766487595212,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/proof_bench","measuredAt":"2026-11-09","stderr":3.47,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":10,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"atria-dawn-preview","providerId":"shanghai-artificial-intelligence-laboratory","slug":"atria-dawn-preview","name":"Atria Dawn Preview","provider":"Shanghai Artificial Intelligence Laboratory","description":"Atria Dawn Preview is a reasoning model from Shanghai Artificial Intelligence Laboratory in the Atria Dawn family. It has 10 scored results in 2 categories. It has no index.","logoUrl":null,"releaseDate":"2026-09-14","contextWindowTokens":256000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":true,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":64.12951002946812,"coding":59.59533855271731,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"agentic":9,"coding":1},"benchmarks":[{"benchmarkId":"deepSearchQa","benchmarkName":"DeepSearchQA","category":"agentic","score":96,"unit":"percent","family":"deepSearchQa","index":70.15008439057824,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"browseComp","benchmarkName":"BrowseComp","category":"agentic","score":92.5,"unit":"percent","family":"browseComp","index":72.98582610553473,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"cyberGym","benchmarkName":"CyberGym","category":"agentic","score":86.5,"unit":"percent","family":"cyberGym","index":70.72473178036064,"aboveFit":false,"run":null,"source":"Zhun Wang, Tianneng Shi, Jingxuan He, Matthew Cai, Jialin Zhang, Dawn Song","sourceUrl":"https://www.cybergym.io/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mleBenchLite","benchmarkName":"MLE-Bench Lite","category":"agentic","score":86.2,"unit":"percent","family":"mleBenchLite","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"wideResearch","benchmarkName":"WideResearch","category":"agentic","score":81.9,"unit":"percent","family":"wideResearch","index":64.10531244057069,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"terminalBench21","benchmarkName":"Terminal-Bench 2.1","category":"agentic","score":78.3,"unit":"percent","family":"terminal-bench-2-1","index":66.93529624558916,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"bfclV4","benchmarkName":"Berkeley Function Calling Leaderboard v4","category":"agentic","score":77,"unit":"percent","family":"bfclV4","index":56.66140622354742,"aboveFit":false,"run":null,"source":"Shishir G. Patil et al.","sourceUrl":"https://gorilla.cs.berkeley.edu/leaderboard.html","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"swePro","benchmarkName":"SWE-bench Pro","category":"coding","score":59.6,"unit":"percent","family":"swePro","index":59.59533855271731,"aboveFit":false,"run":null,"source":"Xiang Deng, Jeff Da, Edwin Pan, Yannis Yiming He, Charles Ide, Kanak Garg, Niklas Lauffer, Andrew Park, Nitin Pasari, Chetan Rane, Karmini Sampath, Maya Krishnan, Srivatsa Kundurthy, Sean Hendryx, Zifan Wang, Chen Bo Calvin Zhang, Noah Jacobson, Bing Liu, Brad Kenstler","sourceUrl":"https://arxiv.org/abs/2509.16941","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"automationBench","benchmarkName":"AutomationBench","category":"agentic","score":53.8,"unit":"percent","family":"automationBench","index":84.4229434916949,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"jobBench","benchmarkName":"JobBench","category":"agentic","score":50.3,"unit":"percent","family":"jobBench","index":64.3144801424048,"aboveFit":false,"run":null,"source":"Yuetai Li, Yichen Feng, Zhangchen Xu, Zixian Ma, Kaiyuan Zheng, Fengqing Jiang, Xinghua Sun, Rulin Shao, Zichen Chen, Yue Huang, Xinyang Han, Brian Lee, Kayla Xu, Shenglai Zeng, Hang Hua, Xiangliang Zhang, Basel Alomair, Ranjay Krishna, Luke Zettlemoyer, Pang Wei Koh, Bhaskar Ramasubramanian, Luyao Niu, Xiang Yue, Radha Poovendran","sourceUrl":"https://arxiv.org/abs/2605.26329","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"tau3Bench","benchmarkName":"τ³-Bench Tool-Agent-User Evaluation","category":"agentic","score":41.2,"unit":"percent","family":"tau3Bench","index":26.865509444932435,"aboveFit":false,"run":null,"source":"Sierra Research","sourceUrl":"https://github.com/sierra-research/tau2-bench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":40,"benchmarkCount":10,"categoryCount":2,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"audio-flamingo-3-7b","providerId":"nvidia","slug":"audio-flamingo-3-7b","name":"Audio Flamingo 3 7B","provider":"NVIDIA","description":"Audio Flamingo 3 7B is a standard model from NVIDIA. It has no scored result.","logoUrl":null,"releaseDate":"2025-07-10","contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"tencent-auk","providerId":"tencent","slug":"tencent-auk","name":"AuK","provider":"Tencent","description":"AuK is a standard model from Tencent. It has no scored result.","logoUrl":null,"releaseDate":"2026-08-18","contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"autojev-27b","providerId":"denis-pplx","slug":"autojev-27b","name":"AutoJev-27B","provider":"denis-pplx","description":"AutoJev-27B is a standard model from denis-pplx. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":63.30760694074438,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"reasoning":1},"benchmarks":[{"benchmarkId":"jevBench15","benchmarkName":"jevBench15","category":"reasoning","score":19.54135920374164,"unit":"percent","family":"jevBench15","index":63.30760694074438,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":15,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"decision-index-autojev-27b","providerId":"denis-pplx","slug":"decision-index-autojev-27b","name":"AutoJev-27B (Decision Index reference)","provider":"denis-pplx","description":"AutoJev-27B (Decision Index reference) is a model from denis-pplx. It has no scored result.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"autojev-27b-rtxpro6000-a2","providerId":"denis-pplx","slug":"autojev-27b-rtxpro6000-a2","name":"AutoJev-27B (RTX PRO 6000)","provider":"denis-pplx","description":"AutoJev-27B (RTX PRO 6000) is a standard model from denis-pplx. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":63.27209645264295,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"reasoning":1},"benchmarks":[{"benchmarkId":"jevBench15","benchmarkName":"jevBench15","category":"reasoning","score":19.482719520484327,"unit":"percent","family":"jevBench15","index":63.27209645264295,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":15,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"autoloops-gemma-4-31b-it","providerId":"autoloops","slug":"autoloops-gemma-4-31b-it","name":"Autoloops Gemma 4 31B IT","provider":"Autoloops","description":"Autoloops Gemma 4 31B IT is a standard model from Autoloops. It has 2 scored results in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":false,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":73.28945811864992,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"reasoning":2},"benchmarks":[{"benchmarkId":"jevBench15","benchmarkName":"jevBench15","category":"reasoning","score":40.52814707638268,"unit":"percent","family":"jevBench15","index":76.01659579677974,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"jevBench14","benchmarkName":"jevBench14","category":"reasoning","score":29.962547814634828,"unit":"percent","family":"jevBench14","index":70.5623204405201,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":15,"benchmarkCount":2,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"aya238b","providerId":"unknown","slug":"unknown-aya-23-8b","name":"Aya 23 8B","provider":"Unknown","description":"Aya 23 8B is a model. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":6.084848905618379,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-dutch:overall","benchmarkName":"EuroEval Dutch","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":33.14888888888889,"unit":"percent","family":"euroeval","index":17.195650338698464,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/dutch","measuredAt":null,"stderr":0.6657985088005136,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-french:overall","benchmarkName":"EuroEval French","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":33.043749999999996,"unit":"percent","family":"euroeval","index":17.05300105824535,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/french","measuredAt":null,"stderr":0.6796403000852731,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-portuguese:overall","benchmarkName":"EuroEval Portuguese","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":28.357499999999998,"unit":"percent","family":"euroeval","index":10.694838083650637,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/portuguese","measuredAt":null,"stderr":0.9236205931008684,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":28.35571428571429,"unit":"percent","family":"euroeval","index":10.692415280246081,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.7678382057465502,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":21.563750000000002,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":0.9780648943193903,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":20.925,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.7153823278499407,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-italian:overall","benchmarkName":"EuroEval Italian","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":20.10125,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/italian","measuredAt":null,"stderr":0.8144399916507048,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-spanish:overall","benchmarkName":"EuroEval Spanish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":18.1225,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/spanish","measuredAt":null,"stderr":0.6625235844858657,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"ayaexpanse8b","providerId":"unknown","slug":"unknown-aya-expanse-8b","name":"Aya Expanse 8B","provider":"Unknown","description":"Aya Expanse 8B is a model. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":29.349091756913328,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-french:overall","benchmarkName":"EuroEval French","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":47.394999999999996,"unit":"percent","family":"euroeval","index":36.52434517969691,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/french","measuredAt":null,"stderr":0.6132852109744699,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-dutch:overall","benchmarkName":"EuroEval Dutch","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":45.48777777777778,"unit":"percent","family":"euroeval","index":33.936683463472306,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/dutch","measuredAt":null,"stderr":0.6784659530018555,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-italian:overall","benchmarkName":"EuroEval Italian","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":43.60125,"unit":"percent","family":"euroeval","index":31.377099346702792,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/italian","measuredAt":null,"stderr":0.7022430579934558,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-portuguese:overall","benchmarkName":"EuroEval Portuguese","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":43.38875,"unit":"percent","family":"euroeval","index":31.088785741559796,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/portuguese","measuredAt":null,"stderr":0.6214888876721771,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":40.82428571428572,"unit":"percent","family":"euroeval","index":27.609397772266863,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.5499202168478253,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-spanish:overall","benchmarkName":"EuroEval Spanish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":39.16375,"unit":"percent","family":"euroeval","index":25.35643288636371,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/spanish","measuredAt":null,"stderr":0.5413005172729839,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":36.08375,"unit":"percent","family":"euroeval","index":21.177581574173427,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":0.7711385656936112,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":35.44,"unit":"percent","family":"euroeval","index":20.304160946828453,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.7028657855522632,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":{"elo":1185.118217509901,"eloUpper":1192.036278259802,"eloLower":1178.2001567599998,"votes":9818,"rank":334,"rankUpper":null,"rankLower":null,"arenaName":"Aya Expanse 8B","organization":"cohere","updatedAt":"2026-11-09"},"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"baguettotronnothinking","providerId":"pleias","slug":"pleias-baguettotron-no-thinking","name":"Baguettotron No Thinking","provider":"PleIAs","description":"Baguettotron No Thinking is a model from PleIAs. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":5,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-dutch:overall","benchmarkName":"EuroEval Dutch","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":7.847777777777777,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":"no-thinking effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/dutch","measuredAt":null,"stderr":0.4864624127436216,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":5.0685714285714285,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":"no-thinking effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.261311449173946,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-french:overall","benchmarkName":"EuroEval French","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":4.673749999999999,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":"no-thinking effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/french","measuredAt":null,"stderr":0.3261158728121034,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-italian:overall","benchmarkName":"EuroEval Italian","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":4.66,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":"no-thinking effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/italian","measuredAt":null,"stderr":0.34524674147629547,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":4.182499999999999,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":"no-thinking effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.415733538459432,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-portuguese:overall","benchmarkName":"EuroEval Portuguese","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":4.12,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":"no-thinking effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/portuguese","measuredAt":null,"stderr":0.2885551290481595,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-spanish:overall","benchmarkName":"EuroEval Spanish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":4.016249999999999,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":"no-thinking effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/spanish","measuredAt":null,"stderr":0.307296680750053,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":3.3312500000000003,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":"no-thinking effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":0.29895651857753497,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"baichuan-audio","providerId":"baichuan","slug":"baichuan-audio","name":"Baichuan-Audio","provider":"Baichuan","description":"Baichuan-Audio is a standard model from Baichuan. It has no scored result.","logoUrl":null,"releaseDate":"2025-02-24","contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"baichuan-omni-1-5","providerId":"baichuan","slug":"baichuan-omni-1-5","name":"Baichuan-Omni 1.5","provider":"Baichuan","description":"Baichuan-Omni 1.5 is a standard model from Baichuan. It has no scored result.","logoUrl":null,"releaseDate":"2025-01-26","contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"decision-index-0-3-bekko-system-one-v0-17m","providerId":"hotchpotch","slug":"decision-index-0-3-bekko-system-one-v0-17m","name":"Bekko System One v0 17M","provider":"hotchpotch","description":"Bekko System One v0 17M is a model from hotchpotch. It has no scored result.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"decision-index-0-3-bekko-system-one-v0-400m","providerId":"hotchpotch","slug":"decision-index-0-3-bekko-system-one-v0-400m","name":"Bekko System One v0 400M","provider":"hotchpotch","description":"Bekko System One v0 400M is a model from hotchpotch. It has no scored result.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"decision-index-0-3-bekko-system-one-v0-68m","providerId":"hotchpotch","slug":"decision-index-0-3-bekko-system-one-v0-68m","name":"Bekko System One v0 68M","provider":"hotchpotch","description":"Bekko System One v0 68M is a model from hotchpotch. It has no scored result.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"nimble-9b","providerId":"bespoke-labs","slug":"nimble-9b","name":"Bespoke Nimble 9B","provider":"Bespoke Labs","description":"Bespoke Nimble 9B is a standard model from Bespoke Labs. It has 2 scored results in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":65.44592992679387,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"reasoning":2},"benchmarks":[{"benchmarkId":"jevBench15","benchmarkName":"jevBench15","category":"reasoning","score":31.8195434029124,"unit":"percent","family":"jevBench15","index":70.74291848563463,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"jevBench14","benchmarkName":"jevBench14","category":"reasoning","score":18.66373818414425,"unit":"percent","family":"jevBench14","index":60.148941367953114,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":15,"benchmarkCount":2,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"decision-index-0-3-bespoke-nimble-9b-v2","providerId":"bespokelabs","slug":"decision-index-0-3-bespoke-nimble-9b-v2","name":"Bespoke Nimble 9B v2","provider":"bespokelabs","description":"Bespoke Nimble 9B v2 is a model from bespokelabs. It has no scored result.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"decision-index-0-3-bespoke-nimble-9b-v3","providerId":"bespokelabs","slug":"decision-index-0-3-bespoke-nimble-9b-v3","name":"Bespoke Nimble 9B v3","provider":"bespokelabs","description":"Bespoke Nimble 9B v3 is a model from bespokelabs. It has no scored result.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"bev-bonsai-27b","providerId":"reza-sayar","slug":"bev-bonsai-27b","name":"Bev / Bonsai 27B","provider":"Reza Sayar","description":"Bev / Bonsai 27B is a standard model from Reza Sayar. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":61.02183796284754,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"reasoning":1},"benchmarks":[{"benchmarkId":"jevBench15","benchmarkName":"jevBench15","category":"reasoning","score":15.76679073032579,"unit":"percent","family":"jevBench15","index":61.02183796284754,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":15,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"bggpt7binstructv01","providerId":"unknown","slug":"unknown-bggpt-7b-instruct-v0-1","name":"BgGPT 7B Instruct V0.1","provider":"Unknown","description":"BgGPT 7B Instruct V0.1 is a model. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":14.274287833552037,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":30.995714285714286,"unit":"percent","family":"euroeval","index":14.274287833552037,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.59377897351435,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"bggpt7binstructv02","providerId":"unknown","slug":"unknown-bggpt-7b-instruct-v0-2","name":"BgGPT 7B Instruct V0.2","provider":"Unknown","description":"BgGPT 7B Instruct V0.2 is a model. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":19.27301581784644,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":34.68,"unit":"percent","family":"euroeval","index":19.27301581784644,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.7861998499628658,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"bggptgemma227bitv10","providerId":"unknown","slug":"unknown-bggpt-gemma-2-27b-it-v1-0","name":"BgGPT Gemma 2 27B IT V1.0","provider":"Unknown","description":"BgGPT Gemma 2 27B IT V1.0 is a model. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":5,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":10.732857142857144,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.9657301351193049,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"bggptgemma29bitv10","providerId":"unknown","slug":"unknown-bggpt-gemma-2-9b-it-v1-0","name":"BgGPT Gemma 2 9B IT V1.0","provider":"Unknown","description":"BgGPT Gemma 2 9B IT V1.0 is a model. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":37.37620285674126,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":48.02285714285714,"unit":"percent","family":"euroeval","index":37.37620285674126,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.602274600060098,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"bggptgemma312bit","providerId":"unknown","slug":"unknown-bggpt-gemma-3-12b-it","name":"BgGPT Gemma 3 12B IT","provider":"Unknown","description":"BgGPT Gemma 3 12B IT is a model. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":40.05679254354978,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":49.99857142857143,"unit":"percent","family":"euroeval","index":40.05679254354978,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.7258548795196248,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"bggptgemma327bit","providerId":"unknown","slug":"unknown-bggpt-gemma-3-27b-it","name":"BgGPT Gemma 3 27B IT","provider":"Unknown","description":"BgGPT Gemma 3 27B IT is a model. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":47.28837614548946,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":55.32857142857143,"unit":"percent","family":"euroeval","index":47.28837614548946,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.5748859691721658,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"bggptgemma34bit","providerId":"unknown","slug":"unknown-bggpt-gemma-3-4b-it","name":"BgGPT Gemma 3 4B IT","provider":"Unknown","description":"BgGPT Gemma 3 4B IT is a model. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":27.049245625131878,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":40.411428571428566,"unit":"percent","family":"euroeval","index":27.049245625131878,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.5992682612817,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"bielik15bv30instruct","providerId":"unknown","slug":"unknown-bielik-1-5b-v3-0-instruct","name":"Bielik 1.5B V3.0 Instruct","provider":"Unknown","description":"Bielik 1.5B V3.0 Instruct is a model. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":10.024932942288963,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-french:overall","benchmarkName":"EuroEval French","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":35.9825,"unit":"percent","family":"euroeval","index":21.040208621134703,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/french","measuredAt":null,"stderr":0.3109637237685451,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":34.145714285714284,"unit":"percent","family":"euroeval","index":18.548113039201187,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.3112384740919767,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-portuguese:overall","benchmarkName":"EuroEval Portuguese","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":31.0725,"unit":"percent","family":"euroeval","index":14.378468379948245,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/portuguese","measuredAt":null,"stderr":0.34480066705271906,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-italian:overall","benchmarkName":"EuroEval Italian","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":28.014999999999997,"unit":"percent","family":"euroeval","index":10.230144390655447,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/italian","measuredAt":null,"stderr":0.34587976595921305,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-spanish:overall","benchmarkName":"EuroEval Spanish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":27.7125,"unit":"percent","family":"euroeval","index":9.81972149392248,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/spanish","measuredAt":null,"stderr":0.38836596787051253,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":27.20125,"unit":"percent","family":"euroeval","index":9.126072879196094,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":0.31022421327162714,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-dutch:overall","benchmarkName":"EuroEval Dutch","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":26.624444444444446,"unit":"percent","family":"euroeval","index":8.343480459484418,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/dutch","measuredAt":null,"stderr":1.4149798236139512,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":26.145000000000003,"unit":"percent","family":"euroeval","index":7.692984665397077,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.5958358100181627,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"bielik11bv30instruct","providerId":"unknown","slug":"unknown-bielik-11b-v3-0-instruct","name":"Bielik 11B V3.0 Instruct","provider":"Unknown","description":"Bielik 11B V3.0 Instruct is a model. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":40.42093989525559,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-french:overall","benchmarkName":"EuroEval French","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":55.01625,"unit":"percent","family":"euroeval","index":46.86462783003139,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/french","measuredAt":null,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":54.017500000000005,"unit":"percent","family":"euroeval","index":45.509553885859305,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-dutch:overall","benchmarkName":"EuroEval Dutch","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":53.78333333333333,"unit":"percent","family":"euroeval","index":45.1918435994076,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/dutch","measuredAt":null,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":52.05142857142857,"unit":"percent","family":"euroeval","index":42.84204733743542,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-spanish:overall","benchmarkName":"EuroEval Spanish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":48.482499999999995,"unit":"percent","family":"euroeval","index":37.999832453075776,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/spanish","measuredAt":null,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-italian:overall","benchmarkName":"EuroEval Italian","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":47.635,"unit":"percent","family":"euroeval","index":36.849969957270176,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/italian","measuredAt":null,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-portuguese:overall","benchmarkName":"EuroEval Portuguese","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":44.74,"unit":"percent","family":"euroeval","index":32.922121077792625,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/portuguese","measuredAt":null,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":42.79875,"unit":"percent","family":"euroeval","index":30.28829149669217,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"bielikminitron7bv30instruct","providerId":"unknown","slug":"unknown-bielik-minitron-7b-v3-0-instruct","name":"Bielik Minitron 7B V3.0 Instruct","provider":"Unknown","description":"Bielik Minitron 7B V3.0 Instruct is a model. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":36.397148000957344,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-french:overall","benchmarkName":"EuroEval French","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":52.607499999999995,"unit":"percent","family":"euroeval","index":43.59650831761633,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/french","measuredAt":null,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-dutch:overall","benchmarkName":"EuroEval Dutch","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":51.42111111111111,"unit":"percent","family":"euroeval","index":41.98685157570033,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/dutch","measuredAt":null,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":48.2325,"unit":"percent","family":"euroeval","index":37.66063997643697,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":47.96999999999999,"unit":"percent","family":"euroeval","index":37.30448787596619,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-spanish:overall","benchmarkName":"EuroEval Spanish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":46.6325,"unit":"percent","family":"euroeval","index":35.4898081259485,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/spanish","measuredAt":null,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-portuguese:overall","benchmarkName":"EuroEval Portuguese","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":43.3925,"unit":"percent","family":"euroeval","index":31.09387362870937,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/portuguese","measuredAt":null,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":43.34,"unit":"percent","family":"euroeval","index":31.02264320861523,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-italian:overall","benchmarkName":"EuroEval Italian","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":40.459999999999994,"unit":"percent","family":"euroeval","index":27.11514587773599,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/italian","measuredAt":null,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"decision-index-0-3-blink-v0-3-26b-a4b-nvfp4","providerId":"pixilabai","slug":"decision-index-0-3-blink-v0-3-26b-a4b-nvfp4","name":"Blink v0.3 26B-A4B (NVFP4)","provider":"PixilabAI","description":"Blink v0.3 26B-A4B (NVFP4) is a model from PixilabAI. It has no scored result.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"blsp-7b","providerId":"blsp-authors","slug":"blsp-7b","name":"BLSP 7B","provider":"BLSP authors","description":"BLSP 7B is a standard model from BLSP authors. It has no scored result.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"borealis12b","providerId":"unknown","slug":"unknown-borealis-12b","name":"Borealis 12B","provider":"Unknown","description":"Borealis 12B is a model. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":39.99089229094567,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":49.95,"unit":"percent","family":"euroeval","index":39.99089229094567,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":0.8014976216433833,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"borealis1b","providerId":"unknown","slug":"unknown-borealis-1b","name":"Borealis 1B","provider":"Unknown","description":"Borealis 1B is a model. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":5,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":18.7525,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":0.6973207655017883,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"borealis270m","providerId":"unknown","slug":"unknown-borealis-270m","name":"Borealis 270M","provider":"Unknown","description":"Borealis 270M is a model. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":5,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":5.733750000000001,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":1.0353501823054845,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"borealis27b","providerId":"unknown","slug":"unknown-borealis-27b","name":"Borealis 27B","provider":"Unknown","description":"Borealis 27B is a model. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":45.17544929637005,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":53.771249999999995,"unit":"percent","family":"euroeval","index":45.17544929637005,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":0.6791619376113477,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"borealis4b","providerId":"unknown","slug":"unknown-borealis-4b","name":"Borealis 4B","provider":"Unknown","description":"Borealis 4B is a model. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":22.08661741156547,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":36.753750000000004,"unit":"percent","family":"euroeval","index":22.08661741156547,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":0.69154491900382,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"borealisopen12b","providerId":"unknown","slug":"unknown-borealis-open-12b","name":"Borealis Open 12B","provider":"Unknown","description":"Borealis Open 12B is a model. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":40.31312514375256,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":50.18750000000001,"unit":"percent","family":"euroeval","index":40.31312514375256,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":0.8406247676579605,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"borealisopen1b","providerId":"unknown","slug":"unknown-borealis-open-1b","name":"Borealis Open 1B","provider":"Unknown","description":"Borealis Open 1B is a model. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":5,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":18.166250000000005,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":0.6818139867295185,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"borealisopen270m","providerId":"unknown","slug":"unknown-borealis-open-270m","name":"Borealis Open 270M","provider":"Unknown","description":"Borealis Open 270M is a model. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":5,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":6.8637500000000005,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":0.9943315908186766,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"borealisopen27b","providerId":"unknown","slug":"unknown-borealis-open-27b","name":"Borealis Open 27B","provider":"Unknown","description":"Borealis Open 27B is a model. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":45.155097747771734,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":53.75625000000001,"unit":"percent","family":"euroeval","index":45.155097747771734,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":0.7216138596922873,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"borealisopen4b","providerId":"unknown","slug":"unknown-borealis-open-4b","name":"Borealis Open 4B","provider":"Unknown","description":"Borealis Open 4B is a model. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":18.17742411830305,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":33.8725,"unit":"percent","family":"euroeval","index":18.17742411830305,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":0.7566951169394447,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"bosun-v31-0-6b","providerId":"clause-logic","slug":"bosun-v31-0-6b","name":"Bosun v3.1 0.6B","provider":"Clause Logic","description":"Bosun v3.1 0.6B is a standard model from Clause Logic. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":52.987571681285445,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"reasoning":1},"benchmarks":[{"benchmarkId":"jevBench15","benchmarkName":"jevBench15","category":"reasoning","score":2.499532093665368,"unit":"percent","family":"jevBench15","index":52.987571681285445,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":15,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"decision-index-0-3-bosun-v3-1-1-7b","providerId":"hanno-labs","slug":"decision-index-0-3-bosun-v3-1-1-7b","name":"Bosun v3.1 1.7B","provider":"Hanno-Labs","description":"Bosun v3.1 1.7B is a model from Hanno-Labs. It has no scored result.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"btl-3","providerId":"bad-theory-labs","slug":"btl-3","name":"BTL-3","provider":"Bad Theory Labs","description":"BTL-3 is a reasoning model from Bad Theory Labs in the BTL family. It has 3 scored results in 2 categories. It has no index.","logoUrl":null,"releaseDate":"2026-07-20","contextWindowTokens":262000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":true,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":67.34670036080716,"coding":46.03963260328297,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"coding":2,"agentic":1},"benchmarks":[{"benchmarkId":"humaneval","benchmarkName":"Evaluating Large Language Models Trained on Code","category":"coding","score":95.12,"unit":"percent","family":"humaneval","index":35.327059441352134,"aboveFit":false,"run":null,"source":"Mark Chen, Jerry Tworek, Heewoo Jun, Qiming Yuan, Henrique Ponde de Oliveira Pinto, Jared Kaplan, Harri Edwards, Yuri Burda, Nicholas Joseph, Greg Brockman, Alex Ray, Raul Puri, Gretchen Krueger, Michael Petrov, Heidy Khlaaf, Girish Sastry, Pamela Mishkin, Brooke Chan, Scott Gray, Nick Ryder, Mikhail Pavlov, Alethea Power, Lukasz Kaiser, Mohammad Bavarian, Clemens Winter, Philippe Tillet, Felipe Petroski Such, Dave Cummings, Matthias Plappert, Fotios Chantzis, Elizabeth Barnes, Ariel Herbert-Voss, William Hebgen Guss, Alex Nichol, Alex Paino, Nikolas Tezak, Jie Tang, Igor Babuschkin, Suchir Balaji, Shantanu Jain, William Saunders, Christopher Hesse, Andrew N. Carr, Jan Leike, Josh Achiam, Vedant Misra, Evan Morikawa, Alec Radford, Matthew Knight, Miles Brundage, Mira Murati, Katie Mayer, Peter Welinder, Bob McGrew, Dario Amodei, Sam McCandlish, Ilya Sutskever, Wojciech Zaremba","sourceUrl":"https://arxiv.org/abs/2107.03374","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"bfclV4","benchmarkName":"Berkeley Function Calling Leaderboard v4","category":"agentic","score":88.5,"unit":"percent","family":"bfclV4","index":67.34670036080716,"aboveFit":false,"run":null,"source":"Shishir G. Patil et al.","sourceUrl":"https://gorilla.cs.berkeley.edu/leaderboard.html","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"liveCodeBenchV6","benchmarkName":"LiveCodeBench v6","category":"coding","score":88.1,"unit":"percent","family":"liveCodeBenchV6","index":56.752205765213795,"aboveFit":false,"run":null,"source":"LiveCodeBench maintainers","sourceUrl":"https://github.com/LiveCodeBench/LiveCodeBench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":40,"benchmarkCount":3,"categoryCount":2,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"btl-4","providerId":"bad-theory-labs","slug":"btl-4","name":"BTL-4","provider":"Bad Theory Labs","description":"BTL-4 is a reasoning model from Bad Theory Labs in the BTL family. It has 3 scored results in 2 categories. It has no index.","logoUrl":null,"releaseDate":"2026-08-05","contextWindowTokens":262000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":true,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":53.40936018177271,"coding":48.870695078872316,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"coding":2,"agentic":1},"benchmarks":[{"benchmarkId":"sweVerified","benchmarkName":"Software Engineering Benchmark Verified","category":"coding","score":78.4,"unit":"percent","family":"swe-bench-verified","index":59.052441627713094,"aboveFit":false,"run":null,"source":"Carlos E. Jimenez, John Yang, Alexander Wettig, Shunyu Yao, Kexin Pei, Ofir Press, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2310.06770","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"bfclV4","benchmarkName":"Berkeley Function Calling Leaderboard v4","category":"agentic","score":73.5,"unit":"percent","family":"bfclV4","index":53.40936018177271,"aboveFit":false,"run":null,"source":"Shishir G. Patil et al.","sourceUrl":"https://gorilla.cs.berkeley.edu/leaderboard.html","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"liveCodeBenchV6","benchmarkName":"LiveCodeBench v6","category":"coding","score":66.1,"unit":"percent","family":"liveCodeBenchV6","index":38.68894853003154,"aboveFit":false,"run":null,"source":"LiveCodeBench maintainers","sourceUrl":"https://github.com/LiveCodeBench/LiveCodeBench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":40,"benchmarkCount":3,"categoryCount":2,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"btl-4-compact","providerId":"bad-theory-labs","slug":"btl-4-compact","name":"BTL-4 Compact","provider":"Bad Theory Labs","description":"BTL-4 Compact is a reasoning model from Bad Theory Labs in the BTL family. It has no scored result.","logoUrl":null,"releaseDate":"2026-08-05","contextWindowTokens":262000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":true,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"canary-180m-flash","providerId":"nvidia","slug":"canary-180m-flash","name":"Canary 180M Flash","provider":"NVIDIA","description":"Canary 180M Flash is a standard model from NVIDIA. It has no scored result.","logoUrl":null,"releaseDate":"2025-03-12","contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"celeris-1-decision","providerId":"celeris","slug":"celeris-1-decision","name":"Celeris-1 Decision","provider":"Celeris","description":"Celeris-1 Decision is a model from Celeris. It has no scored result.","logoUrl":null,"releaseDate":"2026-10-08","contextWindowTokens":null,"modalities":["text"],"openWeights":false,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":0.04,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"celeris-1-1-decision","providerId":"celeris","slug":"celeris-1-1-decision","name":"Celeris-1.1 Decision","provider":"Celeris","description":"Celeris-1.1 Decision is a model from Celeris in the Celeris-1 Decision family. It has no scored result.","logoUrl":null,"releaseDate":"2026-11-05","contextWindowTokens":131000,"modalities":["text"],"openWeights":false,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":0.004,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"certo","providerId":"altslate-labs","slug":"certo","name":"Certo v1","provider":"AltSlate Labs","description":"Certo v1 is a standard model from AltSlate Labs. It has 2 scored results in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":47.210857056580174,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"reasoning":2},"benchmarks":[{"benchmarkId":"jevBench14","benchmarkName":"jevBench14","category":"reasoning","score":5.344170110166062e-7,"unit":"percent","family":"jevBench14","index":42.94778643096958,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"jevBench15","benchmarkName":"jevBench15","category":"reasoning","score":0,"unit":"percent","family":"jevBench15","index":51.47392768219078,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":15,"benchmarkCount":2,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"chirp3","providerId":"unknown","slug":"unknown-chirp-3","name":"Chirp 3","provider":"Unknown","description":"Chirp 3 is a model. It has no scored result.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"classifier-dev-fast","providerId":"classifier-dev","slug":"classifier-dev-fast","name":"classifier.dev (fast)","provider":"classifier.dev","description":"classifier.dev (fast) is a standard model from classifier.dev. It has 2 scored results in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":false,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":95,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"reasoning":2},"benchmarks":[{"benchmarkId":"jevBench15","benchmarkName":"jevBench15","category":"reasoning","score":74.65566746650249,"unit":"percent","family":"jevBench15","index":95,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"jevBench14","benchmarkName":"jevBench14","category":"reasoning","score":70.82340862727794,"unit":"percent","family":"jevBench14","index":95,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":15,"benchmarkCount":2,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"claude2","providerId":"anthropic","slug":"anthropic-claude-2","name":"Claude 2","provider":"Anthropic","description":"Claude 2 is a model from Anthropic. It has 5 scored results in 3 categories. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":5.887347723048848,"reasoning":null,"multimodal":null,"knowledge":8.997471857228447,"multilingual":null,"instruction":null,"math":7.686787995448732},"capabilityCounts":{"knowledge":1,"math":2,"coding":2},"benchmarks":[{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":34.659090909090914,"unit":"percent","family":"gpqa-diamond","index":8.997471857228447,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.5076255983551476,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-math-level-5:overall","benchmarkName":"MATH level 5","category":"math","domain":null,"eligible":true,"ranked":true,"score":11.725830815709969,"unit":"percent","family":"epoch-ai:epoch-math-level-5:overall","index":5,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0.6595670913969979,"costPerTest":null,"latency":null},{"benchmarkId":"swe-bench:swe-bench-verified:overall","benchmarkName":"SWE-bench Verified","category":"coding","domain":null,"eligible":true,"ranked":true,"score":4.4,"unit":"percent","family":"swe-bench-verified","index":5,"aboveFit":false,"run":"RAG baseline","source":"SWE-bench team","sourceUrl":"https://www.swebench.com/","measuredAt":"2026-02-26","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"swebench-experiments:swe-bench-verified:overall","benchmarkName":"SWE-bench Verified","category":"coding","domain":null,"eligible":true,"ranked":true,"score":4.3999999999999995,"unit":"percent","family":"swe-bench-verified","index":5,"aboveFit":false,"run":"RAG baseline","source":"SWE-bench team","sourceUrl":"https://github.com/SWE-bench/experiments/tree/main/evaluation/verified","measuredAt":"2026-09-01","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"swe-bench:swe-bench-lite:overall","benchmarkName":"SWE-bench Lite","category":"coding","domain":null,"eligible":true,"ranked":true,"score":3,"unit":"percent","family":"swe-bench-lite","index":6.774695446097709,"aboveFit":false,"run":"RAG baseline","source":"SWE-bench team","sourceUrl":"https://www.swebench.com/","measuredAt":"2025-09-11","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"swebench-experiments:swe-bench-lite:overall","benchmarkName":"SWE-bench Lite","category":"coding","domain":null,"eligible":true,"ranked":true,"score":3,"unit":"percent","family":"swe-bench-lite","index":6.774695446097709,"aboveFit":false,"run":"RAG baseline","source":"SWE-bench team","sourceUrl":"https://github.com/SWE-bench/experiments/tree/main/evaluation/lite","measuredAt":"2025-09-11","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":2.5,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":10.37357599089745,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.5179897658722632,"costPerTest":null,"latency":null},{"benchmarkId":"swebench-experiments:swe-bench-test:overall","benchmarkName":"SWE-bench (full test split)","category":"coding","domain":null,"eligible":true,"ranked":true,"score":1.9616390584132521,"unit":"percent","family":"swebench-experiments:swe-bench-test:overall","index":null,"aboveFit":false,"run":"RAG baseline","source":"SWE-bench team","sourceUrl":"https://github.com/SWE-bench/experiments/tree/main/evaluation/test","measuredAt":"2025-12-19","stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":45,"benchmarkCount":5,"categoryCount":3,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:swe-bench-team","name":"SWE-bench team","url":"https://www.swebench.com","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"claude21","providerId":"anthropic","slug":"anthropic-claude-2-1","name":"Claude 2.1","provider":"Anthropic","description":"Claude 2.1 is a model from Anthropic. It has 2 scored results in 2 categories. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":7.434109343160621,"multilingual":null,"instruction":null,"math":10.050357076601287},"capabilityCounts":{"knowledge":1,"math":1},"benchmarks":[{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":32.95454545454545,"unit":"percent","family":"gpqa-diamond","index":7.434109343160621,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.324885083265289,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":1.9444444444444444,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":10.050357076601287,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.0477515876628662,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":25,"benchmarkCount":2,"categoryCount":2,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"claude3sonnet","providerId":"anthropic","slug":"anthropic-claude-3-sonnet","name":"Claude 3 Sonnet","provider":"Anthropic","description":"Claude 3 Sonnet is a model from Anthropic. It has 3 scored results in 2 categories. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":14.440289498797874,"multilingual":null,"instruction":null,"math":9.099825408531458},"capabilityCounts":{"knowledge":1,"math":2},"benchmarks":[{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":40.59343434343434,"unit":"percent","family":"gpqa-diamond","index":14.440289498797874,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.3920223491748827,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-math-level-5:overall","benchmarkName":"MATH level 5","category":"math","domain":null,"eligible":true,"ranked":true,"score":18.174093655589125,"unit":"percent","family":"epoch-ai:epoch-math-level-5:overall","index":7.826074826165467,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0.7981817344047291,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":2.5,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":10.37357599089745,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.950297827553939,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":{"elo":1218.0455543286646,"eloUpper":1221.9972131402365,"eloLower":1214.0938955170927,"votes":109284,"rank":316,"rankUpper":null,"rankLower":null,"arenaName":"Claude 3 Sonnet-20240229","organization":"anthropic","updatedAt":"2026-11-09"},"quality":{"coveragePercent":25,"benchmarkCount":3,"categoryCount":2,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"claude35sonnetoctober2024","providerId":"anthropic","slug":"anthropic-claude-3-5-sonnet-october-2024","name":"Claude 3.5 Sonnet (October 2024)","provider":"Anthropic","description":"Claude 3.5 Sonnet (October 2024) is a model from Anthropic. It has 5 scored results in 2 categories. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":27.931528972049758,"multilingual":null,"instruction":null,"math":28.86304158924125},"capabilityCounts":{"math":4,"knowledge":1},"benchmarks":[{"benchmarkId":"epoch-ai:epoch-math-level-5:overall","benchmarkName":"MATH level 5","category":"math","domain":null,"eligible":true,"ranked":true,"score":56.948640483383684,"unit":"percent","family":"epoch-ai:epoch-math-level-5:overall","index":26.66101948488849,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.1091284283577039,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":55.3030303030303,"unit":"percent","family":"gpqa-diamond","index":27.931528972049758,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.820809014707885,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":8.472222222222223,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":13.848179319581163,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.636395685590248,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-2025-02-28-private:overall","benchmarkName":"FrontierMath-2025-02-28-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":2.0689655172413794,"unit":"percent","family":"epoch-ai:epoch-frontiermath-2025-02-28-private:overall","index":31.740609825329734,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0.8373130807525473,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-public:overall","benchmarkName":"FrontierMath-Tier-4-2025-07-01-Public","category":"math","domain":null,"eligible":false,"ranked":true,"score":0,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-public:overall","index":null,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-private:overall","benchmarkName":"FrontierMath-Tier-4-2025-07-01-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":0,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-private:overall","index":43.20235772716562,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-2025-02-28-public:overall","benchmarkName":"FrontierMath-2025-02-28-Public","category":"math","domain":null,"eligible":false,"ranked":true,"score":0,"unit":"percent","family":"epoch-ai:epoch-frontiermath-2025-02-28-public:overall","index":null,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":25,"benchmarkCount":5,"categoryCount":2,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"claude37","providerId":"anthropic","slug":"anthropic-claude-3-7","name":"Claude 3.7","provider":"Anthropic","description":"Claude 3.7 is a model from Anthropic. It has 2 scored results in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":38.12183696808693,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"reasoning":2},"benchmarks":[{"benchmarkId":"arc-prize:v1-semi-private:overall","benchmarkName":"ARC-AGI-1 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":21.2,"unit":"percent","family":"arc-prize:v1-semi-private:overall","index":36.73376424324801,"aboveFit":false,"run":"8k effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.21,"latency":null},{"benchmarkId":"arc-prize:v2-public-eval:overall","benchmarkName":"ARC-AGI-2 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":0.8,"unit":"percent","family":"arc-prize:v2-public-eval:overall","index":null,"aboveFit":false,"run":"16k effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.57,"latency":null},{"benchmarkId":"arc-prize:v2-semi-private:overall","benchmarkName":"ARC-AGI-2 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":0.7000000000000001,"unit":"percent","family":"arc-prize:v2-semi-private:overall","index":39.50990969292585,"aboveFit":false,"run":"16k effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.51,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":15,"benchmarkCount":2,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:arc-prize-foundation","name":"ARC Prize Foundation","url":"https://arcprize.org","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"claude37sonnetwreviewheavy","providerId":"anthropic","slug":"anthropic-claude-3-7-sonnet-w-review-heavy","name":"Claude 3.7 Sonnet w/ Review Heavy","provider":"Anthropic","description":"Claude 3.7 Sonnet w/ Review Heavy is a model from Anthropic. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":47.251520383174025,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"coding":1},"benchmarks":[{"benchmarkId":"swe-bench:swe-bench-verified:overall","benchmarkName":"SWE-bench Verified","category":"coding","domain":null,"eligible":true,"ranked":true,"score":62.4,"unit":"percent","family":"swe-bench-verified","index":47.251520383174025,"aboveFit":false,"run":"SWE-agent","source":"SWE-bench team","sourceUrl":"https://www.swebench.com/","measuredAt":"2026-02-26","stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":20,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:swe-bench-team","name":"SWE-bench team","url":"https://www.swebench.com","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"claude-4-opus","providerId":"anthropic","slug":"claude-4-opus","name":"Claude 4 Opus","provider":"Anthropic","description":"Claude 4 Opus is a standard model from Anthropic. It has 7 scored results in 5 categories. It has no index.","logoUrl":null,"releaseDate":"2025-05-22","contextWindowTokens":200000,"modalities":["text"],"openWeights":false,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":53.15198100544356,"reasoning":null,"multimodal":41.80891126163407,"knowledge":39.81804627863298,"multilingual":null,"instruction":29.487939014182373,"math":17.40223693334525},"capabilityCounts":{"knowledge":3,"multimodal":1,"coding":1,"instruction":1,"math":1},"benchmarks":[{"benchmarkId":"mmmlu","benchmarkName":"MMMLU","category":"knowledge","score":87.4,"unit":"percent","family":"mmmlu","index":45.1429598338668,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gpqa","benchmarkName":"Graduate-Level Google-Proof Q&A","category":"knowledge","score":74.9,"unit":"percent","family":"gpqa-diamond","index":45.90533408934144,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmmu","benchmarkName":"Massive Multi-discipline Multimodal Understanding","category":"multimodal","score":73.7,"unit":"percent","family":"mmmu","index":41.80891126163407,"aboveFit":false,"run":null,"source":"MMMU authors","sourceUrl":"https://arxiv.org/abs/2401.05508","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"sweVerified","benchmarkName":"Software Engineering Benchmark Verified","category":"coding","score":72.5,"unit":"percent","family":"swe-bench-verified","index":54.70085191878931,"aboveFit":false,"run":null,"source":"Carlos E. Jimenez, John Yang, Alexander Wettig, Shunyu Yao, Kexin Pei, Ofir Press, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2310.06770","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"swebench-experiments:swe-bench-verified:overall","benchmarkName":"SWE-bench Verified","category":"coding","domain":null,"eligible":true,"ranked":true,"score":70.4,"unit":"percent","family":"swe-bench-verified","index":53.15198100544356,"aboveFit":false,"run":"Tools","source":"SWE-bench team","sourceUrl":"https://github.com/SWE-bench/experiments/tree/main/evaluation/verified","measuredAt":"2026-09-01","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":70.1,"unit":"percent","family":"gpqa-diamond","index":41.50290524972645,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"swe-bench:swe-bench-verified:overall","benchmarkName":"SWE-bench Verified","category":"coding","domain":null,"eligible":true,"ranked":true,"score":67.6,"unit":"percent","family":"swe-bench-verified","index":51.08681978764922,"aboveFit":false,"run":"mini-SWE-agent","source":"SWE-bench team","sourceUrl":"https://www.swebench.com/","measuredAt":"2026-02-26","stderr":null,"costPerTest":1.131270447,"latency":null},{"benchmarkId":"aaIfBench","benchmarkName":"Artificial Analysis IFBench","category":"instruction","score":43.3,"unit":"percent","family":"ifbench-family","index":29.487939014182373,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aime2025","benchmarkName":"American Invitational Mathematics Examination 2025","category":"math","score":33.9,"unit":"percent","family":"aime2025","index":17.40223693334525,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":6.2,"unit":"percent","family":"hle","index":30.6070593324982,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":60,"benchmarkCount":7,"categoryCount":5,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:swe-bench-team","name":"SWE-bench team","url":"https://github.com","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"claude-4-1-opus","providerId":"anthropic","slug":"claude-4-1-opus","name":"Claude 4.1 Opus","provider":"Anthropic","description":"Claude 4.1 Opus is a standard model from Anthropic. It has 2 scored results in 2 categories. It has no index.","logoUrl":null,"releaseDate":"2025-08-05","contextWindowTokens":200000,"modalities":["text"],"openWeights":false,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":48.606460862576476,"coding":56.1759670743567,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"coding":1,"agentic":1},"benchmarks":[{"benchmarkId":"sweVerified","benchmarkName":"Software Engineering Benchmark Verified","category":"coding","score":74.5,"unit":"percent","family":"swe-bench-verified","index":56.1759670743567,"aboveFit":false,"run":null,"source":"Carlos E. Jimenez, John Yang, Alexander Wettig, Shunyu Yao, Kexin Pei, Ofir Press, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2310.06770","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"jobBench","benchmarkName":"JobBench","category":"agentic","score":21.9,"unit":"percent","family":"jobBench","index":48.606460862576476,"aboveFit":false,"run":null,"source":"Yuetai Li, Yichen Feng, Zhangchen Xu, Zixian Ma, Kaiyuan Zheng, Fengqing Jiang, Xinghua Sun, Rulin Shao, Zichen Chen, Yue Huang, Xinyang Han, Brian Lee, Kayla Xu, Shenglai Zeng, Hang Hua, Xiangliang Zhang, Basel Alomair, Ranjay Krishna, Luke Zettlemoyer, Pang Wei Koh, Bhaskar Ramasubramanian, Luyao Niu, Xiang Yue, Radha Poovendran","sourceUrl":"https://arxiv.org/abs/2605.26329","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":15,"outputPerMillion":75,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":29,"timeToFirstToken":1.66,"source":"BenchLM","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":40,"benchmarkCount":2,"categoryCount":2,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"claude45haiku","providerId":"anthropic","slug":"anthropic-claude-4-5-haiku","name":"Claude 4.5 Haiku","provider":"Anthropic","description":"Claude 4.5 Haiku is a model from Anthropic. It has no scored result.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[{"benchmarkId":"swe-bench:swe-bench-multilingual:overall","benchmarkName":"SWE-bench Multilingual","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":64.7,"unit":"percent","family":"swe-bench-multilingual","index":null,"aboveFit":false,"run":"mini-SWE-agent","source":"SWE-bench team","sourceUrl":"https://www.swebench.com/","measuredAt":"2026-02-20","stderr":null,"costPerTest":0.3794902823333333,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:swe-bench-team","name":"SWE-bench team","url":"https://www.swebench.com","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"claude45opus","providerId":"anthropic","slug":"anthropic-claude-4-5-opus","name":"Claude 4.5 Opus","provider":"Anthropic","description":"Claude 4.5 Opus is a model from Anthropic. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":56.98728040991875,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"coding":1},"benchmarks":[{"benchmarkId":"swebench-experiments:swe-bench-verified:overall","benchmarkName":"SWE-bench Verified","category":"coding","domain":null,"eligible":true,"ranked":true,"score":76.8,"unit":"percent","family":"swe-bench-verified","index":57.87234950325919,"aboveFit":false,"run":"high effort · mini-SWE-agent","source":"SWE-bench team","sourceUrl":"https://github.com/SWE-bench/experiments/tree/main/evaluation/verified","measuredAt":"2026-09-01","stderr":null,"costPerTest":0.753907997,"latency":null},{"benchmarkId":"swe-bench:swe-bench-verified:overall","benchmarkName":"SWE-bench Verified","category":"coding","domain":null,"eligible":true,"ranked":true,"score":74.4,"unit":"percent","family":"swe-bench-verified","index":56.102211316578334,"aboveFit":false,"run":"medium effort · mini-SWE-agent","source":"SWE-bench team","sourceUrl":"https://www.swebench.com/","measuredAt":"2026-02-26","stderr":null,"costPerTest":0.721244997,"latency":null},{"benchmarkId":"swe-bench:swe-bench-multilingual:overall","benchmarkName":"SWE-bench Multilingual","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":70.7,"unit":"percent","family":"swe-bench-multilingual","index":null,"aboveFit":false,"run":"mini-SWE-agent","source":"SWE-bench team","sourceUrl":"https://www.swebench.com/","measuredAt":"2026-02-20","stderr":null,"costPerTest":0.8335529758333333,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":20,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:swe-bench-team","name":"SWE-bench team","url":"https://github.com","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"claude45sonnet","providerId":"anthropic","slug":"anthropic-claude-4-5-sonnet","name":"Claude 4.5 Sonnet","provider":"Anthropic","description":"Claude 4.5 Sonnet is a model from Anthropic. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":53.299492521000296,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"coding":1},"benchmarks":[{"benchmarkId":"swe-bench:swe-bench-verified:overall","benchmarkName":"SWE-bench Verified","category":"coding","domain":null,"eligible":true,"ranked":true,"score":70.6,"unit":"percent","family":"swe-bench-verified","index":53.299492521000296,"aboveFit":false,"run":"mini-SWE-agent","source":"SWE-bench team","sourceUrl":"https://www.swebench.com/","measuredAt":"2026-02-26","stderr":null,"costPerTest":0.5583347409,"latency":null},{"benchmarkId":"swe-bench:swe-bench-multilingual:overall","benchmarkName":"SWE-bench Multilingual","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":67,"unit":"percent","family":"swe-bench-multilingual","index":null,"aboveFit":false,"run":"mini-SWE-agent","source":"SWE-bench team","sourceUrl":"https://www.swebench.com/","measuredAt":"2026-02-20","stderr":null,"costPerTest":0.669532236,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":20,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:swe-bench-team","name":"SWE-bench team","url":"https://www.swebench.com","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"claude46opus","providerId":"anthropic","slug":"anthropic-claude-4-6-opus","name":"Claude 4.6 Opus","provider":"Anthropic","description":"Claude 4.6 Opus is a model from Anthropic. It has no scored result.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[{"benchmarkId":"swe-bench:swe-bench-multilingual:overall","benchmarkName":"SWE-bench Multilingual","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":72,"unit":"percent","family":"swe-bench-multilingual","index":null,"aboveFit":false,"run":"mini-SWE-agent","source":"SWE-bench team","sourceUrl":"https://www.swebench.com/","measuredAt":"2026-02-20","stderr":null,"costPerTest":0.6629735766666667,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:swe-bench-team","name":"SWE-bench team","url":"https://www.swebench.com","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"claude47","providerId":"anthropic","slug":"anthropic-claude-4-7","name":"Claude 4.7","provider":"Anthropic","description":"Claude 4.7 is a model from Anthropic. It has 2 scored results in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":71.64389005554986,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"reasoning":2},"benchmarks":[{"benchmarkId":"arc-prize:v1-public-eval:overall","benchmarkName":"ARC-AGI-1 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":97,"unit":"percent","family":"arc-prize:v1-public-eval:overall","index":null,"aboveFit":false,"run":"max effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":1.77,"latency":null},{"benchmarkId":"arc-prize:v1-semi-private:overall","benchmarkName":"ARC-AGI-1 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":92,"unit":"percent","family":"arc-prize:v1-semi-private:overall","index":69.28308033383526,"aboveFit":false,"run":"max effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":2.58,"latency":null},{"benchmarkId":"arc-prize:v2-public-eval:overall","benchmarkName":"ARC-AGI-2 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":82.23,"unit":"percent","family":"arc-prize:v2-public-eval:overall","index":null,"aboveFit":false,"run":"max effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":8.21,"latency":null},{"benchmarkId":"arc-prize:v2-semi-private:overall","benchmarkName":"ARC-AGI-2 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":75.83,"unit":"percent","family":"arc-prize:v2-semi-private:overall","index":74.00469977726446,"aboveFit":false,"run":"max effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":7.43,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":15,"benchmarkCount":2,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:arc-prize-foundation","name":"ARC Prize Foundation","url":"https://arcprize.org","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"claudefable5exa","providerId":"anthropic","slug":"anthropic-claude-fable-5-exa","name":"Claude Fable 5 Exa","provider":"Anthropic","description":"Claude Fable 5 Exa is a model from Anthropic. It has no scored result.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[{"benchmarkId":"vals-ai:web_search:overall","benchmarkName":"Vals Web Search Index","category":null,"domain":"Other","eligible":false,"ranked":true,"score":48.454,"unit":"percent","family":"vals-ai:web_search:overall","index":null,"aboveFit":false,"run":"Exa","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/web_search","measuredAt":"2026-11-04","stderr":2.031857832132947,"costPerTest":3.481848,"latency":351.537}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"claudefablelatest","providerId":"anthropic","slug":"anthropic-claude-fable-latest","name":"Claude Fable Latest","provider":"Anthropic","description":"Claude Fable Latest is a model from Anthropic. It has no scored result.","logoUrl":null,"releaseDate":"2026-06-09T18:32:24.000Z","contextWindowTokens":1000000,"modalities":["text","image","file"],"openWeights":null,"details":{"apiId":"~anthropic/claude-fable-latest","maxOutputTokens":128000,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.19999999999999998,"weightsUrl":null},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":8,"outputPerMillion":40,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":65,"timeToFirstToken":4.9395,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"claudehaiku45thinking16k","providerId":"anthropic","slug":"anthropic-claude-haiku-4-5-thinking-16k","name":"Claude Haiku 4.5 (Thinking 16K)","provider":"Anthropic","description":"Claude Haiku 4.5 (Thinking 16K) is a model from Anthropic. It has 2 scored results in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":42.307108727590645,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"reasoning":2},"benchmarks":[{"benchmarkId":"arc-prize:v1-public-eval:overall","benchmarkName":"ARC-AGI-1 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":51.38,"unit":"percent","family":"arc-prize:v1-public-eval:overall","index":null,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.0979,"latency":null},{"benchmarkId":"arc-prize:v1-semi-private:overall","benchmarkName":"ARC-AGI-1 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":37.330000000000005,"unit":"percent","family":"arc-prize:v1-semi-private:overall","index":44.14930758422503,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.1042,"latency":null},{"benchmarkId":"arc-prize:v2-public-eval:overall","benchmarkName":"ARC-AGI-2 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":4.17,"unit":"percent","family":"arc-prize:v2-public-eval:overall","index":null,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.1432,"latency":null},{"benchmarkId":"arc-prize:v2-semi-private:overall","benchmarkName":"ARC-AGI-2 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":2.78,"unit":"percent","family":"arc-prize:v2-semi-private:overall","index":40.46490987095626,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.1392,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":15,"benchmarkCount":2,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:arc-prize-foundation","name":"ARC Prize Foundation","url":"https://arcprize.org","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"claudehaiku45thinking1k","providerId":"anthropic","slug":"anthropic-claude-haiku-4-5-thinking-1k","name":"Claude Haiku 4.5 (Thinking 1K)","provider":"Anthropic","description":"Claude Haiku 4.5 (Thinking 1K) is a model from Anthropic. It has 2 scored results in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":37.24357562585632,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"reasoning":2},"benchmarks":[{"benchmarkId":"arc-prize:v1-public-eval:overall","benchmarkName":"ARC-AGI-1 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":27.12,"unit":"percent","family":"arc-prize:v1-public-eval:overall","index":null,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.0268,"latency":null},{"benchmarkId":"arc-prize:v1-semi-private:overall","benchmarkName":"ARC-AGI-1 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":16.830000000000002,"unit":"percent","family":"arc-prize:v1-semi-private:overall","index":34.7247174732499,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.0306,"latency":null},{"benchmarkId":"arc-prize:v2-semi-private:overall","benchmarkName":"ARC-AGI-2 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":1.25,"unit":"percent","family":"arc-prize:v2-semi-private:overall","index":39.76243377846274,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.0471,"latency":null},{"benchmarkId":"arc-prize:v2-public-eval:overall","benchmarkName":"ARC-AGI-2 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":0,"unit":"percent","family":"arc-prize:v2-public-eval:overall","index":null,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.0478,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":15,"benchmarkCount":2,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:arc-prize-foundation","name":"ARC Prize Foundation","url":"https://arcprize.org","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"claudehaiku45thinking32k","providerId":"anthropic","slug":"anthropic-claude-haiku-4-5-thinking-32k","name":"Claude Haiku 4.5 (Thinking 32K)","provider":"Anthropic","description":"Claude Haiku 4.5 (Thinking 32K) is a model from Anthropic. It has 2 scored results in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":44.97090356807837,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"reasoning":2},"benchmarks":[{"benchmarkId":"arc-prize:v1-public-eval:overall","benchmarkName":"ARC-AGI-1 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":62.88,"unit":"percent","family":"arc-prize:v1-public-eval:overall","index":null,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.2139,"latency":null},{"benchmarkId":"arc-prize:v1-semi-private:overall","benchmarkName":"ARC-AGI-1 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":47.67,"unit":"percent","family":"arc-prize:v1-semi-private:overall","index":48.90297888898028,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.2558,"latency":null},{"benchmarkId":"arc-prize:v2-public-eval:overall","benchmarkName":"ARC-AGI-2 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":5.140000000000001,"unit":"percent","family":"arc-prize:v2-public-eval:overall","index":null,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.3927,"latency":null},{"benchmarkId":"arc-prize:v2-semi-private:overall","benchmarkName":"ARC-AGI-2 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":4.03,"unit":"percent","family":"arc-prize:v2-semi-private:overall","index":41.038828247176454,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.3766,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":15,"benchmarkCount":2,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:arc-prize-foundation","name":"ARC Prize Foundation","url":"https://arcprize.org","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"claudehaiku45thinking8k","providerId":"anthropic","slug":"anthropic-claude-haiku-4-5-thinking-8k","name":"Claude Haiku 4.5 (Thinking 8K)","provider":"Anthropic","description":"Claude Haiku 4.5 (Thinking 8K) is a model from Anthropic. It has 2 scored results in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":39.332949919455324,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"reasoning":2},"benchmarks":[{"benchmarkId":"arc-prize:v1-public-eval:overall","benchmarkName":"ARC-AGI-1 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":45,"unit":"percent","family":"arc-prize:v1-public-eval:overall","index":null,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.0644,"latency":null},{"benchmarkId":"arc-prize:v1-semi-private:overall","benchmarkName":"ARC-AGI-1 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":25.5,"unit":"percent","family":"arc-prize:v1-semi-private:overall","index":38.71062948603792,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.0687,"latency":null},{"benchmarkId":"arc-prize:v2-public-eval:overall","benchmarkName":"ARC-AGI-2 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":2.08,"unit":"percent","family":"arc-prize:v2-public-eval:overall","index":null,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.0952,"latency":null},{"benchmarkId":"arc-prize:v2-semi-private:overall","benchmarkName":"ARC-AGI-2 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":1.67,"unit":"percent","family":"arc-prize:v2-semi-private:overall","index":39.95527035287273,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.0909,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":15,"benchmarkCount":2,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:arc-prize-foundation","name":"ARC Prize Foundation","url":"https://arcprize.org","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"claudehaikulatest","providerId":"anthropic","slug":"anthropic-claude-haiku-latest","name":"Claude Haiku Latest","provider":"Anthropic","description":"Claude Haiku Latest is a model from Anthropic. It has no scored result.","logoUrl":null,"releaseDate":"2026-04-27T19:34:52.000Z","contextWindowTokens":1000000,"modalities":["text","image","file"],"openWeights":null,"details":{"apiId":"~anthropic/claude-haiku-latest","maxOutputTokens":128000,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.01,"weightsUrl":null},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":0.09999999999999999,"outputPerMillion":0.5,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":159,"timeToFirstToken":1.5865,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"claude-mythos-5-1","providerId":"anthropic","slug":"claude-mythos-5-1","name":"Claude Mythos 5.1","provider":"Anthropic","description":"Claude Mythos 5.1 is a reasoning model from Anthropic in the Claude Mythos family. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":"2026-09-01","contextWindowTokens":1000000,"modalities":["text"],"openWeights":false,"details":null,"reasoning":true,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":73.18527641027936,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"agentic":1},"benchmarks":[{"benchmarkId":"proteinBinderDesign","benchmarkName":"Anthropic de novo protein-binder design evaluation","category":"knowledge","score":79.1,"unit":"percent","family":"proteinBinderDesign","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"cveRediscovery","benchmarkName":"Anthropic CVE Rediscovery","category":"agentic","score":78.3,"unit":"percent","family":"cveRediscovery","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"protocolsTroubleshooting","benchmarkName":"Molecular Biology Protocols Troubleshooting","category":"knowledge","score":70.2,"unit":"percent","family":"protocolsTroubleshooting","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"proteinDomainIdentification","benchmarkName":"Protein domain identification","category":"knowledge","score":63.5,"unit":"percent","family":"proteinDomainIdentification","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"singleCellBench","benchmarkName":"LatchBio SingleCellBench","category":"knowledge","score":61.8,"unit":"percent","family":"singleCellBench","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"medicinalChemistry","benchmarkName":"Anthropic medicinal-chemistry evaluation","category":"knowledge","score":57.7,"unit":"percent","family":"medicinalChemistry","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"proteinMutationAttribution","benchmarkName":"Protein mutation attribution","category":"knowledge","score":48,"unit":"percent","family":"proteinMutationAttribution","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"exploitGym","benchmarkName":"ExploitGym","category":"agentic","score":23.4,"unit":"percent","family":"exploitGym","index":73.18527641027936,"aboveFit":false,"run":null,"source":"Zhun Wang, Nico Schiller, Hongwei Li, Srijiith Sesha Narayana, Milad Nasr, Nicholas Carlini, Xiangyu Qi, Eric Wallace, Elie Bursztein, Luca Invernizzi, Kurt Thomas, Yan Shoshitaishvili, Wenbo Guo, Jingxuan He, Thorsten Holz, Dawn Song","sourceUrl":"https://arxiv.org/abs/2605.11086","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":20,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"claude-mythos-5-2","providerId":"anthropic","slug":"claude-mythos-5-2","name":"Claude Mythos 5.2","provider":"Anthropic","description":"Claude Mythos 5.2 is a reasoning model from Anthropic in the Claude Mythos family. It has 2 scored results in 1 category. It has no index.","logoUrl":null,"releaseDate":"2026-11-09","contextWindowTokens":1000000,"modalities":["text"],"openWeights":false,"details":null,"reasoning":true,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":84.75714176800929,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"agentic":2},"benchmarks":[{"benchmarkId":"proteinDomainIdentification","benchmarkName":"Protein domain identification","category":"knowledge","score":91.8,"unit":"percent","family":"proteinDomainIdentification","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"cveRediscovery","benchmarkName":"Anthropic CVE Rediscovery","category":"agentic","score":89.6,"unit":"percent","family":"cveRediscovery","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"proteinBinderDesign","benchmarkName":"Anthropic de novo protein-binder design evaluation","category":"knowledge","score":85.1,"unit":"percent","family":"proteinBinderDesign","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"biomedicalImageAnalysis","benchmarkName":"Anthropic biomedical-image-analysis evaluation","category":"multimodal","score":80.7,"unit":"percent","family":"biomedicalImageAnalysis","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"protocolsTroubleshooting","benchmarkName":"Molecular Biology Protocols Troubleshooting","category":"knowledge","score":77.1,"unit":"percent","family":"protocolsTroubleshooting","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"terminalBenchScience","benchmarkName":"Terminal-Bench-Science 0.1","category":"agentic","score":75.3,"unit":"percent","family":"terminalBenchScience","index":87.40879594748614,"aboveFit":false,"run":null,"source":"Terminal-Bench-Science Team","sourceUrl":"https://www.terminal-bench-science.ai/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"medicinalChemistry","benchmarkName":"Anthropic medicinal-chemistry evaluation","category":"knowledge","score":69,"unit":"percent","family":"medicinalChemistry","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"singleCellBench","benchmarkName":"LatchBio SingleCellBench","category":"knowledge","score":65.6,"unit":"percent","family":"singleCellBench","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"proteinDesign","benchmarkName":"Anthropic Protein Design evaluation","category":"knowledge","score":64.6,"unit":"percent","family":"proteinDesign","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"proteinMutationAttribution","benchmarkName":"Protein mutation attribution","category":"knowledge","score":51.5,"unit":"percent","family":"proteinMutationAttribution","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"exploitGym","benchmarkName":"ExploitGym","category":"agentic","score":38.1,"unit":"percent","family":"exploitGym","index":82.10548758853241,"aboveFit":false,"run":null,"source":"Zhun Wang, Nico Schiller, Hongwei Li, Srijiith Sesha Narayana, Milad Nasr, Nicholas Carlini, Xiangyu Qi, Eric Wallace, Elie Bursztein, Luca Invernizzi, Kurt Thomas, Yan Shoshitaishvili, Wenbo Guo, Jingxuan He, Thorsten Holz, Dawn Song","sourceUrl":"https://arxiv.org/abs/2605.11086","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"morphologyToMolecule","benchmarkName":"Axiom Bio morphology-to-molecule matching","category":"knowledge","score":35.2,"unit":"percent","family":"morphologyToMolecule","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":8,"outputPerMillion":40,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":20,"benchmarkCount":2,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"claude-mythos-preview","providerId":"anthropic","slug":"claude-mythos-preview","name":"Claude Mythos Preview","provider":"Anthropic","description":"Claude Mythos Preview is a reasoning model from Anthropic in the Claude Mythos family. It has 2 scored results in 1 category. It has no index.","logoUrl":null,"releaseDate":"2026-04-07","contextWindowTokens":null,"modalities":["text"],"openWeights":false,"details":null,"reasoning":true,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":69.11392341533944,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"agentic":2},"benchmarks":[{"benchmarkId":"cyberGym","benchmarkName":"CyberGym","category":"agentic","score":83.1,"unit":"percent","family":"cyberGym","index":68.62279123343987,"aboveFit":false,"run":null,"source":"Zhun Wang, Tianneng Shi, Jingxuan He, Matthew Cai, Jialin Zhang, Dawn Song","sourceUrl":"https://www.cybergym.io/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"exploitGym","benchmarkName":"ExploitGym","category":"agentic","score":17.5,"unit":"percent","family":"exploitGym","index":69.60505559723902,"aboveFit":false,"run":null,"source":"Zhun Wang, Nico Schiller, Hongwei Li, Srijiith Sesha Narayana, Milad Nasr, Nicholas Carlini, Xiangyu Qi, Eric Wallace, Elie Bursztein, Luca Invernizzi, Kurt Thomas, Yan Shoshitaishvili, Wenbo Guo, Jingxuan He, Thorsten Holz, Dawn Song","sourceUrl":"https://arxiv.org/abs/2605.11086","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":25,"outputPerMillion":125,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":20,"benchmarkCount":2,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"claudeopus4thinking16k","providerId":"anthropic","slug":"anthropic-claude-opus-4-thinking-16k","name":"Claude Opus 4 (Thinking 16K)","provider":"Anthropic","description":"Claude Opus 4 (Thinking 16K) is a model from Anthropic. It has 2 scored results in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":43.26390541546739,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"reasoning":2},"benchmarks":[{"benchmarkId":"arc-prize:v1-public-eval:overall","benchmarkName":"ARC-AGI-1 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":54.25,"unit":"percent","family":"arc-prize:v1-public-eval:overall","index":null,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":1.1405,"latency":null},{"benchmarkId":"arc-prize:v1-semi-private:overall","benchmarkName":"ARC-AGI-1 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":35.67,"unit":"percent","family":"arc-prize:v1-semi-private:overall","index":43.38614565328753,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":1.2496,"latency":null},{"benchmarkId":"arc-prize:v2-semi-private:overall","benchmarkName":"ARC-AGI-2 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":8.61,"unit":"percent","family":"arc-prize:v2-semi-private:overall","index":43.14166517764725,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":1.9284,"latency":null},{"benchmarkId":"arc-prize:v2-public-eval:overall","benchmarkName":"ARC-AGI-2 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":3.3300000000000005,"unit":"percent","family":"arc-prize:v2-public-eval:overall","index":null,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":2.0182,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":{"elo":1377.148823998799,"eloUpper":1381.5883957885876,"eloLower":1372.70925220901,"votes":35852,"rank":182,"rankUpper":null,"rankLower":null,"arenaName":"Claude Opus 4-thinking-16k-20250514","organization":"anthropic","updatedAt":"2026-11-09"},"quality":{"coveragePercent":15,"benchmarkCount":2,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:arc-prize-foundation","name":"ARC Prize Foundation","url":"https://arcprize.org","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"claudeopus4thinking1k","providerId":"anthropic","slug":"anthropic-claude-opus-4-thinking-1k","name":"Claude Opus 4 (Thinking 1K)","provider":"Anthropic","description":"Claude Opus 4 (Thinking 1K) is a model from Anthropic. It has 2 scored results in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":39.294374521371026,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"reasoning":2},"benchmarks":[{"benchmarkId":"arc-prize:v1-public-eval:overall","benchmarkName":"ARC-AGI-1 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":43.309999999999995,"unit":"percent","family":"arc-prize:v1-public-eval:overall","index":null,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.4236,"latency":null},{"benchmarkId":"arc-prize:v1-semi-private:overall","benchmarkName":"ARC-AGI-1 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":27,"unit":"percent","family":"arc-prize:v1-semi-private:overall","index":39.40023364049951,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.5021,"latency":null},{"benchmarkId":"arc-prize:v2-public-eval:overall","benchmarkName":"ARC-AGI-2 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":0.43,"unit":"percent","family":"arc-prize:v2-public-eval:overall","index":null,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.7578,"latency":null},{"benchmarkId":"arc-prize:v2-semi-private:overall","benchmarkName":"ARC-AGI-2 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":0,"unit":"percent","family":"arc-prize:v2-semi-private:overall","index":39.188515402242544,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.7503,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":15,"benchmarkCount":2,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:arc-prize-foundation","name":"ARC Prize Foundation","url":"https://arcprize.org","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"claudeopus4thinking8k","providerId":"anthropic","slug":"anthropic-claude-opus-4-thinking-8k","name":"Claude Opus 4 (Thinking 8K)","provider":"Anthropic","description":"Claude Opus 4 (Thinking 8K) is a model from Anthropic. It has 2 scored results in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":41.17563469453516,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"reasoning":2},"benchmarks":[{"benchmarkId":"arc-prize:v1-public-eval:overall","benchmarkName":"ARC-AGI-1 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":45.62,"unit":"percent","family":"arc-prize:v1-public-eval:overall","index":null,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.7817,"latency":null},{"benchmarkId":"arc-prize:v1-semi-private:overall","benchmarkName":"ARC-AGI-1 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":30.669999999999998,"unit":"percent","family":"arc-prize:v1-semi-private:overall","index":41.087465138415546,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.7408,"latency":null},{"benchmarkId":"arc-prize:v2-semi-private:overall","benchmarkName":"ARC-AGI-2 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":4.52,"unit":"percent","family":"arc-prize:v2-semi-private:overall","index":41.26380425065477,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":1.1569,"latency":null},{"benchmarkId":"arc-prize:v2-public-eval:overall","benchmarkName":"ARC-AGI-2 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":2.5,"unit":"percent","family":"arc-prize:v2-public-eval:overall","index":null,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":1.2021,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":15,"benchmarkCount":2,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:arc-prize-foundation","name":"ARC Prize Foundation","url":"https://arcprize.org","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"claudeopus420250514thinking","providerId":"anthropic","slug":"anthropic-claude-opus-4-thinking","name":"Claude Opus 4 Thinking","provider":"Anthropic","description":"Claude Opus 4 Thinking is a model from Anthropic. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":47.58164388474303,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"coding":1},"benchmarks":[{"benchmarkId":"vals-ai:lcb:overall","benchmarkName":"LiveCodeBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":70.188,"unit":"percent","family":"livecodebench","index":47.58164388474303,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","measuredAt":"2026-09-01","stderr":1.111,"costPerTest":0.397382,"latency":93.538}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":20,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"claudeopus4120250805thinking","providerId":"anthropic","slug":"anthropic-claude-opus-4-1-thinking","name":"Claude Opus 4.1 Thinking","provider":"Anthropic","description":"Claude Opus 4.1 Thinking is a model from Anthropic. It has 6 scored results in 4 categories. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":44.48504961810481,"reasoning":null,"multimodal":56.03048534628264,"knowledge":51.20494231996795,"multilingual":null,"instruction":null,"math":47.305941608705616},"capabilityCounts":{"math":2,"knowledge":2,"multimodal":1,"coding":1},"benchmarks":[{"benchmarkId":"vals-ai:math500:overall","benchmarkName":"MATH 500","category":"math","domain":null,"eligible":true,"ranked":true,"score":95.4,"unit":"percent","family":"math-500","index":44.94151672218784,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/math500","measuredAt":"2026-01-09","stderr":0.937,"costPerTest":0.202262,"latency":45.979},{"benchmarkId":"vals-ai:mgsm:overall","benchmarkName":"MGSM","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":94.436,"unit":"percent","family":"vals-ai:mgsm:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mgsm","measuredAt":"2026-01-09","stderr":1.429,"costPerTest":0.054705,"latency":14.281},{"benchmarkId":"vals-ai:medqa:overall","benchmarkName":"MedQA","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":93.592,"unit":"percent","family":"vals-ai:medqa:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medqa","measuredAt":"2026-04-16","stderr":0.224,"costPerTest":0.088223,"latency":23.61},{"benchmarkId":"vals-ai:mmlu_pro:overall","benchmarkName":"MMLU Pro","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":87.924,"unit":"percent","family":"mmlu-pro","index":55.25444419467879,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","measuredAt":"2026-09-01","stderr":0.325,"costPerTest":0.124083,"latency":27.668},{"benchmarkId":"vals-ai:aime:overall","benchmarkName":"AIME","category":"math","domain":null,"eligible":true,"ranked":true,"score":78.179,"unit":"percent","family":"vals-ai:aime:overall","index":49.67036649522339,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/aime","measuredAt":"2026-04-16","stderr":1.629,"costPerTest":0.85994,"latency":214.504},{"benchmarkId":"vals-ai:mmmu:overall","benchmarkName":"MMMU Pro","category":"multimodal","domain":null,"eligible":true,"ranked":true,"score":77.514,"unit":"percent","family":"mmmu-pro","index":56.03048534628264,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmmu","measuredAt":"2026-09-01","stderr":1.004,"costPerTest":0.168667,"latency":2.446},{"benchmarkId":"vals-ai:gpqa:overall","benchmarkName":"GPQA Diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":76.263,"unit":"percent","family":"gpqa-diamond","index":47.1554404452571,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","measuredAt":"2026-09-01","stderr":2.16,"costPerTest":0.144678,"latency":35.772},{"benchmarkId":"vals-ai:medscribe:overall","benchmarkName":"MedScribe","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":73.901,"unit":"percent","family":"vals-ai:medscribe:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medscribe","measuredAt":"2026-10-29","stderr":1.965,"costPerTest":0.263427,"latency":57.404},{"benchmarkId":"vals-ai:tax_eval_v2:overall","benchmarkName":"TaxEval v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":73.672,"unit":"percent","family":"vals-ai:tax_eval_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","measuredAt":"2026-09-01","stderr":0.864,"costPerTest":0.093482,"latency":26.637},{"benchmarkId":"vals-ai:lcb:overall","benchmarkName":"LiveCodeBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":66.456,"unit":"percent","family":"livecodebench","index":44.48504961810481,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","measuredAt":"2026-09-01","stderr":1.125,"costPerTest":0.360655,"latency":92.167},{"benchmarkId":"vals-ai:mortgage_tax:overall","benchmarkName":"MortgageTax","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":56.121,"unit":"percent","family":"vals-ai:mortgage_tax:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mortgage_tax","measuredAt":"2026-09-01","stderr":0.988,"costPerTest":0.135806,"latency":23.52},{"benchmarkId":"vals-ai:medcode:overall","benchmarkName":"MedCode","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":47.235,"unit":"percent","family":"vals-ai:medcode:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medcode","measuredAt":"2026-10-29","stderr":2.067,"costPerTest":0.269254,"latency":33.263}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":55.00000000000001,"benchmarkCount":6,"categoryCount":4,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"claudeopus4520251101thinking64k","providerId":"anthropic","slug":"anthropic-claude-opus-4-5-thinking-64k","name":"Claude Opus 4.5 Thinking 64K","provider":"Anthropic","description":"Claude Opus 4.5 Thinking 64K is a model from Anthropic. It has 6 scored results in 6 categories. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":50.10153035600389,"coding":66.85027508519791,"reasoning":61.710241566519954,"multimodal":null,"knowledge":66.44946207146916,"multilingual":null,"instruction":53.14450075339277,"math":65.75171747230172},"capabilityCounts":{"math":1,"knowledge":1,"reasoning":1,"coding":1,"instruction":1,"agentic":1},"benchmarks":[{"benchmarkId":"livebench:livebench-math:overall","benchmarkName":"LiveBench Mathematics","category":"math","domain":null,"eligible":true,"ranked":true,"score":90.38900000000001,"unit":"percent","family":"livebench:livebench-math:overall","index":65.75171747230172,"aboveFit":false,"run":"high effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Mathematics","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.4044007915567282,"latency":null},{"benchmarkId":"livebench:livebench-language:overall","benchmarkName":"LiveBench Language","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":81.26100000000001,"unit":"percent","family":"livebench:livebench-language:overall","index":66.44946207146916,"aboveFit":false,"run":"high effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Language","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.18626933333333334,"latency":null},{"benchmarkId":"livebench:livebench-reasoning:overall","benchmarkName":"LiveBench Reasoning","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":80.0865,"unit":"percent","family":"livebench","index":65.3664960678106,"aboveFit":false,"run":"high effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Reasoning","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.49901980198019796,"latency":null},{"benchmarkId":"livebench:livebench-coding:overall","benchmarkName":"LiveBench Coding","category":"coding","domain":null,"eligible":true,"ranked":true,"score":79.654,"unit":"percent","family":"livebench:livebench-coding:overall","index":66.85027508519791,"aboveFit":false,"run":"high effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Coding","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.1743222222222222,"latency":null},{"benchmarkId":"livebench:livebench-data-analysis:overall","benchmarkName":"LiveBench Data Analysis","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":74.44166666666666,"unit":"percent","family":"livebench","index":58.05398706522931,"aboveFit":false,"run":"high effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Data%20Analysis","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.518286,"latency":null},{"benchmarkId":"livebench:livebench-instruction-following:overall","benchmarkName":"LiveBench Instruction Following","category":"instruction","domain":null,"eligible":true,"ranked":true,"score":62.54599999999999,"unit":"percent","family":"livebench:livebench-instruction-following:overall","index":53.14450075339277,"aboveFit":false,"run":"high effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=IF","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.40027050000000003,"latency":null},{"benchmarkId":"livebench:livebench-agentic-coding:overall","benchmarkName":"LiveBench Agentic Coding","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":39.697,"unit":"percent","family":"livebench:livebench-agentic-coding:overall","index":50.10153035600389,"aboveFit":false,"run":"high effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Agentic%20Coding","measuredAt":"2026-06-25","stderr":null,"costPerTest":1.4234055555555556,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":85,"benchmarkCount":6,"categoryCount":6,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:livebench","name":"LiveBench","url":"https://livebench.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"claudeopus46120khigh","providerId":"anthropic","slug":"anthropic-claude-opus-4-6-120k-high","name":"Claude Opus 4.6 (120K, High)","provider":"Anthropic","description":"Claude Opus 4.6 (120K, High) is a model from Anthropic. It has 2 scored results in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":70.57470760427366,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"reasoning":2},"benchmarks":[{"benchmarkId":"arc-prize:v1-public-eval:overall","benchmarkName":"ARC-AGI-1 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":96.25,"unit":"percent","family":"arc-prize:v1-public-eval:overall","index":null,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.8727,"latency":null},{"benchmarkId":"arc-prize:v1-semi-private:overall","benchmarkName":"ARC-AGI-1 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":94,"unit":"percent","family":"arc-prize:v1-semi-private:overall","index":70.20255253978405,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":1.3948,"latency":null},{"benchmarkId":"arc-prize:v2-public-eval:overall","benchmarkName":"ARC-AGI-2 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":79.03,"unit":"percent","family":"arc-prize:v2-public-eval:overall","index":null,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":3.8092,"latency":null},{"benchmarkId":"arc-prize:v2-semi-private:overall","benchmarkName":"ARC-AGI-2 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":69.17,"unit":"percent","family":"arc-prize:v2-semi-private:overall","index":70.94686266876326,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":3.4719,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":15,"benchmarkCount":2,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:arc-prize-foundation","name":"ARC Prize Foundation","url":"https://arcprize.org","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"claudeopus46120klow","providerId":"anthropic","slug":"anthropic-claude-opus-4-6-120k-low","name":"Claude Opus 4.6 (120K, Low)","provider":"Anthropic","description":"Claude Opus 4.6 (120K, Low) is a model from Anthropic. It has 2 scored results in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":67.6820490536358,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"reasoning":2},"benchmarks":[{"benchmarkId":"arc-prize:v1-public-eval:overall","benchmarkName":"ARC-AGI-1 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":89.60000000000001,"unit":"percent","family":"arc-prize:v1-public-eval:overall","index":null,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.4105,"latency":null},{"benchmarkId":"arc-prize:v1-semi-private:overall","benchmarkName":"ARC-AGI-1 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":86,"unit":"percent","family":"arc-prize:v1-semi-private:overall","index":66.52466371598888,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.648,"latency":null},{"benchmarkId":"arc-prize:v2-semi-private:overall","benchmarkName":"ARC-AGI-2 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":64.58,"unit":"percent","family":"arc-prize:v2-semi-private:overall","index":68.83943439128271,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":2.2507,"latency":null},{"benchmarkId":"arc-prize:v2-public-eval:overall","benchmarkName":"ARC-AGI-2 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":59.86,"unit":"percent","family":"arc-prize:v2-public-eval:overall","index":null,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":2.3966,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":15,"benchmarkCount":2,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:arc-prize-foundation","name":"ARC Prize Foundation","url":"https://arcprize.org","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"claudeopus46120kmax","providerId":"anthropic","slug":"anthropic-claude-opus-4-6-120k-max","name":"Claude Opus 4.6 (120K, Max)","provider":"Anthropic","description":"Claude Opus 4.6 (120K, Max) is a model from Anthropic. It has 2 scored results in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":70.24842126558147,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"reasoning":2},"benchmarks":[{"benchmarkId":"arc-prize:v1-public-eval:overall","benchmarkName":"ARC-AGI-1 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":96.75,"unit":"percent","family":"arc-prize:v1-public-eval:overall","index":null,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":1.2958,"latency":null},{"benchmarkId":"arc-prize:v1-semi-private:overall","benchmarkName":"ARC-AGI-1 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":93,"unit":"percent","family":"arc-prize:v1-semi-private:overall","index":69.74281643680965,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":1.8791,"latency":null},{"benchmarkId":"arc-prize:v2-public-eval:overall","benchmarkName":"ARC-AGI-2 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":74.85000000000001,"unit":"percent","family":"arc-prize:v2-public-eval:overall","index":null,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":3.7707,"latency":null},{"benchmarkId":"arc-prize:v2-semi-private:overall","benchmarkName":"ARC-AGI-2 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":68.75,"unit":"percent","family":"arc-prize:v2-semi-private:overall","index":70.75402609435328,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":3.641,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":15,"benchmarkCount":2,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:arc-prize-foundation","name":"ARC Prize Foundation","url":"https://arcprize.org","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"claudeopus46120kmedium","providerId":"anthropic","slug":"anthropic-claude-opus-4-6-120k-medium","name":"Claude Opus 4.6 (120K, Medium)","provider":"Anthropic","description":"Claude Opus 4.6 (120K, Medium) is a model from Anthropic. It has 2 scored results in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":69.44463483787408,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"reasoning":2},"benchmarks":[{"benchmarkId":"arc-prize:v1-public-eval:overall","benchmarkName":"ARC-AGI-1 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":94.75,"unit":"percent","family":"arc-prize:v1-public-eval:overall","index":null,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.6183,"latency":null},{"benchmarkId":"arc-prize:v1-semi-private:overall","benchmarkName":"ARC-AGI-1 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":92,"unit":"percent","family":"arc-prize:v1-semi-private:overall","index":69.28308033383526,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.9647,"latency":null},{"benchmarkId":"arc-prize:v2-public-eval:overall","benchmarkName":"ARC-AGI-2 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":73.61,"unit":"percent","family":"arc-prize:v2-public-eval:overall","index":null,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":3.0162,"latency":null},{"benchmarkId":"arc-prize:v2-semi-private:overall","benchmarkName":"ARC-AGI-2 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":66.25,"unit":"percent","family":"arc-prize:v2-semi-private:overall","index":69.60618934191288,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":2.7209,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":15,"benchmarkCount":2,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:arc-prize-foundation","name":"ARC Prize Foundation","url":"https://arcprize.org","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"claudeopus46thinkingauto","providerId":"anthropic","slug":"anthropic-claude-opus-4-6-thinking-auto","name":"Claude Opus 4.6 Thinking Auto","provider":"Anthropic","description":"Claude Opus 4.6 Thinking Auto is a model from Anthropic. It has 6 scored results in 6 categories. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":59.684899219606955,"coding":64.78358851077434,"reasoning":64.32561485190685,"multimodal":null,"knowledge":68.72088169675573,"multilingual":null,"instruction":54.69548274446828,"math":64.25765038967658},"capabilityCounts":{"math":1,"reasoning":1,"knowledge":1,"coding":1,"instruction":1,"agentic":1},"benchmarks":[{"benchmarkId":"livebench:livebench-math:overall","benchmarkName":"LiveBench Mathematics","category":"math","domain":null,"eligible":true,"ranked":true,"score":89.31700000000001,"unit":"percent","family":"livebench:livebench-math:overall","index":64.25765038967658,"aboveFit":false,"run":"high effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Mathematics","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.2543335092348285,"latency":null},{"benchmarkId":"livebench:livebench-reasoning:overall","benchmarkName":"LiveBench Reasoning","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":88.673,"unit":"percent","family":"livebench","index":76.48974007572302,"aboveFit":false,"run":"high effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Reasoning","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.27745495049504953,"latency":null},{"benchmarkId":"livebench:livebench-language:overall","benchmarkName":"LiveBench Language","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":83.26966666666667,"unit":"percent","family":"livebench:livebench-language:overall","index":68.72088169675573,"aboveFit":false,"run":"high effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Language","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.10086733333333334,"latency":null},{"benchmarkId":"livebench:livebench-coding:overall","benchmarkName":"LiveBench Coding","category":"coding","domain":null,"eligible":true,"ranked":true,"score":78.1845,"unit":"percent","family":"livebench:livebench-coding:overall","index":64.78358851077434,"aboveFit":false,"run":"high effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Coding","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.029984615384615384,"latency":null},{"benchmarkId":"livebench:livebench-data-analysis:overall","benchmarkName":"LiveBench Data Analysis","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":69.893,"unit":"percent","family":"livebench","index":52.16148962809068,"aboveFit":false,"run":"high effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Data%20Analysis","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.36225999999999997,"latency":null},{"benchmarkId":"livebench:livebench-instruction-following:overall","benchmarkName":"LiveBench Instruction Following","category":"instruction","domain":null,"eligible":true,"ranked":true,"score":63.3125,"unit":"percent","family":"livebench:livebench-instruction-following:overall","index":54.69548274446828,"aboveFit":false,"run":"high effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=IF","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.3349805,"latency":null},{"benchmarkId":"livebench:livebench-agentic-coding:overall","benchmarkName":"LiveBench Agentic Coding","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":48.989666666666665,"unit":"percent","family":"livebench:livebench-agentic-coding:overall","index":59.684899219606955,"aboveFit":false,"run":"high effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Agentic%20Coding","measuredAt":"2026-06-25","stderr":null,"costPerTest":1.2429708333333334,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":85,"benchmarkCount":6,"categoryCount":6,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:livebench","name":"LiveBench","url":"https://livebench.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"claudeopus48claudecode","providerId":"anthropic","slug":"anthropic-claude-opus-4-8-claude-code","name":"Claude Opus 4.8 Claude Code","provider":"Anthropic","description":"Claude Opus 4.8 Claude Code is a model from Anthropic. It has 3 scored results in 2 categories. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":61.88390046263605,"coding":67.3588478374634,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"coding":2,"agentic":1},"benchmarks":[{"benchmarkId":"vals-ai:swebench:overall","benchmarkName":"SWE-bench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":85.8,"unit":"percent","family":"swe-bench-verified","index":64.51036770331241,"aboveFit":false,"run":"Claude Code","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/swebench","measuredAt":"2026-09-01","stderr":1.563,"costPerTest":0.672451,"latency":135.395},{"benchmarkId":"vals-ai:vibe-code:overall","benchmarkName":"Vibe Code Bench v1.1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":77.485,"unit":"percent","family":"vals-ai:vibe-code:overall","index":70.20732797161438,"aboveFit":false,"run":"Claude Code","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","measuredAt":"2026-11-09","stderr":3.737,"costPerTest":6.8557,"latency":1142.14},{"benchmarkId":"vals-ai:terminal-bench-2-1:overall","benchmarkName":"Terminal-Bench 2.1","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":69.663,"unit":"percent","family":"terminal-bench-2-1","index":61.88390046263605,"aboveFit":false,"run":"Claude Code","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2-1","measuredAt":"2026-10-01","stderr":4.901,"costPerTest":0.968098,"latency":359.967}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":40,"benchmarkCount":3,"categoryCount":2,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"claudeopus55paralleladvanced","providerId":"anthropic","slug":"anthropic-claude-opus-5-5-parallel-advanced","name":"Claude Opus 5.5 Parallel Advanced","provider":"Anthropic","description":"Claude Opus 5.5 Parallel Advanced is a model from Anthropic. It has no scored result.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[{"benchmarkId":"vals-ai:web_search:overall","benchmarkName":"Vals Web Search Index","category":null,"domain":"Other","eligible":false,"ranked":true,"score":54.05218,"unit":"percent","family":"vals-ai:web_search:overall","index":null,"aboveFit":false,"run":"Parallel Advanced","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/web_search","measuredAt":"2026-11-04","stderr":2.021519,"costPerTest":2.212197,"latency":405.006905}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"claudeopus55parallelfast","providerId":"anthropic","slug":"anthropic-claude-opus-5-5-parallel-fast","name":"Claude Opus 5.5 Parallel Fast","provider":"Anthropic","description":"Claude Opus 5.5 Parallel Fast is a model from Anthropic. It has no scored result.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[{"benchmarkId":"vals-ai:web_search:overall","benchmarkName":"Vals Web Search Index","category":null,"domain":"Other","eligible":false,"ranked":true,"score":57.535937,"unit":"percent","family":"vals-ai:web_search:overall","index":null,"aboveFit":false,"run":"Parallel Fast","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/web_search","measuredAt":"2026-11-04","stderr":2.006197,"costPerTest":2.267447,"latency":400.260666}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"claudeopus55range","providerId":"anthropic","slug":"anthropic-claude-opus-5-5-range","name":"Claude Opus 5.5 Range","provider":"Anthropic","description":"Claude Opus 5.5 Range is a model from Anthropic. It has no scored result.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[{"benchmarkId":"vals-ai:cyber:overall","benchmarkName":"CyberBench v1.1","category":null,"domain":"Other","eligible":false,"ranked":true,"score":74.584,"unit":"percent","family":"vals-ai:cyber:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/cyber","measuredAt":"2026-11-05","stderr":5.348,"costPerTest":4.169947,"latency":1381.95},{"benchmarkId":"vals-ai:srebench:overall","benchmarkName":"SRE Bench","category":null,"domain":"Other","eligible":false,"ranked":true,"score":21.756,"unit":"percent","family":"vals-ai:srebench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/srebench","measuredAt":"2026-11-05","stderr":2.554,"costPerTest":33.133504,"latency":12117.679}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"claudeopuslatest","providerId":"anthropic","slug":"anthropic-claude-opus-latest","name":"Claude Opus Latest","provider":"Anthropic","description":"Claude Opus Latest is a model from Anthropic. It has no scored result.","logoUrl":null,"releaseDate":"2026-04-21T18:16:01.000Z","contextWindowTokens":1000000,"modalities":["text","image","file"],"openWeights":null,"details":{"apiId":"~anthropic/claude-opus-latest","maxOutputTokens":128000,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.19999999999999998,"weightsUrl":null},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":4,"outputPerMillion":20,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":75,"timeToFirstToken":3.288,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"claudesonnet35","providerId":"anthropic","slug":"anthropic-claude-sonnet-3-5","name":"Claude Sonnet 3.5","provider":"Anthropic","description":"Claude Sonnet 3.5 is a model from Anthropic. It has no scored result.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[{"benchmarkId":"swe-bench:swe-bench-multimodal:overall","benchmarkName":"SWE-bench Multimodal","category":"multimodal","domain":null,"eligible":true,"ranked":true,"score":12.09,"unit":"percent","family":"swe-bench-multimodal","index":null,"aboveFit":false,"run":"SWE-agent","source":"SWE-bench team","sourceUrl":"https://www.swebench.com/","measuredAt":"2025-11-17","stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:swe-bench-team","name":"SWE-bench team","url":"https://www.swebench.com","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"claudesonnet4thinking16k","providerId":"anthropic","slug":"anthropic-claude-sonnet-4-thinking-16k","name":"Claude Sonnet 4 (Thinking 16K)","provider":"Anthropic","description":"Claude Sonnet 4 (Thinking 16K) is a model from Anthropic. It has 2 scored results in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":43.643993579098904,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"reasoning":2},"benchmarks":[{"benchmarkId":"arc-prize:v1-public-eval:overall","benchmarkName":"ARC-AGI-1 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":56.75,"unit":"percent","family":"arc-prize:v1-public-eval:overall","index":null,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.322,"latency":null},{"benchmarkId":"arc-prize:v1-semi-private:overall","benchmarkName":"ARC-AGI-1 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":40,"unit":"percent","family":"arc-prize:v1-semi-private:overall","index":45.37680297916666,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.3658,"latency":null},{"benchmarkId":"arc-prize:v2-semi-private:overall","benchmarkName":"ARC-AGI-2 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":5.93,"unit":"percent","family":"arc-prize:v2-semi-private:overall","index":41.91118417903115,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.4857,"latency":null},{"benchmarkId":"arc-prize:v2-public-eval:overall","benchmarkName":"ARC-AGI-2 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":4.44,"unit":"percent","family":"arc-prize:v2-public-eval:overall","index":null,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.5102,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":15,"benchmarkCount":2,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:arc-prize-foundation","name":"ARC Prize Foundation","url":"https://arcprize.org","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"claudesonnet4thinking1k","providerId":"anthropic","slug":"anthropic-claude-sonnet-4-thinking-1k","name":"Claude Sonnet 4 (Thinking 1K)","provider":"Anthropic","description":"Claude Sonnet 4 (Thinking 1K) is a model from Anthropic. It has 2 scored results in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":39.71937482077309,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"reasoning":2},"benchmarks":[{"benchmarkId":"arc-prize:v1-public-eval:overall","benchmarkName":"ARC-AGI-1 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":31.3,"unit":"percent","family":"arc-prize:v1-public-eval:overall","index":null,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.0811,"latency":null},{"benchmarkId":"arc-prize:v1-semi-private:overall","benchmarkName":"ARC-AGI-1 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":28.000000000000004,"unit":"percent","family":"arc-prize:v1-semi-private:overall","index":39.85996974347391,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.0937,"latency":null},{"benchmarkId":"arc-prize:v2-public-eval:overall","benchmarkName":"ARC-AGI-2 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":1.25,"unit":"percent","family":"arc-prize:v2-public-eval:overall","index":null,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.1466,"latency":null},{"benchmarkId":"arc-prize:v2-semi-private:overall","benchmarkName":"ARC-AGI-2 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":0.8500000000000001,"unit":"percent","family":"arc-prize:v2-semi-private:overall","index":39.57877989807228,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.1425,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":15,"benchmarkCount":2,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:arc-prize-foundation","name":"ARC Prize Foundation","url":"https://arcprize.org","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"claudesonnet4thinking8k","providerId":"anthropic","slug":"anthropic-claude-sonnet-4-thinking-8k","name":"Claude Sonnet 4 (Thinking 8K)","provider":"Anthropic","description":"Claude Sonnet 4 (Thinking 8K) is a model from Anthropic. It has 2 scored results in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":40.24079340738015,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"reasoning":2},"benchmarks":[{"benchmarkId":"arc-prize:v1-public-eval:overall","benchmarkName":"ARC-AGI-1 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":48.620000000000005,"unit":"percent","family":"arc-prize:v1-public-eval:overall","index":null,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.1936,"latency":null},{"benchmarkId":"arc-prize:v1-semi-private:overall","benchmarkName":"ARC-AGI-1 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":28.999999999999996,"unit":"percent","family":"arc-prize:v1-semi-private:overall","index":40.3197058464483,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.1952,"latency":null},{"benchmarkId":"arc-prize:v2-public-eval:overall","benchmarkName":"ARC-AGI-2 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":2.92,"unit":"percent","family":"arc-prize:v2-public-eval:overall","index":null,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.2724,"latency":null},{"benchmarkId":"arc-prize:v2-semi-private:overall","benchmarkName":"ARC-AGI-2 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":2.12,"unit":"percent","family":"arc-prize:v2-semi-private:overall","index":40.161880968312,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.2654,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":15,"benchmarkCount":2,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:arc-prize-foundation","name":"ARC Prize Foundation","url":"https://arcprize.org","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"claudesonnet45thinking16k","providerId":"anthropic","slug":"anthropic-claude-sonnet-4-5-thinking-16k","name":"Claude Sonnet 4.5 (Thinking 16K)","provider":"Anthropic","description":"Claude Sonnet 4.5 (Thinking 16K) is a model from Anthropic. It has 2 scored results in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":45.790657471980225,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"reasoning":2},"benchmarks":[{"benchmarkId":"arc-prize:v1-public-eval:overall","benchmarkName":"ARC-AGI-1 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":63.62,"unit":"percent","family":"arc-prize:v1-public-eval:overall","index":null,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.2423,"latency":null},{"benchmarkId":"arc-prize:v1-semi-private:overall","benchmarkName":"ARC-AGI-1 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":48.33,"unit":"percent","family":"arc-prize:v1-semi-private:overall","index":49.20640471694338,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.2683,"latency":null},{"benchmarkId":"arc-prize:v2-semi-private:overall","benchmarkName":"ARC-AGI-2 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":6.94,"unit":"percent","family":"arc-prize:v2-semi-private:overall","index":42.37491022701707,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.3499,"latency":null},{"benchmarkId":"arc-prize:v2-public-eval:overall","benchmarkName":"ARC-AGI-2 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":6.529999999999999,"unit":"percent","family":"arc-prize:v2-public-eval:overall","index":null,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.3557,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":15,"benchmarkCount":2,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:arc-prize-foundation","name":"ARC Prize Foundation","url":"https://arcprize.org","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"claudesonnet45thinking1k","providerId":"anthropic","slug":"anthropic-claude-sonnet-4-5-thinking-1k","name":"Claude Sonnet 4.5 (Thinking 1K)","provider":"Anthropic","description":"Claude Sonnet 4.5 (Thinking 1K) is a model from Anthropic. It has 2 scored results in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":41.552224380665315,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"reasoning":2},"benchmarks":[{"benchmarkId":"arc-prize:v1-public-eval:overall","benchmarkName":"ARC-AGI-1 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":36.620000000000005,"unit":"percent","family":"arc-prize:v1-public-eval:overall","index":null,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.0823,"latency":null},{"benchmarkId":"arc-prize:v1-semi-private:overall","benchmarkName":"ARC-AGI-1 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":31,"unit":"percent","family":"arc-prize:v1-semi-private:overall","index":41.239178052397094,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.0928,"latency":null},{"benchmarkId":"arc-prize:v2-semi-private:overall","benchmarkName":"ARC-AGI-2 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":5.83,"unit":"percent","family":"arc-prize:v2-semi-private:overall","index":41.86527070893354,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.1424,"latency":null},{"benchmarkId":"arc-prize:v2-public-eval:overall","benchmarkName":"ARC-AGI-2 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":2.08,"unit":"percent","family":"arc-prize:v2-public-eval:overall","index":null,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.1424,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":15,"benchmarkCount":2,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:arc-prize-foundation","name":"ARC Prize Foundation","url":"https://arcprize.org","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"claudesonnet45thinking32k","providerId":"anthropic","slug":"anthropic-claude-sonnet-4-5-thinking-32k","name":"Claude Sonnet 4.5 (Thinking 32K)","provider":"Anthropic","description":"Claude Sonnet 4.5 (Thinking 32K) is a model from Anthropic. It has 2 scored results in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":50.848047609549326,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"reasoning":2},"benchmarks":[{"benchmarkId":"arc-prize:v1-public-eval:overall","benchmarkName":"ARC-AGI-1 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":73.75,"unit":"percent","family":"arc-prize:v1-public-eval:overall","index":null,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.4419,"latency":null},{"benchmarkId":"arc-prize:v1-semi-private:overall","benchmarkName":"ARC-AGI-1 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":63.67,"unit":"percent","family":"arc-prize:v1-semi-private:overall","index":56.258756536570615,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.5193,"latency":null},{"benchmarkId":"arc-prize:v2-public-eval:overall","benchmarkName":"ARC-AGI-2 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":14.719999999999999,"unit":"percent","family":"arc-prize:v2-public-eval:overall","index":null,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.8403,"latency":null},{"benchmarkId":"arc-prize:v2-semi-private:overall","benchmarkName":"ARC-AGI-2 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":13.61,"unit":"percent","family":"arc-prize:v2-semi-private:overall","index":45.43733868252803,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.7589,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":15,"benchmarkCount":2,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:arc-prize-foundation","name":"ARC Prize Foundation","url":"https://arcprize.org","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"claudesonnet45thinking8k","providerId":"anthropic","slug":"anthropic-claude-sonnet-4-5-thinking-8k","name":"Claude Sonnet 4.5 (Thinking 8K)","provider":"Anthropic","description":"Claude Sonnet 4.5 (Thinking 8K) is a model from Anthropic. It has 2 scored results in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":45.36999893775865,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"reasoning":2},"benchmarks":[{"benchmarkId":"arc-prize:v1-public-eval:overall","benchmarkName":"ARC-AGI-1 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":53.5,"unit":"percent","family":"arc-prize:v1-public-eval:overall","index":null,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.1681,"latency":null},{"benchmarkId":"arc-prize:v1-semi-private:overall","benchmarkName":"ARC-AGI-1 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":46.5,"unit":"percent","family":"arc-prize:v1-semi-private:overall","index":48.365087648500236,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.1776,"latency":null},{"benchmarkId":"arc-prize:v2-semi-private:overall","benchmarkName":"ARC-AGI-2 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":6.94,"unit":"percent","family":"arc-prize:v2-semi-private:overall","index":42.37491022701707,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.2349,"latency":null},{"benchmarkId":"arc-prize:v2-public-eval:overall","benchmarkName":"ARC-AGI-2 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":4.859999999999999,"unit":"percent","family":"arc-prize:v2-public-eval:overall","index":null,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.2409,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":15,"benchmarkCount":2,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:arc-prize-foundation","name":"ARC Prize Foundation","url":"https://arcprize.org","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"claudesonnet4520250929nothinking","providerId":"anthropic","slug":"anthropic-claude-sonnet-4-5-no-thinking","name":"Claude Sonnet 4.5 No Thinking","provider":"Anthropic","description":"Claude Sonnet 4.5 No Thinking is a model from Anthropic. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":53.01600453404664,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-french:overall","benchmarkName":"EuroEval French","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":65.61625000000001,"unit":"percent","family":"euroeval","index":61.24638883951744,"aboveFit":false,"run":"no-thinking effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/french","measuredAt":null,"stderr":0.49476951704808986,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-dutch:overall","benchmarkName":"EuroEval Dutch","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":62.46777777777777,"unit":"percent","family":"euroeval","index":56.97463647678105,"aboveFit":false,"run":"no-thinking effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/dutch","measuredAt":null,"stderr":0.38042374035044424,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-italian:overall","benchmarkName":"EuroEval Italian","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":61.125,"unit":"percent","family":"euroeval","index":55.152795996700995,"aboveFit":false,"run":"no-thinking effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/italian","measuredAt":null,"stderr":0.3465792658252943,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":60.39874999999999,"unit":"percent","family":"euroeval","index":54.16744185206521,"aboveFit":false,"run":"no-thinking effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":0.6486897852594875,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":58.70142857142857,"unit":"percent","family":"euroeval","index":51.86456721602808,"aboveFit":false,"run":"no-thinking effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.4414216365613378,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-portuguese:overall","benchmarkName":"EuroEval Portuguese","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":58.5075,"unit":"percent","family":"euroeval","index":51.60145076629253,"aboveFit":false,"run":"no-thinking effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/portuguese","measuredAt":null,"stderr":0.4201562209464475,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-spanish:overall","benchmarkName":"EuroEval Spanish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":57.14125,"unit":"percent","family":"euroeval","index":49.74776388146137,"aboveFit":false,"run":"no-thinking effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/spanish","measuredAt":null,"stderr":0.34479160372607687,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":56.90374999999999,"unit":"percent","family":"euroeval","index":49.425531028654476,"aboveFit":false,"run":"no-thinking effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.47985674945758555,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"claudesonnet46claudecode","providerId":"anthropic","slug":"anthropic-claude-sonnet-4-6-claude-code","name":"Claude Sonnet 4.6 Claude Code","provider":"Anthropic","description":"Claude Sonnet 4.6 Claude Code is a model from Anthropic. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":62.37365806622165,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"coding":1},"benchmarks":[{"benchmarkId":"vals-ai:vibe-code:overall","benchmarkName":"Vibe Code Bench v1.1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":55.774,"unit":"percent","family":"vals-ai:vibe-code:overall","index":62.37365806622165,"aboveFit":false,"run":"Claude Code","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","measuredAt":"2026-11-09","stderr":4.682,"costPerTest":7.354471,"latency":2982.342}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":20,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"claudesonnet46thinkingauto","providerId":"anthropic","slug":"anthropic-claude-sonnet-4-6-thinking-auto","name":"Claude Sonnet 4.6 Thinking Auto","provider":"Anthropic","description":"Claude Sonnet 4.6 Thinking Auto is a model from Anthropic. It has 6 scored results in 6 categories. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":53.12215817021151,"coding":66.31233182918089,"reasoning":67.0131546061624,"multimodal":null,"knowledge":60.61636902445454,"multilingual":null,"instruction":54.50983030130954,"math":61.020040470890955},"capabilityCounts":{"math":1,"reasoning":1,"coding":1,"knowledge":1,"instruction":1,"agentic":1},"benchmarks":[{"benchmarkId":"livebench:livebench-math:overall","benchmarkName":"LiveBench Mathematics","category":"math","domain":null,"eligible":true,"ranked":true,"score":86.994,"unit":"percent","family":"livebench:livebench-math:overall","index":61.020040470890955,"aboveFit":false,"run":"medium effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Mathematics","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.23262928759894458,"latency":null},{"benchmarkId":"livebench:livebench-reasoning:overall","benchmarkName":"LiveBench Reasoning","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":84.76925,"unit":"percent","family":"livebench","index":71.4326896132659,"aboveFit":false,"run":"medium effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Reasoning","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.20732425742574256,"latency":null},{"benchmarkId":"livebench:livebench-coding:overall","benchmarkName":"LiveBench Coding","category":"coding","domain":null,"eligible":true,"ranked":true,"score":79.2715,"unit":"percent","family":"livebench:livebench-coding:overall","index":66.31233182918089,"aboveFit":false,"run":"medium effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Coding","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.005922222222222223,"latency":null},{"benchmarkId":"livebench:livebench-data-analysis:overall","benchmarkName":"LiveBench Data Analysis","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":77.946,"unit":"percent","family":"livebench","index":62.5936195990589,"aboveFit":false,"run":"medium effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Data%20Analysis","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.3262706666666667,"latency":null},{"benchmarkId":"livebench:livebench-language:overall","benchmarkName":"LiveBench Language","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":76.10266666666666,"unit":"percent","family":"livebench:livebench-language:overall","index":60.61636902445454,"aboveFit":false,"run":"medium effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Language","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.06102733333333333,"latency":null},{"benchmarkId":"livebench:livebench-instruction-following:overall","benchmarkName":"LiveBench Instruction Following","category":"instruction","domain":null,"eligible":true,"ranked":true,"score":63.22075,"unit":"percent","family":"livebench:livebench-instruction-following:overall","index":54.50983030130954,"aboveFit":false,"run":"medium effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=IF","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.2482945,"latency":null},{"benchmarkId":"livebench:livebench-agentic-coding:overall","benchmarkName":"LiveBench Agentic Coding","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":42.626,"unit":"percent","family":"livebench:livebench-agentic-coding:overall","index":53.12215817021151,"aboveFit":false,"run":"medium effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Agentic%20Coding","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.6304291666666667,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":85,"benchmarkCount":6,"categoryCount":6,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:livebench","name":"LiveBench","url":"https://livebench.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"claudesonnet5range","providerId":"anthropic","slug":"anthropic-claude-sonnet-5-range","name":"Claude Sonnet 5 Range","provider":"Anthropic","description":"Claude Sonnet 5 Range is a model from Anthropic. It has no scored result.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[{"benchmarkId":"vals-ai:cyber:overall","benchmarkName":"CyberBench v1.1","category":null,"domain":"Other","eligible":false,"ranked":true,"score":70.298,"unit":"percent","family":"vals-ai:cyber:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/cyber","measuredAt":"2026-11-05","stderr":5.652,"costPerTest":4.844769,"latency":1376.575}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"claudesonnetlatest","providerId":"anthropic","slug":"anthropic-claude-sonnet-latest","name":"Claude Sonnet Latest","provider":"Anthropic","description":"Claude Sonnet Latest is a model from Anthropic. It has no scored result.","logoUrl":null,"releaseDate":"2026-04-27T19:32:48.000Z","contextWindowTokens":1000000,"modalities":["text","image","file"],"openWeights":null,"details":{"apiId":"~anthropic/claude-sonnet-latest","maxOutputTokens":128000,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.09999999999999999,"weightsUrl":null},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":2,"outputPerMillion":10,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":107,"timeToFirstToken":1.613,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"cloudflare-clef","providerId":"cloudflare","slug":"cloudflare-clef","name":"Clef","provider":"Cloudflare","description":"Clef is a standard model from Cloudflare. It has no scored result.","logoUrl":null,"releaseDate":"2026-10-01","contextWindowTokens":64000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"cloudflare-clef-flash","providerId":"cloudflare","slug":"cloudflare-clef-flash","name":"Clef-Flash","provider":"Cloudflare","description":"Clef-Flash is a standard model from Cloudflare. It has no scored result.","logoUrl":null,"releaseDate":"2026-10-01","contextWindowTokens":64000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"cloudflare-clef-omni","providerId":"cloudflare","slug":"cloudflare-clef-omni","name":"Clef-Omni","provider":"Cloudflare","description":"Clef-Omni is a model from Cloudflare. It has no scored result.","logoUrl":null,"releaseDate":"2026-10-09","contextWindowTokens":64000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":0.15,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"clm-8b","providerId":"contrastive-lm","slug":"clm-8b","name":"CLM-8B","provider":"Contrastive-LM","description":"CLM-8B is a standard model from Contrastive-LM. It has 2 scored results in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":51.61687631028684,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"reasoning":2},"benchmarks":[{"benchmarkId":"jevBench14","benchmarkName":"jevBench14","category":"reasoning","score":8.570462299422305,"unit":"percent","family":"jevBench14","index":50.84662445945547,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"jevBench15","benchmarkName":"jevBench15","category":"reasoning","score":1.5079991770819052,"unit":"percent","family":"jevBench15","index":52.38712816111822,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":15,"benchmarkCount":2,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"decision-index-0-3-clm-v0-1-8b","providerId":"contrastive-lm","slug":"decision-index-0-3-clm-v0-1-8b","name":"CLM-v0.1-8B","provider":"Contrastive-LM","description":"CLM-v0.1-8B is a model from Contrastive-LM. It has no scored result.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"codeactv18","providerId":"unknown","slug":"unknown-codeact-v1-8","name":"CodeAct v1.8","provider":"Unknown","description":"CodeAct v1.8 is a model. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":26.023752651399715,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"coding":1},"benchmarks":[{"benchmarkId":"swe-bench:swe-bench-lite:overall","benchmarkName":"SWE-bench Lite","category":"coding","domain":null,"eligible":true,"ranked":true,"score":26.67,"unit":"percent","family":"swe-bench-lite","index":26.023752651399715,"aboveFit":false,"run":"OpenHands","source":"SWE-bench team","sourceUrl":"https://www.swebench.com/","measuredAt":"2025-09-11","stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":20,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:swe-bench-team","name":"SWE-bench team","url":"https://www.swebench.com","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"codeactv21claude35sonnet20241022","providerId":"unknown","slug":"unknown-codeact-v2-1-claude-3-5-sonnet-20241022","name":"CodeAct v2.1 (claude-3-5-sonnet-20241022)","provider":"Unknown","description":"CodeAct v2.1 (claude-3-5-sonnet-20241022) is a model. It has 2 scored results in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":39.27031019996144,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"coding":2},"benchmarks":[{"benchmarkId":"swe-bench:swe-bench-verified:overall","benchmarkName":"SWE-bench Verified","category":"coding","domain":null,"eligible":true,"ranked":true,"score":53,"unit":"percent","family":"swe-bench-verified","index":40.31847915200732,"aboveFit":false,"run":"OpenHands","source":"SWE-bench team","sourceUrl":"https://www.swebench.com/","measuredAt":"2026-02-26","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"swe-bench:swe-bench-lite:overall","benchmarkName":"SWE-bench Lite","category":"coding","domain":null,"eligible":true,"ranked":true,"score":41.67,"unit":"percent","family":"swe-bench-lite","index":38.22214124791556,"aboveFit":false,"run":"OpenHands","source":"SWE-bench team","sourceUrl":"https://www.swebench.com/","measuredAt":"2025-09-11","stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":20,"benchmarkCount":2,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:swe-bench-team","name":"SWE-bench team","url":"https://www.swebench.com","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"codestral2508","providerId":"mistral","slug":"mistral-codestral-2508","name":"Codestral 2508","provider":"Mistral","description":"Codestral 2508 is a model from Mistral. It has no scored result.","logoUrl":null,"releaseDate":"2025-08-01T20:20:30.000Z","contextWindowTokens":256000,"modalities":["text","file"],"openWeights":null,"details":{"apiId":"mistralai/codestral-2508","maxOutputTokens":204800,"knowledgeCutoff":"2025-03-31","retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.03,"weightsUrl":null},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":0.3,"outputPerMillion":0.8999999999999999,"batch":{"inputPerMillion":0.15,"outputPerMillion":0.44999999999999996},"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":111,"timeToFirstToken":0.282,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[{"host":"google-vertex","name":"Google Vertex AI","kind":"cloud","url":"https://cloud.google.com/vertex-ai/generative-ai/docs/models","inputPerMillion":0.3,"outputPerMillion":0.9,"cacheReadPerMillion":null,"contextTokens":128000,"maxOutputTokens":8192,"deprecated":false},{"host":"mistral","name":"Mistral","kind":"maker","url":"https://docs.mistral.ai/getting-started/models/","inputPerMillion":0.3,"outputPerMillion":0.9,"cacheReadPerMillion":0.03,"contextTokens":256000,"maxOutputTokens":8192,"deprecated":false},{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":0.3,"outputPerMillion":0.9,"cacheReadPerMillion":0.03,"contextTokens":256000,"maxOutputTokens":204800,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"codexminilatest","providerId":"openai","slug":"openai-codex-mini-latest","name":"Codex Mini (Latest)","provider":"OpenAI","description":"Codex Mini (Latest) is a model from OpenAI. It has 2 scored results in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":39.661781513481664,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"reasoning":2},"benchmarks":[{"benchmarkId":"arc-prize:v1-public-eval:overall","benchmarkName":"ARC-AGI-1 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":33.379999999999995,"unit":"percent","family":"arc-prize:v1-public-eval:overall","index":null,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.1269,"latency":null},{"benchmarkId":"arc-prize:v1-semi-private:overall","benchmarkName":"ARC-AGI-1 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":27.33,"unit":"percent","family":"arc-prize:v1-semi-private:overall","index":39.55194655448106,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.1597,"latency":null},{"benchmarkId":"arc-prize:v2-semi-private:overall","benchmarkName":"ARC-AGI-2 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":1.27,"unit":"percent","family":"arc-prize:v2-semi-private:overall","index":39.771616472482265,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.23,"latency":null},{"benchmarkId":"arc-prize:v2-public-eval:overall","benchmarkName":"ARC-AGI-2 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":0,"unit":"percent","family":"arc-prize:v2-public-eval:overall","index":null,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.2272,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":15,"benchmarkCount":2,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:arc-prize-foundation","name":"ARC Prize Foundation","url":"https://arcprize.org","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"cogitov1previewllama3b","providerId":"unknown","slug":"unknown-cogito-v1-preview-llama-3b","name":"Cogito V1 Preview Llama 3B","provider":"Unknown","description":"Cogito V1 Preview Llama 3B is a model. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":11.35711139428794,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-french:overall","benchmarkName":"EuroEval French","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":34.06624999999999,"unit":"percent","family":"euroeval","index":18.44029828769812,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/french","measuredAt":null,"stderr":0.5345412753006076,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-dutch:overall","benchmarkName":"EuroEval Dutch","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":32.342222222222226,"unit":"percent","family":"euroeval","index":16.101189280743867,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/dutch","measuredAt":null,"stderr":0.33861008567887757,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-portuguese:overall","benchmarkName":"EuroEval Portuguese","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":32.285000000000004,"unit":"percent","family":"euroeval","index":16.023551891646534,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/portuguese","measuredAt":null,"stderr":0.3867391724405481,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-italian:overall","benchmarkName":"EuroEval Italian","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":28.957500000000003,"unit":"percent","family":"euroeval","index":11.50890002758382,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/italian","measuredAt":null,"stderr":0.3826899038386041,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":28.733749999999997,"unit":"percent","family":"euroeval","index":11.205322760992061,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":0.3440158063229072,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":27.774285714285714,"unit":"percent","family":"euroeval","index":9.90355049172036,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.3792285402159043,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":27.10875,"unit":"percent","family":"euroeval","index":9.000571662839732,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.5305274498082073,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-spanish:overall","benchmarkName":"EuroEval Spanish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":24.865000000000002,"unit":"percent","family":"euroeval","index":5.956319185006308,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/spanish","measuredAt":null,"stderr":0.3428602120398341,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"cogitov1previewllama8b","providerId":"unknown","slug":"unknown-cogito-v1-preview-llama-8b","name":"Cogito V1 Preview Llama 8B","provider":"Unknown","description":"Cogito V1 Preview Llama 8B is a model. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":25.89490094302784,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-french:overall","benchmarkName":"EuroEval French","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":47.14874999999999,"unit":"percent","family":"euroeval","index":36.19024059020766,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/french","measuredAt":null,"stderr":0.6131157619079777,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-dutch:overall","benchmarkName":"EuroEval Dutch","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":45.70000000000001,"unit":"percent","family":"euroeval","index":34.22462018808571,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/dutch","measuredAt":null,"stderr":0.4611954407492569,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-portuguese:overall","benchmarkName":"EuroEval Portuguese","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":43.08125,"unit":"percent","family":"euroeval","index":30.67157899529404,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/portuguese","measuredAt":null,"stderr":0.5124817069906008,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-italian:overall","benchmarkName":"EuroEval Italian","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":40.3275,"unit":"percent","family":"euroeval","index":26.935373865117423,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/italian","measuredAt":null,"stderr":0.5972384364724025,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":38.793749999999996,"unit":"percent","family":"euroeval","index":24.854428020938244,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.6133718285020922,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":38.790000000000006,"unit":"percent","family":"euroeval","index":24.849340133788676,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":0.3637607386731009,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":38.355714285714285,"unit":"percent","family":"euroeval","index":24.260114345798943,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.3195213256592123,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-spanish:overall","benchmarkName":"EuroEval Spanish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":35.50125,"unit":"percent","family":"euroeval","index":20.38726310360497,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/spanish","measuredAt":null,"stderr":0.630386786029022,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"cogitov1previewqwen14b","providerId":"unknown","slug":"unknown-cogito-v1-preview-qwen-14b","name":"Cogito V1 Preview Qwen 14B","provider":"Unknown","description":"Cogito V1 Preview Qwen 14B is a model. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":44.178977176111125,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-french:overall","benchmarkName":"EuroEval French","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":58.75,"unit":"percent","family":"euroeval","index":51.93046746863219,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/french","measuredAt":null,"stderr":0.2833918444486362,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-portuguese:overall","benchmarkName":"EuroEval Portuguese","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":55.849999999999994,"unit":"percent","family":"euroeval","index":47.99583473962185,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/portuguese","measuredAt":null,"stderr":0.3763745640980538,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":53.98875000000001,"unit":"percent","family":"euroeval","index":45.470546751045845,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":0.25409151894543824,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-dutch:overall","benchmarkName":"EuroEval Dutch","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":53.501111111111115,"unit":"percent","family":"euroeval","index":44.80893298133533,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/dutch","measuredAt":null,"stderr":0.22558018475275354,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-italian:overall","benchmarkName":"EuroEval Italian","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":52.572500000000005,"unit":"percent","family":"euroeval","index":43.54902137088691,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/italian","measuredAt":null,"stderr":0.33789560148069403,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-spanish:overall","benchmarkName":"EuroEval Spanish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":51.205,"unit":"percent","family":"euroeval","index":41.69363852367255,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/spanish","measuredAt":null,"stderr":0.35442955929211095,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":50.449999999999996,"unit":"percent","family":"euroeval","index":40.6692772442233,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.25275623490967397,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":48.105,"unit":"percent","family":"euroeval","index":37.487651813351164,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.36800348164113883,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"cogitov1previewqwen32b","providerId":"unknown","slug":"unknown-cogito-v1-preview-qwen-32b","name":"Cogito V1 Preview Qwen 32B","provider":"Unknown","description":"Cogito V1 Preview Qwen 32B is a model. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":49.67606236070523,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-french:overall","benchmarkName":"EuroEval French","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":61.84125,"unit":"percent","family":"euroeval","index":56.12458244227122,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/french","measuredAt":null,"stderr":0.2978359447749717,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":59.85875,"unit":"percent","family":"euroeval","index":53.434786102525365,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":0.23368648869799896,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-portuguese:overall","benchmarkName":"EuroEval Portuguese","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":59.83125,"unit":"percent","family":"euroeval","index":53.397474930095086,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/portuguese","measuredAt":null,"stderr":0.35216739627625954,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-italian:overall","benchmarkName":"EuroEval Italian","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":57.48125000000001,"unit":"percent","family":"euroeval","index":50.20906564969018,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/italian","measuredAt":null,"stderr":0.2668010728988922,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-dutch:overall","benchmarkName":"EuroEval Dutch","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":56.69555555555556,"unit":"percent","family":"euroeval","index":49.14305907172027,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/dutch","measuredAt":null,"stderr":0.20929924232903038,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":56.25285714285714,"unit":"percent","family":"euroeval","index":48.542419187691266,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.1871374045007179,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-spanish:overall","benchmarkName":"EuroEval Spanish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":54.17500000000001,"unit":"percent","family":"euroeval","index":45.723245146141764,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/spanish","measuredAt":null,"stderr":0.28818667994895253,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":53.50375,"unit":"percent","family":"euroeval","index":44.812513346366515,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.39860890920801056,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"cogitov2previewllama70b","providerId":"unknown","slug":"unknown-cogito-v2-preview-llama-70b","name":"Cogito V2 Preview Llama 70B","provider":"Unknown","description":"Cogito V2 Preview Llama 70B is a model. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":47.93163044940093,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-french:overall","benchmarkName":"EuroEval French","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":59.790000000000006,"unit":"percent","family":"euroeval","index":53.341508171449696,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/french","measuredAt":null,"stderr":0.34223073357020406,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-portuguese:overall","benchmarkName":"EuroEval Portuguese","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":56.6125,"unit":"percent","family":"euroeval","index":49.03037179337026,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/portuguese","measuredAt":null,"stderr":0.25627438908326367,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":54.99285714285713,"unit":"percent","family":"euroeval","index":46.8328891054316,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.23895606290697044,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":50.92999999999999,"unit":"percent","family":"euroeval","index":41.32052679936984,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.4089047872060194,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"commandr","providerId":"cohere","slug":"cohere-command-r","name":"Command R","provider":"Cohere","description":"Command R is a model from Cohere. It has no scored result.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[{"benchmarkId":"vals-ai:legal_bench:overall","benchmarkName":"LegalBench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":32.965,"unit":"percent","family":"vals-ai:legal_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","measuredAt":"2026-10-13","stderr":0.71,"costPerTest":null,"latency":0.244}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":{"elo":1203.8379514458884,"eloUpper":1208.1207209240338,"eloLower":1199.555181967743,"votes":77554,"rank":323,"rankUpper":null,"rankLower":null,"arenaName":"Command R+","organization":"cohere","updatedAt":"2026-11-09"},"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"commandr082024","providerId":"cohere","slug":"cohere-command-r-08-2024","name":"Command R (08-2024)","provider":"Cohere","description":"Command R (08-2024) is a model from Cohere. It has no scored result.","logoUrl":null,"releaseDate":"2024-08-30T00:00:00.000Z","contextWindowTokens":128000,"modalities":["text"],"openWeights":true,"details":{"apiId":"cohere/command-r-08-2024","maxOutputTokens":4000,"knowledgeCutoff":"2024-03-31","retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":null,"weightsUrl":"https://huggingface.co/CohereLabs/c4ai-command-r-08-2024"},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":0.15,"outputPerMillion":0.6,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":28,"timeToFirstToken":0.663,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":{"elo":1187.3663461094707,"eloUpper":1193.8854707719788,"eloLower":1180.8472214469625,"votes":10140,"rank":330,"rankUpper":null,"rankLower":null,"arenaName":"Command R-08-2024","organization":"cohere","updatedAt":"2026-11-09"},"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[{"host":"cohere","name":"Cohere","kind":"maker","url":"https://docs.cohere.com/docs/models","inputPerMillion":0.15,"outputPerMillion":0.6,"cacheReadPerMillion":null,"contextTokens":128000,"maxOutputTokens":4000,"deprecated":false},{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":0.15,"outputPerMillion":0.6,"cacheReadPerMillion":null,"contextTokens":128000,"maxOutputTokens":4000,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"commandrplus","providerId":"cohere","slug":"cohere-command-r-plus","name":"Command R Plus","provider":"Cohere","description":"Command R Plus is a model from Cohere. It has 3 scored results in 2 categories. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":5,"reasoning":null,"multimodal":null,"knowledge":5.348242010429033,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"knowledge":2,"coding":1},"benchmarks":[{"benchmarkId":"vals-ai:legal_bench:overall","benchmarkName":"LegalBench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":68.287,"unit":"percent","family":"vals-ai:legal_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","measuredAt":"2026-10-13","stderr":0.65,"costPerTest":null,"latency":0.396},{"benchmarkId":"vals-ai:mmlu_pro:overall","benchmarkName":"MMLU Pro","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":44.003,"unit":"percent","family":"mmlu-pro","index":5,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","measuredAt":"2026-09-01","stderr":0.479,"costPerTest":0.00566,"latency":4.419},{"benchmarkId":"vals-ai:gpqa:overall","benchmarkName":"GPQA Diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":31.06,"unit":"percent","family":"gpqa-diamond","index":5.696484020858058,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","measuredAt":"2026-09-01","stderr":2.28,"costPerTest":0.007438,"latency":8.194},{"benchmarkId":"vals-ai:lcb:overall","benchmarkName":"LiveCodeBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":18.238,"unit":"percent","family":"livecodebench","index":5,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","measuredAt":"2026-09-01","stderr":1.016,"costPerTest":0.003267,"latency":5.463},{"benchmarkId":"vals-ai:medqa:overall","benchmarkName":"MedQA","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":2.651,"unit":"percent","family":"vals-ai:medqa:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medqa","measuredAt":"2026-04-16","stderr":0.29,"costPerTest":0.003456,"latency":6.056}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":35,"benchmarkCount":3,"categoryCount":2,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"commandr7b122024","providerId":"cohere","slug":"cohere-command-r7b-12-2024","name":"Command R7B (12-2024)","provider":"Cohere","description":"Command R7B (12-2024) is a model from Cohere. It has no scored result.","logoUrl":null,"releaseDate":"2024-12-14T06:35:52.000Z","contextWindowTokens":128000,"modalities":["text"],"openWeights":true,"details":{"apiId":"cohere/command-r7b-12-2024","maxOutputTokens":4000,"knowledgeCutoff":"2024-08-31","retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":null,"weightsUrl":"https://huggingface.co/CohereLabs/c4ai-command-r7b-12-2024"},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":0.0375,"outputPerMillion":0.15,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":67,"timeToFirstToken":0.2605,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[{"host":"cohere","name":"Cohere","kind":"maker","url":"https://docs.cohere.com/docs/models","inputPerMillion":0.0375,"outputPerMillion":0.15,"cacheReadPerMillion":null,"contextTokens":128000,"maxOutputTokens":4000,"deprecated":false},{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":0.0375,"outputPerMillion":0.15,"cacheReadPerMillion":null,"contextTokens":128000,"maxOutputTokens":4000,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"composer-2","providerId":"cursor","slug":"composer-2","name":"Composer 2","provider":"Cursor","description":"Composer 2 is a reasoning model from Cursor in the Composer family. It has 2 scored results in 2 categories. It has no index.","logoUrl":null,"releaseDate":"2026-03-19","contextWindowTokens":200000,"modalities":["text"],"openWeights":false,"details":null,"reasoning":true,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":57.787655656273536,"coding":74.18426883434671,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"coding":1,"agentic":1},"benchmarks":[{"benchmarkId":"reactNativeEvals","benchmarkName":"React Native Evals","category":"coding","score":96.1,"unit":"percent","family":"reactNativeEvals","index":74.18426883434671,"aboveFit":false,"run":null,"source":"Callstack","sourceUrl":"https://rn-evals.vercel.app/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"terminalBench2","benchmarkName":"Terminal-Bench 2.0","category":"agentic","score":61.7,"unit":"percent","family":"terminalBench2","index":57.787655656273536,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"sweRebench","benchmarkName":"SWE-Rebench","category":"coding","score":58,"unit":"percent","family":"sweRebench","index":null,"aboveFit":false,"run":null,"source":"Nebius","sourceUrl":"https://swe-rebench.com","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":0.5,"outputPerMillion":2.5,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":40,"benchmarkCount":2,"categoryCount":2,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"composer-2-fast","providerId":"cursor","slug":"composer-2-fast","name":"Composer 2 Fast","provider":"Cursor","description":"Composer 2 Fast is a reasoning model from Cursor in the Composer family. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":"2026-03-19","contextWindowTokens":200000,"modalities":["text"],"openWeights":false,"details":null,"reasoning":true,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":72.88693920179156,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"coding":1},"benchmarks":[{"benchmarkId":"reactNativeEvals","benchmarkName":"React Native Evals","category":"coding","score":94.9,"unit":"percent","family":"reactNativeEvals","index":72.88693920179156,"aboveFit":false,"run":null,"source":"Callstack","sourceUrl":"https://rn-evals.vercel.app/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":20,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"composer-2-5","providerId":"cursor","slug":"composer-2-5","name":"Composer 2.5","provider":"Cursor","description":"Composer 2.5 is a reasoning model from Cursor in the Composer family. It has 6 scored results in 2 categories. It has no index.","logoUrl":null,"releaseDate":"2026-05-18","contextWindowTokens":200000,"modalities":["text"],"openWeights":false,"details":null,"reasoning":true,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":59.253173523254745,"coding":59.773175771219705,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"coding":4,"agentic":2},"benchmarks":[{"benchmarkId":"vals-ai:swebench:overall","benchmarkName":"SWE-bench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":79.6,"unit":"percent","family":"swe-bench-verified","index":59.93751072105352,"aboveFit":false,"run":"Cursor CLI","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/swebench","measuredAt":"2026-09-01","stderr":1.804,"costPerTest":null,"latency":0},{"benchmarkId":"terminalBench2","benchmarkName":"Terminal-Bench 2.0","category":"agentic","score":69.3,"unit":"percent","family":"terminalBench2","index":63.1938814590918,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"cursorBench31","benchmarkName":"cursorBench31","category":"coding","score":63.2,"unit":"percent","family":"cursorBench31","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:terminal-bench-2-1:overall","benchmarkName":"Terminal-Bench 2.1","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":58.427,"unit":"percent","family":"terminal-bench-2-1","index":55.31246558741768,"aboveFit":false,"run":"Cursor CLI","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2-1","measuredAt":"2026-10-01","stderr":5.254,"costPerTest":null,"latency":0},{"benchmarkId":"cursorBench32","benchmarkName":"cursorBench32","category":"coding","score":56.1,"unit":"percent","family":"cursorBench32","index":61.93113911775612,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:vibe-code:overall","benchmarkName":"Vibe Code Bench v1.1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":49.614,"unit":"percent","family":"vals-ai:vibe-code:overall","index":60.15103328536314,"aboveFit":false,"run":"Cursor CLI","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","measuredAt":"2026-11-09","stderr":4.972,"costPerTest":0.502233,"latency":592.9},{"benchmarkId":"cursorBench40","benchmarkName":"cursorBench40","category":"coding","score":27.7,"unit":"percent","family":"cursorBench40","index":57.07301996070605,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":0.5,"outputPerMillion":2.5,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":40,"benchmarkCount":6,"categoryCount":2,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"cosmos3-edge","providerId":"nvidia","slug":"cosmos3-edge","name":"Cosmos3-Edge","provider":"NVIDIA","description":"Cosmos3-Edge is a reasoning model from NVIDIA in the Cosmos 3 family. It has no scored result.","logoUrl":null,"releaseDate":"2026-07-20","contextWindowTokens":256000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":true,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"cosmos3-nano","providerId":"nvidia","slug":"cosmos3-nano","name":"Cosmos3-Nano","provider":"NVIDIA","description":"Cosmos3-Nano is a reasoning model from NVIDIA in the Cosmos 3 family. It has no scored result.","logoUrl":null,"releaseDate":"2026-05-31","contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":true,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"cosmos3-super","providerId":"nvidia","slug":"cosmos3-super","name":"Cosmos3-Super","provider":"NVIDIA","description":"Cosmos3-Super is a reasoning model from NVIDIA in the Cosmos 3 family. It has no scored result.","logoUrl":null,"releaseDate":"2026-05-31","contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":true,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"croutonalpha","providerId":"stealth","slug":"stealth-crouton-alpha","name":"Crouton Alpha","provider":"Stealth","description":"Crouton Alpha is a model from Stealth. It has no scored result.","logoUrl":null,"releaseDate":"2026-11-03T01:38:50.000Z","contextWindowTokens":1048576,"modalities":["text","image","video","file","audio"],"openWeights":null,"details":{"apiId":"stealth/crouton-alpha","maxOutputTokens":943718,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":null,"weightsUrl":null},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":0,"outputPerMillion":0,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":33,"timeToFirstToken":2.486,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"cydonia24bv41","providerId":"thedrummer","slug":"thedrummer-cydonia-24b-v4-1","name":"Cydonia 24B V4.1","provider":"TheDrummer","description":"Cydonia 24B V4.1 is a model from TheDrummer. It has no scored result.","logoUrl":null,"releaseDate":"2025-09-27T00:11:18.000Z","contextWindowTokens":131072,"modalities":["text"],"openWeights":true,"details":{"apiId":"thedrummer/cydonia-24b-v4.1","maxOutputTokens":117964,"knowledgeCutoff":"2024-04-30","retiresAt":null,"tools":false,"structuredOutput":true,"cacheReadPerMillion":0.15,"weightsUrl":"https://huggingface.co/thedrummer/cydonia-24b-v4.1"},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":0.3,"outputPerMillion":0.5,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":40,"timeToFirstToken":0.762,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":0.3,"outputPerMillion":0.5,"cacheReadPerMillion":0.15,"contextTokens":131072,"maxOutputTokens":117964,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"cygnet","providerId":"blockbrain","slug":"cygnet","name":"Cygnet","provider":"blockbrain","description":"Cygnet is a standard model from blockbrain. It has 2 scored results in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":95,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"reasoning":2},"benchmarks":[{"benchmarkId":"jevBench15","benchmarkName":"jevBench15","category":"reasoning","score":73.7012674756234,"unit":"percent","family":"jevBench15","index":95,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"jevBench14","benchmarkName":"jevBench14","category":"reasoning","score":61.76221691847706,"unit":"percent","family":"jevBench14","index":95,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":15,"benchmarkCount":2,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"liquid-d1","providerId":"liquidai","slug":"liquid-d1","name":"d1","provider":"LiquidAI","description":"d1 is a model from LiquidAI. It has no scored result.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":false,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"d1-2b","providerId":"liquidai","slug":"d1-2b","name":"d1-2B","provider":"LiquidAI","description":"d1-2B is a standard model from LiquidAI in the Liquid d1 open family. It has no scored result.","logoUrl":null,"releaseDate":"2026-10-27","contextWindowTokens":128000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"d1-3b","providerId":"liquidai","slug":"d1-3b","name":"d1-3B","provider":"LiquidAI","description":"d1-3B is a standard model from LiquidAI in the Liquid d1 open family. It has no scored result.","logoUrl":null,"releaseDate":"2026-10-07","contextWindowTokens":32000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"d1-omni-2b","providerId":"liquidai","slug":"d1-omni-2b","name":"d1-Omni-2B","provider":"LiquidAI","description":"d1-Omni-2B is a standard model from LiquidAI in the Liquid d1 open family. It has no scored result.","logoUrl":null,"releaseDate":"2026-10-27","contextWindowTokens":128000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"d1-omni-600m","providerId":"liquidai","slug":"d1-omni-600m","name":"d1-omni-600M","provider":"LiquidAI","description":"d1-omni-600M is a standard model from LiquidAI in the Liquid d1 open family. It has no scored result.","logoUrl":null,"releaseDate":"2026-10-07","contextWindowTokens":16000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"d1-ultra","providerId":"liquidai","slug":"d1-ultra","name":"d1-ultra","provider":"LiquidAI","description":"d1-ultra is a standard model from LiquidAI. It has no scored result.","logoUrl":null,"releaseDate":"2026-10-15","contextWindowTokens":null,"modalities":["text"],"openWeights":false,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":0.042,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"dbrx","providerId":"databricks","slug":"databricks-dbrx","name":"DBRX","provider":"Databricks","description":"DBRX is a model from Databricks. It has 2 scored results in 2 categories. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":7.37620702782479,"multilingual":null,"instruction":null,"math":5},"capabilityCounts":{"knowledge":1,"math":1},"benchmarks":[{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":32.891414141414145,"unit":"percent","family":"gpqa-diamond","index":7.37620702782479,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":3.256944333564935,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-math-level-5:overall","benchmarkName":"MATH level 5","category":"math","domain":null,"eligible":true,"ranked":true,"score":11.650302114803624,"unit":"percent","family":"epoch-ai:epoch-math-level-5:overall","index":5,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0.7355853708151388,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":25,"benchmarkCount":2,"categoryCount":2,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"dbrx-instruct","providerId":"databricks","slug":"dbrx-instruct","name":"DBRX Instruct","provider":"Databricks","description":"DBRX Instruct is a standard model from Databricks in the DBRX family. It has no scored result.","logoUrl":null,"releaseDate":null,"contextWindowTokens":32000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"decision-index-0-3-decider-4b","providerId":"mapika","slug":"decision-index-0-3-decider-4b","name":"Decider 4B","provider":"Mapika","description":"Decider 4B is a model from Mapika. It has no scored result.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"decider-2b","providerId":"mapika","slug":"decider-2b","name":"decider-2b","provider":"Mapika","description":"decider-2b is a standard model from Mapika. It has 2 scored results in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":75.03035911249897,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"reasoning":2},"benchmarks":[{"benchmarkId":"jevBench15","benchmarkName":"jevBench15","category":"reasoning","score":45.09827651852322,"unit":"percent","family":"jevBench15","index":78.78413337819877,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"jevBench14","benchmarkName":"jevBench14","category":"reasoning","score":30.7375448157181,"unit":"percent","family":"jevBench14","index":71.27658484679918,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":15,"benchmarkCount":2,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"decider-35b-a3b","providerId":"mapika","slug":"decider-35b-a3b","name":"decider-35b-a3b","provider":"Mapika","description":"decider-35b-a3b is a standard model from Mapika. It has 2 scored results in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":74.51465967325642,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"reasoning":2},"benchmarks":[{"benchmarkId":"jevBench14","benchmarkName":"jevBench14","category":"reasoning","score":41.18326776449841,"unit":"percent","family":"jevBench14","index":80.90372932572944,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"jevBench15","benchmarkName":"jevBench15","category":"reasoning","score":27.497459411252194,"unit":"percent","family":"jevBench15","index":68.12559002078338,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":15,"benchmarkCount":2,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"decider-4b-v2","providerId":"mapika","slug":"decider-4b-v2","name":"decider-4b v2","provider":"Mapika","description":"decider-4b v2 is a standard model from Mapika. It has 2 scored results in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":94.82077633716273,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"reasoning":2},"benchmarks":[{"benchmarkId":"jevBench15","benchmarkName":"jevBench15","category":"reasoning","score":71.28417523518583,"unit":"percent","family":"jevBench15","index":94.64155267432547,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"jevBench14","benchmarkName":"jevBench14","category":"reasoning","score":64.12889492043318,"unit":"percent","family":"jevBench14","index":95,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":15,"benchmarkCount":2,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"decision-2b","providerId":"flymy-ai","slug":"decision-2b","name":"Decision 2B v59","provider":"FlyMy.AI","description":"Decision 2B v59 is a standard model from FlyMy.AI. It has 2 scored results in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":70.52595105042718,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"reasoning":2},"benchmarks":[{"benchmarkId":"jevBench14","benchmarkName":"jevBench14","category":"reasoning","score":35.800088231124995,"unit":"percent","family":"jevBench14","index":75.94240234262604,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"jevBench15","benchmarkName":"jevBench15","category":"reasoning","score":22.51688642767283,"unit":"percent","family":"jevBench15","index":65.10949975822834,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":15,"benchmarkCount":2,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"decision-4b-v11","providerId":"flymy-ai","slug":"decision-4b-v11","name":"Decision 4B v1.1","provider":"FlyMy.AI","description":"Decision 4B v1.1 is a standard model from FlyMy.AI. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":94.09749348667937,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"reasoning":1},"benchmarks":[{"benchmarkId":"jevBench15","benchmarkName":"jevBench15","category":"reasoning","score":70.38575169491591,"unit":"percent","family":"jevBench15","index":94.09749348667937,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":15,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"decision-4b-v12","providerId":"flymy-ai","slug":"decision-4b-v12","name":"Decision 4B v1.2","provider":"FlyMy.AI","description":"Decision 4B v1.2 is a standard model from FlyMy.AI. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":94.36493091036424,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"reasoning":1},"benchmarks":[{"benchmarkId":"jevBench15","benchmarkName":"jevBench15","category":"reasoning","score":70.82738025747577,"unit":"percent","family":"jevBench15","index":94.36493091036424,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":15,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"decision-fast","providerId":"flymy-ai","slug":"decision-fast","name":"Decision Fast v53a","provider":"FlyMy.AI","description":"Decision Fast v53a is a standard model from FlyMy.AI. It has 2 scored results in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":62.18383372373992,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"reasoning":2},"benchmarks":[{"benchmarkId":"jevBench14","benchmarkName":"jevBench14","category":"reasoning","score":32.492203456270644,"unit":"percent","family":"jevBench14","index":72.89373976528907,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"jevBench15","benchmarkName":"jevBench15","category":"reasoning","score":0,"unit":"percent","family":"jevBench15","index":51.47392768219078,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":15,"benchmarkCount":2,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"evalengine-decision-4b","providerId":"eval-engine-chromia","slug":"evalengine-decision-4b","name":"Decision-4B (Eval Engine / Chromia)","provider":"Eval Engine / Chromia","description":"Decision-4B (Eval Engine / Chromia) is a standard model from Eval Engine / Chromia. It has no scored result.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"decision-machine-1","providerId":"milliseconds-ai","slug":"decision-machine-1","name":"decision-machine-1","provider":"milliseconds.ai","description":"decision-machine-1 is a standard model from milliseconds.ai. It has 2 scored results in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":false,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":66.59532632669496,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"reasoning":2},"benchmarks":[{"benchmarkId":"jevBench14","benchmarkName":"jevBench14","category":"reasoning","score":39.93541656481836,"unit":"percent","family":"jevBench14","index":79.75366591255717,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"jevBench15","benchmarkName":"jevBench15","category":"reasoning","score":3.2416665489183343,"unit":"percent","family":"jevBench15","index":53.43698674083276,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":15,"benchmarkCount":2,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"deem-0-8-v1","providerId":"libertai","slug":"deem-0-8-v1","name":"Deem 0.8B v1","provider":"LibertAI","description":"Deem 0.8B v1 is a standard model from LibertAI. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":52.7685144606772,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"reasoning":1},"benchmarks":[{"benchmarkId":"jevBench15","benchmarkName":"jevBench15","category":"reasoning","score":2.1377954147719667,"unit":"percent","family":"jevBench15","index":52.7685144606772,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":15,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"deepseek-coder-2-0","providerId":"deepseek","slug":"deepseek-coder-2-0","name":"DeepSeek Coder 2.0","provider":"DeepSeek","description":"DeepSeek Coder 2.0 is a standard model from DeepSeek. It has no scored result.","logoUrl":null,"releaseDate":null,"contextWindowTokens":128000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"deepseekflashlatest","providerId":"deepseek","slug":"deepseek-deepseek-flash-latest","name":"DeepSeek Flash Latest","provider":"DeepSeek","description":"DeepSeek Flash Latest is a model from DeepSeek. It has no scored result.","logoUrl":null,"releaseDate":"2026-09-14T15:19:10.000Z","contextWindowTokens":1048576,"modalities":["text","image"],"openWeights":null,"details":{"apiId":"~deepseek/deepseek-flash-latest","maxOutputTokens":943718,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.004,"weightsUrl":null},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":0.022,"outputPerMillion":0.39999999999999997,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":110,"timeToFirstToken":1.305,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"deepseekflashlow","providerId":"deepseek","slug":"deepseek-deepseek-flash-low","name":"DeepSeek Flash Low","provider":"DeepSeek","description":"DeepSeek Flash Low is a model from DeepSeek. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":60.798654770354176,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":70.765,"unit":"percent","family":"euroeval","index":68.23205789589396,"aboveFit":false,"run":"low effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":0.578077362902233,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-french:overall","benchmarkName":"EuroEval French","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":68.37125,"unit":"percent","family":"euroeval","index":64.98428993207725,"aboveFit":false,"run":"low effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/french","measuredAt":null,"stderr":0.4398081968313005,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-portuguese:overall","benchmarkName":"EuroEval Portuguese","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":66.98875,"unit":"percent","family":"euroeval","index":63.10855553626455,"aboveFit":false,"run":"low effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/portuguese","measuredAt":null,"stderr":0.4860009002049276,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-italian:overall","benchmarkName":"EuroEval Italian","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":65.99375,"unit":"percent","family":"euroeval","index":61.758569479242055,"aboveFit":false,"run":"low effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/italian","measuredAt":null,"stderr":0.4064614834150955,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-spanish:overall","benchmarkName":"EuroEval Spanish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":64.57874999999999,"unit":"percent","family":"euroeval","index":59.8387400614663,"aboveFit":false,"run":"low effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/spanish","measuredAt":null,"stderr":0.4313894702006529,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-dutch:overall","benchmarkName":"EuroEval Dutch","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":64.3288888888889,"unit":"percent","family":"euroeval","index":59.499736025092304,"aboveFit":false,"run":"low effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/dutch","measuredAt":null,"stderr":0.418203093198142,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":62.660000000000004,"unit":"percent","family":"euroeval","index":57.235437803263366,"aboveFit":false,"run":"low effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.4694982123805919,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":60.74374999999999,"unit":"percent","family":"euroeval","index":54.635527469826776,"aboveFit":false,"run":"low effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.583794163211658,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"deepseekflashnothinking","providerId":"deepseek","slug":"deepseek-deepseek-flash-no-thinking","name":"DeepSeek Flash No Thinking","provider":"DeepSeek","description":"DeepSeek Flash No Thinking is a model from DeepSeek. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":62.612203878783085,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-french:overall","benchmarkName":"EuroEval French","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":72.2375,"unit":"percent","family":"euroeval","index":70.22990158329661,"aboveFit":false,"run":"no-thinking effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/french","measuredAt":null,"stderr":0.4822603031558787,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-portuguese:overall","benchmarkName":"EuroEval Portuguese","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":69.8975,"unit":"percent","family":"euroeval","index":67.05506000195723,"aboveFit":false,"run":"no-thinking effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/portuguese","measuredAt":null,"stderr":0.5272155631238516,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":69.0825,"unit":"percent","family":"euroeval","index":65.94929252811468,"aboveFit":false,"run":"no-thinking effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":0.7699786928870175,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-italian:overall","benchmarkName":"EuroEval Italian","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":67.8725,"unit":"percent","family":"euroeval","index":64.30760094118278,"aboveFit":false,"run":"no-thinking effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/italian","measuredAt":null,"stderr":0.5102358523271371,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-dutch:overall","benchmarkName":"EuroEval Dutch","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":65.37333333333333,"unit":"percent","family":"euroeval","index":60.91680681638337,"aboveFit":false,"run":"no-thinking effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/dutch","measuredAt":null,"stderr":0.5338065030664664,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":64.39285714285714,"unit":"percent","family":"euroeval","index":59.5865262270513,"aboveFit":false,"run":"no-thinking effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.5563639243575472,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-spanish:overall","benchmarkName":"EuroEval Spanish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":62.670000000000016,"unit":"percent","family":"euroeval","index":57.249005502328934,"aboveFit":false,"run":"no-thinking effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/spanish","measuredAt":null,"stderr":0.5010987926547019,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":59.47875,"unit":"percent","family":"euroeval","index":52.91921353803435,"aboveFit":false,"run":"no-thinking effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.6624587251293472,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"deepseek-llm-2-0","providerId":"deepseek","slug":"deepseek-llm-2-0","name":"DeepSeek LLM 2.0","provider":"DeepSeek","description":"DeepSeek LLM 2.0 is a standard model from DeepSeek. It has no scored result.","logoUrl":null,"releaseDate":null,"contextWindowTokens":128000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"deepseekllm67b","providerId":"deepseek","slug":"deepseek-deepseek-llm-67b","name":"DeepSeek LLM 67B","provider":"DeepSeek","description":"DeepSeek LLM 67B is a model from DeepSeek. It has 3 scored results in 2 categories. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":5,"multilingual":null,"instruction":null,"math":7.201959624004488},"capabilityCounts":{"knowledge":1,"math":2},"benchmarks":[{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":24.62121212121212,"unit":"percent","family":"gpqa-diamond","index":5,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.3623351950761946,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-math-level-5:overall","benchmarkName":"MATH level 5","category":"math","domain":null,"eligible":true,"ranked":true,"score":6.391616314199395,"unit":"percent","family":"epoch-ai:epoch-math-level-5:overall","index":5,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0.40168441495506657,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":0.8333333333333334,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":9.403919248008975,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0.4700633956814719,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":25,"benchmarkCount":3,"categoryCount":2,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"deepseekprolatest","providerId":"deepseek","slug":"deepseek-deepseek-pro-latest","name":"DeepSeek Pro Latest","provider":"DeepSeek","description":"DeepSeek Pro Latest is a model from DeepSeek. It has no scored result.","logoUrl":null,"releaseDate":"2026-09-14T15:19:34.000Z","contextWindowTokens":1048576,"modalities":["text"],"openWeights":null,"details":{"apiId":"~deepseek/deepseek-pro-latest","maxOutputTokens":393216,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.0999,"weightsUrl":null},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":0.0999,"outputPerMillion":8,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":42,"timeToFirstToken":0.371,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"deepseekr1distillqwen15b","providerId":"deepseek","slug":"deepseek-deepseek-r1-distill-qwen-1-5b","name":"DeepSeek R1 Distill Qwen 1.5B","provider":"DeepSeek","description":"DeepSeek R1 Distill Qwen 1.5B is a model from DeepSeek. It has 2 scored results in 2 categories. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":8.013132496519077,"multilingual":null,"instruction":null,"math":21.363019076966854},"capabilityCounts":{"knowledge":1,"math":1},"benchmarks":[{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":33.58585858585859,"unit":"percent","family":"gpqa-diamond","index":8.013132496519077,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.059241816803904,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":21.38888888888889,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":21.363019076966854,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":5.279771021800918,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":25,"benchmarkCount":2,"categoryCount":2,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"deepseekr1distillqwen14b","providerId":"deepseek","slug":"deepseek-deepseek-r1-distill-qwen-14b","name":"DeepSeek R1 Distill Qwen 14B","provider":"DeepSeek","description":"DeepSeek R1 Distill Qwen 14B is a model from DeepSeek. It has 3 scored results in 2 categories. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":18.2039399956278,"multilingual":null,"instruction":null,"math":39.825038197719834},"capabilityCounts":{"math":2,"knowledge":1},"benchmarks":[{"benchmarkId":"epoch-ai:epoch-math-level-5:overall","benchmarkName":"MATH level 5","category":"math","domain":null,"eligible":true,"ranked":true,"score":87.12235649546828,"unit":"percent","family":"epoch-ai:epoch-math-level-5:overall","index":41.31806431792448,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0.7486217204155208,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":50.55555555555556,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":38.3320120775152,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":5.984981671790839,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":44.696969696969695,"unit":"percent","family":"gpqa-diamond","index":18.2039399956278,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.863094678190378,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":25,"benchmarkCount":3,"categoryCount":2,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"deepseek-r1-distill-qwen-32b","providerId":"deepseek","slug":"deepseek-r1-distill-qwen-32b","name":"DeepSeek R1 Distill Qwen 32B","provider":"DeepSeek","description":"DeepSeek R1 Distill Qwen 32B is a reasoning model from DeepSeek in the DeepSeek R1 Distill family. It has 5 scored results in 4 categories. It has no index.","logoUrl":null,"releaseDate":"2025-01-20","contextWindowTokens":128000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":true,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":13.266950031765951,"multimodal":null,"knowledge":31.914163811346363,"multilingual":null,"instruction":10.423922471660504,"math":41.240982306180626},"capabilityCounts":{"knowledge":2,"math":1,"instruction":1,"reasoning":1},"benchmarks":[{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":64.14141414141415,"unit":"percent","family":"gpqa-diamond","index":36.03785311906807,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.753417701646176,"costPerTest":null,"latency":null},{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":61.5,"unit":"percent","family":"gpqa-diamond","index":33.61522024541631,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":55.55555555555556,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":41.240982306180626,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":6.199996379565215,"costPerTest":null,"latency":null},{"benchmarkId":"aaIfBench","benchmarkName":"Artificial Analysis IFBench","category":"instruction","score":22.9,"unit":"percent","family":"ifbench-family","index":10.423922471660504,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":8.7,"unit":"percent","family":"lcr","index":13.266950031765951,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":4.6,"unit":"percent","family":"hle","index":29.001790940450533,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":60,"timeToFirstToken":0.84,"source":"BenchLM","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":45,"benchmarkCount":5,"categoryCount":4,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"deepseekr10528qwen38b","providerId":"deepseek","slug":"deepseek-deepseek-r1-qwen3-8b","name":"DeepSeek R1 Qwen3 8B","provider":"DeepSeek","description":"DeepSeek R1 Qwen3 8B is a model from DeepSeek. It has 2 scored results in 2 categories. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":5,"multilingual":null,"instruction":null,"math":34.45338510596129},"capabilityCounts":{"math":1,"knowledge":1},"benchmarks":[{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":43.888888888888886,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":34.45338510596129,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":6.772621817662119,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":9.280303030303031,"unit":"percent","family":"gpqa-diamond","index":5,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.0663485573693416,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":25,"benchmarkCount":2,"categoryCount":2,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"deepseekv31terminus","providerId":"deepseek","slug":"deepseek-deepseek-v3-1-terminus","name":"DeepSeek V3.1 Terminus","provider":"DeepSeek","description":"DeepSeek V3.1 Terminus is a model from DeepSeek. It has no scored result.","logoUrl":null,"releaseDate":"2025-09-22T13:37:55.000Z","contextWindowTokens":163840,"modalities":["text"],"openWeights":true,"details":{"apiId":"deepseek/deepseek-v3.1-terminus","maxOutputTokens":147456,"knowledgeCutoff":"2025-03-31","retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":null,"weightsUrl":"https://huggingface.co/deepseek-ai/DeepSeek-V3.1-Terminus"},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":0.27,"outputPerMillion":1,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":13,"timeToFirstToken":2.356,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":{"elo":1417.2197647051198,"eloUpper":1426.9504474370096,"eloLower":1407.4890819732295,"votes":3605,"rank":128,"rankUpper":null,"rankLower":null,"arenaName":"DeepSeek-V3.1-Terminus","organization":"deepseek","updatedAt":"2026-11-09"},"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"deepseekv32exp","providerId":"deepseek","slug":"deepseek-deepseek-v3-2-exp","name":"DeepSeek V3.2 Exp","provider":"DeepSeek","description":"DeepSeek V3.2 Exp is a model from DeepSeek. It has no scored result.","logoUrl":null,"releaseDate":"2025-09-29T12:54:41.000Z","contextWindowTokens":163840,"modalities":["text"],"openWeights":true,"details":{"apiId":"deepseek/deepseek-v3.2-exp","maxOutputTokens":147456,"knowledgeCutoff":"2025-07-31","retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":null,"weightsUrl":"https://huggingface.co/deepseek-ai/DeepSeek-V3.2-Exp"},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":0.27,"outputPerMillion":0.41,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":16,"timeToFirstToken":2.239,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":{"elo":1422.4017320014368,"eloUpper":1428.7458536129293,"eloLower":1416.0576103899443,"votes":11955,"rank":112,"rankUpper":null,"rankLower":null,"arenaName":"DeepSeek-V3.2-Exp","organization":"deepseek","updatedAt":"2026-11-09"},"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"deepseekv32reasoner","providerId":"deepseek","slug":"deepseek-deepseek-v3-2-reasoner","name":"DeepSeek V3.2 Reasoner","provider":"DeepSeek","description":"DeepSeek V3.2 Reasoner is a model from DeepSeek. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":45.481382196493165,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"coding":1},"benchmarks":[{"benchmarkId":"swe-bench:swe-bench-verified:overall","benchmarkName":"SWE-bench Verified","category":"coding","domain":null,"eligible":true,"ranked":true,"score":60,"unit":"percent","family":"swe-bench-verified","index":45.481382196493165,"aboveFit":false,"run":"mini-SWE-agent","source":"SWE-bench team","sourceUrl":"https://www.swebench.com/","measuredAt":"2026-02-26","stderr":null,"costPerTest":0.0280763028,"latency":null},{"benchmarkId":"swebench-experiments:swe-bench-verified:overall","benchmarkName":"SWE-bench Verified","category":"coding","domain":null,"eligible":true,"ranked":true,"score":60,"unit":"percent","family":"swe-bench-verified","index":45.481382196493165,"aboveFit":false,"run":"mini-SWE-agent","source":"SWE-bench team","sourceUrl":"https://github.com/SWE-bench/experiments/tree/main/evaluation/verified","measuredAt":"2026-09-01","stderr":null,"costPerTest":0.0280763028,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":20,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:swe-bench-team","name":"SWE-bench team","url":"https://www.swebench.com","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"deepseekv4flash","providerId":"deepseek","slug":"deepseek-deepseek-v4-flash-0423","name":"DeepSeek V4 Flash 0423","provider":"DeepSeek","description":"DeepSeek V4 Flash 0423 is a model from DeepSeek. It has 6 scored results in 6 categories. It has no index.","logoUrl":null,"releaseDate":"2026-04-24T03:17:46.000Z","contextWindowTokens":1048576,"modalities":["text"],"openWeights":true,"details":{"apiId":"deepseek/deepseek-v4-flash","maxOutputTokens":393216,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.018,"weightsUrl":"https://huggingface.co/deepseek-ai/DeepSeek-V4-Flash"},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":47.96574332861434,"coding":52.187277994391444,"reasoning":51.39637392411663,"multimodal":null,"knowledge":53.85563513810537,"multilingual":null,"instruction":54.34137726705377,"math":50.781081681240266},"capabilityCounts":{"math":1,"reasoning":1,"knowledge":1,"coding":1,"instruction":1,"agentic":1},"benchmarks":[{"benchmarkId":"livebench:livebench-math:overall","benchmarkName":"LiveBench Mathematics","category":"math","domain":null,"eligible":true,"ranked":true,"score":79.64750000000001,"unit":"percent","family":"livebench:livebench-math:overall","index":50.781081681240266,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Mathematics","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.010449340369393139,"latency":null},{"benchmarkId":"livebench:livebench-reasoning:overall","benchmarkName":"LiveBench Reasoning","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":70.58175,"unit":"percent","family":"livebench","index":53.05371979293662,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Reasoning","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.013579207920792079,"latency":null},{"benchmarkId":"livebench:livebench-language:overall","benchmarkName":"LiveBench Language","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":70.12400000000001,"unit":"percent","family":"livebench:livebench-language:overall","index":53.85563513810537,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Language","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.005005333333333334,"latency":null},{"benchmarkId":"livebench:livebench-coding:overall","benchmarkName":"LiveBench Coding","category":"coding","domain":null,"eligible":true,"ranked":true,"score":69.22800000000001,"unit":"percent","family":"livebench:livebench-coding:overall","index":52.187277994391444,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Coding","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.0012700854700854701,"latency":null},{"benchmarkId":"livebench:livebench-data-analysis:overall","benchmarkName":"LiveBench Data Analysis","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":68.02300000000001,"unit":"percent","family":"livebench","index":49.73902805529663,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Data%20Analysis","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.013483333333333333,"latency":null},{"benchmarkId":"livebench:livebench-instruction-following:overall","benchmarkName":"LiveBench Instruction Following","category":"instruction","domain":null,"eligible":true,"ranked":true,"score":63.137499999999996,"unit":"percent","family":"livebench:livebench-instruction-following:overall","index":54.34137726705377,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=IF","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.007332000000000001,"latency":null},{"benchmarkId":"livebench:livebench-agentic-coding:overall","benchmarkName":"LiveBench Agentic Coding","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":37.626,"unit":"percent","family":"livebench:livebench-agentic-coding:overall","index":47.96574332861434,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Agentic%20Coding","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.03175,"latency":null}],"price":{"inputPerMillion":0.09,"outputPerMillion":0.18,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":65,"timeToFirstToken":0.76,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":{"elo":1432.0715129421696,"eloUpper":1436.1016618797694,"eloLower":1428.04136400457,"votes":51368,"rank":93,"rankUpper":null,"rankLower":null,"arenaName":"DeepSeek-V4-Flash","organization":"deepseek","updatedAt":"2026-11-09"},"quality":{"coveragePercent":85,"benchmarkCount":6,"categoryCount":6,"confidence":"unranked","state":"partial"},"hosts":[{"host":"deepinfra","name":"Deep Infra","kind":"host","url":"https://deepinfra.com/models","inputPerMillion":0.09,"outputPerMillion":0.18,"cacheReadPerMillion":0.018,"contextTokens":1048576,"maxOutputTokens":16384,"deprecated":false},{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":0.09,"outputPerMillion":0.18,"cacheReadPerMillion":0.018,"contextTokens":1048576,"maxOutputTokens":393216,"deprecated":false},{"host":"gmi-cloud","name":"GMI Cloud","kind":"host","url":"https://docs.gmicloud.ai/inference-engine/api-reference/llm-api-reference","inputPerMillion":0.112,"outputPerMillion":0.224,"cacheReadPerMillion":0.022,"contextTokens":1048575,"maxOutputTokens":384000,"deprecated":false},{"host":"siliconflow","name":"SiliconFlow","kind":"host","url":"https://cloud.siliconflow.com/models","inputPerMillion":0.13,"outputPerMillion":0.28,"cacheReadPerMillion":0.028,"contextTokens":1000000,"maxOutputTokens":384000,"deprecated":false},{"host":"digitalocean","name":"DigitalOcean","kind":"cloud","url":"https://docs.digitalocean.com/products/gradient-ai-platform/details/models/","inputPerMillion":0.14,"outputPerMillion":0.28,"cacheReadPerMillion":0.028,"contextTokens":1048576,"maxOutputTokens":384000,"deprecated":false},{"host":"azure","name":"Microsoft Azure","kind":"cloud","url":"https://learn.microsoft.com/en-us/azure/ai-services/openai/concepts/models","inputPerMillion":0.19,"outputPerMillion":0.51,"cacheReadPerMillion":null,"contextTokens":1000000,"maxOutputTokens":384000,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:livebench","name":"LiveBench","url":"https://livebench.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"deepseekv4flashlatest","providerId":"deepseek","slug":"deepseek-deepseek-v4-flash-latest","name":"DeepSeek V4 Flash Latest","provider":"DeepSeek","description":"DeepSeek V4 Flash Latest is a model from DeepSeek. It has no scored result.","logoUrl":null,"releaseDate":"2026-08-01T17:40:09.000Z","contextWindowTokens":1048576,"modalities":["text"],"openWeights":null,"details":{"apiId":"~deepseek/deepseek-v4-flash-latest","maxOutputTokens":943718,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.035,"weightsUrl":null},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":0.0434,"outputPerMillion":0.3,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":61,"timeToFirstToken":0.5255,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"deepseekv4prohigh0813","providerId":"deepseek","slug":"deepseek-deepseek-v4-pro-high-0813","name":"DeepSeek V4 Pro (High) (0813)","provider":"DeepSeek","description":"DeepSeek V4 Pro (High) (0813) is a model from DeepSeek. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":70.14559221688286,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"agentic":1},"benchmarks":[{"benchmarkId":"lmarena:agent-bash-recovery:overall","benchmarkName":"Agent Arena command recovery","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":4.396717307829645,"unit":"index","family":"agent-arena","index":70.14559221688286,"aboveFit":false,"run":null,"source":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset","measuredAt":"2026-10-08","stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":20,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:lmarena","name":"LMArena","url":"https://huggingface.co","license":"CC BY 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"deepseekr1may2025","providerId":"deepseek","slug":"deepseek-deepseek-r1-may-2025","name":"DeepSeek-R1 (May 2025)","provider":"DeepSeek","description":"DeepSeek-R1 (May 2025) is a model from DeepSeek. It has 3 scored results in 2 categories. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":47.21299997888615,"multilingual":null,"instruction":null,"math":46.742281504866305},"capabilityCounts":{"math":2,"knowledge":1},"benchmarks":[{"benchmarkId":"epoch-ai:epoch-math-level-5:overall","benchmarkName":"MATH level 5","category":"math","domain":null,"eligible":true,"ranked":true,"score":96.63897280966768,"unit":"percent","family":"epoch-ai:epoch-math-level-5:overall","index":45.94081187477688,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0.42156161735636516,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":76.32575757575758,"unit":"percent","family":"gpqa-diamond","index":47.21299997888615,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.4023818843083005,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":66.38888888888889,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":47.54375113495573,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":4.911177955575991,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-public:overall","benchmarkName":"FrontierMath-Tier-4-2025-07-01-Public","category":"math","domain":null,"eligible":false,"ranked":true,"score":0,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-public:overall","index":null,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":25,"benchmarkCount":3,"categoryCount":2,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"deepseek-v2-5","providerId":"deepseek","slug":"deepseek-v2-5","name":"DeepSeek-V2.5","provider":"DeepSeek","description":"DeepSeek-V2.5 is a standard model from DeepSeek. It has 2 scored results in 2 categories. It has no index.","logoUrl":null,"releaseDate":"2024-09-05","contextWindowTokens":128000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":29.501788748619948,"reasoning":null,"multimodal":null,"knowledge":16.005504886956437,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"coding":1,"knowledge":1},"benchmarks":[{"benchmarkId":"humaneval","benchmarkName":"Evaluating Large Language Models Trained on Code","category":"coding","score":89,"unit":"percent","family":"humaneval","index":29.501788748619948,"aboveFit":false,"run":null,"source":"Mark Chen, Jerry Tworek, Heewoo Jun, Qiming Yuan, Henrique Ponde de Oliveira Pinto, Jared Kaplan, Harri Edwards, Yuri Burda, Nicholas Joseph, Greg Brockman, Alex Ray, Raul Puri, Gretchen Krueger, Michael Petrov, Heidy Khlaaf, Girish Sastry, Pamela Mishkin, Brooke Chan, Scott Gray, Nick Ryder, Mikhail Pavlov, Alethea Power, Lukasz Kaiser, Mohammad Bavarian, Clemens Winter, Philippe Tillet, Felipe Petroski Such, Dave Cummings, Matthias Plappert, Fotios Chantzis, Elizabeth Barnes, Ariel Herbert-Voss, William Hebgen Guss, Alex Nichol, Alex Paino, Nikolas Tezak, Jie Tang, Igor Babuschkin, Suchir Balaji, Shantanu Jain, William Saunders, Christopher Hesse, Andrew N. Carr, Jan Leike, Josh Achiam, Vedant Misra, Evan Morikawa, Alec Radford, Matthew Knight, Miles Brundage, Mira Murati, Katie Mayer, Peter Welinder, Bob McGrew, Dario Amodei, Sam McCandlish, Ilya Sutskever, Wojciech Zaremba","sourceUrl":"https://arxiv.org/abs/2107.03374","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":42.3,"unit":"percent","family":"gpqa-diamond","index":16.005504886956437,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":{"elo":1271.2597507977812,"eloUpper":1275.9393261047417,"eloLower":1266.5801754908207,"votes":24572,"rank":285,"rankUpper":null,"rankLower":null,"arenaName":"DeepSeek-V2.5","organization":"deepseek","updatedAt":"2026-11-09"},"quality":{"coveragePercent":35,"benchmarkCount":2,"categoryCount":2,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"deepseekv3mar2025","providerId":"deepseek","slug":"deepseek-deepseek-v3-mar-2025","name":"DeepSeek-V3 (Mar 2025)","provider":"DeepSeek","description":"DeepSeek-V3 (Mar 2025) is a model from DeepSeek. It has 3 scored results in 2 categories. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":39.22248046253954,"multilingual":null,"instruction":null,"math":33.296766464712306},"capabilityCounts":{"math":2,"knowledge":1},"benchmarks":[{"benchmarkId":"epoch-ai:epoch-math-level-5:overall","benchmarkName":"MATH level 5","category":"math","domain":null,"eligible":true,"ranked":true,"score":75.547583081571,"unit":"percent","family":"epoch-ai:epoch-math-level-5:overall","index":35.69555588072106,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0.8603173106577204,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":67.61363636363636,"unit":"percent","family":"gpqa-diamond","index":39.22248046253954,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.6836302525030096,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":37.77777777777778,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":30.897977048703545,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":6.237220380071937,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-2025-02-28-public:overall","benchmarkName":"FrontierMath-2025-02-28-Public","category":"math","domain":null,"eligible":false,"ranked":true,"score":0,"unit":"percent","family":"epoch-ai:epoch-frontiermath-2025-02-28-public:overall","index":null,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":25,"benchmarkCount":3,"categoryCount":2,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"deepseekmath-v2","providerId":"deepseek","slug":"deepseekmath-v2","name":"DeepSeekMath V2","provider":"DeepSeek","description":"DeepSeekMath V2 is a reasoning model from DeepSeek in the DeepSeekMath family. It has no scored result.","logoUrl":null,"releaseDate":"2025-11-27","contextWindowTokens":128000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":true,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"deepswepreview","providerId":"unknown","slug":"unknown-deepswe-preview","name":"DeepSWE Preview","provider":"Unknown","description":"DeepSWE Preview is a model. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":38.47458520754809,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"coding":1},"benchmarks":[{"benchmarkId":"swebench-experiments:swe-bench-verified:overall","benchmarkName":"SWE-bench Verified","category":"coding","domain":null,"eligible":true,"ranked":true,"score":50.5,"unit":"percent","family":"swe-bench-verified","index":38.47458520754809,"aboveFit":false,"run":"DeepSWE-Preview","source":"SWE-bench team","sourceUrl":"https://github.com/SWE-bench/experiments/tree/main/evaluation/verified","measuredAt":"2026-09-01","stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":20,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:swe-bench-team","name":"SWE-bench team","url":"https://github.com","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"dessa","providerId":"block","slug":"dessa","name":"Dessa","provider":"Block","description":"Dessa is a standard model from Block. It has no scored result.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"devstral2","providerId":"mistral","slug":"mistral-devstral-2-2512","name":"Devstral 2 2512","provider":"Mistral","description":"Devstral 2 2512 is a model from Mistral. It has 4 scored results in 2 categories. It has no index.","logoUrl":null,"releaseDate":"2025-12-09T13:03:39.000Z","contextWindowTokens":262144,"modalities":["text","file"],"openWeights":true,"details":{"apiId":"mistralai/devstral-2512","maxOutputTokens":209715,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.04,"weightsUrl":"https://huggingface.co/mistralai/Devstral-2-123B-Instruct-2512"},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":49.87903655580675,"coding":37.912888796330336,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"coding":3,"agentic":1},"benchmarks":[{"benchmarkId":"vals-ai:swebench:overall","benchmarkName":"SWE-bench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":62.8,"unit":"percent","family":"swe-bench-verified","index":47.5465434142875,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/swebench","measuredAt":"2026-09-01","stderr":2.164,"costPerTest":0.747244,"latency":243.516},{"benchmarkId":"swe-bench:swe-bench-verified:overall","benchmarkName":"SWE-bench Verified","category":"coding","domain":null,"eligible":true,"ranked":true,"score":53.8,"unit":"percent","family":"swe-bench-verified","index":40.908525214234274,"aboveFit":false,"run":"mini-SWE-agent","source":"SWE-bench team","sourceUrl":"https://www.swebench.com/","measuredAt":"2026-02-26","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"swebench-experiments:swe-bench-verified:overall","benchmarkName":"SWE-bench Verified","category":"coding","domain":null,"eligible":true,"ranked":true,"score":53.8,"unit":"percent","family":"swe-bench-verified","index":40.908525214234274,"aboveFit":false,"run":"mini-SWE-agent","source":"SWE-bench team","sourceUrl":"https://github.com/SWE-bench/experiments/tree/main/evaluation/verified","measuredAt":"2026-09-01","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:lcb:overall","benchmarkName":"LiveCodeBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":51.841,"unit":"percent","family":"livecodebench","index":32.35838155622977,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","measuredAt":"2026-09-01","stderr":1.174,"costPerTest":0.056631,"latency":87.57},{"benchmarkId":"vals-ai:terminal-bench:overall","benchmarkName":"Terminal-Bench 1.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":43.75,"unit":"percent","family":"vals-ai:terminal-bench:overall","index":49.87903655580675,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench","measuredAt":"2026-01-12","stderr":5.546,"costPerTest":0.424819,"latency":488.55},{"benchmarkId":"vals-ai:ioi-v1:overall","benchmarkName":"IOI v1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":0.666,"unit":"percent","family":"vals-ai:ioi-v1:overall","index":40.471759618526946,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/ioi-v1","measuredAt":"2026-08-09","stderr":0.608,"costPerTest":1.753753,"latency":8044.645}],"price":{"inputPerMillion":0.39999999999999997,"outputPerMillion":2,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":62,"timeToFirstToken":3.027,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":40,"benchmarkCount":4,"categoryCount":2,"confidence":"unranked","state":"partial"},"hosts":[{"host":"amazon-bedrock","name":"Amazon Bedrock","kind":"cloud","url":"https://docs.aws.amazon.com/bedrock/latest/userguide/models-supported.html","inputPerMillion":0.4,"outputPerMillion":2,"cacheReadPerMillion":null,"contextTokens":262144,"maxOutputTokens":8192,"deprecated":true},{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":0.4,"outputPerMillion":2,"cacheReadPerMillion":0.04,"contextTokens":262144,"maxOutputTokens":209715,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:swe-bench-team","name":"SWE-bench team","url":"https://www.swebench.com","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"devstralsmall2512","providerId":"mistral","slug":"mistral-devstral-small-2512","name":"Devstral Small (2512)","provider":"Mistral","description":"Devstral Small (2512) is a model from Mistral. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":42.82617491647187,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"coding":1},"benchmarks":[{"benchmarkId":"swe-bench:swe-bench-verified:overall","benchmarkName":"SWE-bench Verified","category":"coding","domain":null,"eligible":true,"ranked":true,"score":56.4,"unit":"percent","family":"swe-bench-verified","index":42.82617491647187,"aboveFit":false,"run":"mini-SWE-agent","source":"SWE-bench team","sourceUrl":"https://www.swebench.com/","measuredAt":"2026-02-26","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"swebench-experiments:swe-bench-verified:overall","benchmarkName":"SWE-bench Verified","category":"coding","domain":null,"eligible":true,"ranked":true,"score":56.4,"unit":"percent","family":"swe-bench-verified","index":42.82617491647187,"aboveFit":false,"run":"mini-SWE-agent","source":"SWE-bench team","sourceUrl":"https://github.com/SWE-bench/experiments/tree/main/evaluation/verified","measuredAt":"2026-09-01","stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":20,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:swe-bench-team","name":"SWE-bench team","url":"https://www.swebench.com","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"devstralsmall2505","providerId":"mistral","slug":"mistral-devstral-small-2505","name":"DevStral Small 2505","provider":"Mistral","description":"DevStral Small 2505 is a model from Mistral. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":35.74562216974843,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"coding":1},"benchmarks":[{"benchmarkId":"swebench-experiments:swe-bench-verified:overall","benchmarkName":"SWE-bench Verified","category":"coding","domain":null,"eligible":true,"ranked":true,"score":46.800000000000004,"unit":"percent","family":"swe-bench-verified","index":35.74562216974843,"aboveFit":false,"run":"OpenHands","source":"SWE-bench team","sourceUrl":"https://github.com/SWE-bench/experiments/tree/main/evaluation/verified","measuredAt":"2026-09-01","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"swe-bench:swe-bench-verified:overall","benchmarkName":"SWE-bench Verified","category":"coding","domain":null,"eligible":true,"ranked":true,"score":46.8,"unit":"percent","family":"swe-bench-verified","index":35.74562216974843,"aboveFit":false,"run":"OpenHands","source":"SWE-bench team","sourceUrl":"https://www.swebench.com/","measuredAt":"2026-02-26","stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":20,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:swe-bench-team","name":"SWE-bench team","url":"https://github.com","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"devstralsmall2507","providerId":"mistral","slug":"mistral-devstral-small-2507","name":"DevStral Small 2507","provider":"Mistral","description":"DevStral Small 2507 is a model from Mistral. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":29.25511548525194,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"coding":1},"benchmarks":[{"benchmarkId":"swe-bench:swe-bench-verified:overall","benchmarkName":"SWE-bench Verified","category":"coding","domain":null,"eligible":true,"ranked":true,"score":38,"unit":"percent","family":"swe-bench-verified","index":29.25511548525194,"aboveFit":false,"run":"SWE-agent","source":"SWE-bench team","sourceUrl":"https://www.swebench.com/","measuredAt":"2026-02-26","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"swebench-experiments:swe-bench-verified:overall","benchmarkName":"SWE-bench Verified","category":"coding","domain":null,"eligible":true,"ranked":true,"score":38,"unit":"percent","family":"swe-bench-verified","index":29.25511548525194,"aboveFit":false,"run":"SWE-agent","source":"SWE-bench team","sourceUrl":"https://github.com/SWE-bench/experiments/tree/main/evaluation/verified","measuredAt":"2026-09-01","stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":20,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:swe-bench-team","name":"SWE-bench team","url":"https://www.swebench.com","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"dfmmimir","providerId":"unknown","slug":"unknown-dfm-mimir","name":"DFM Mimir","provider":"Unknown","description":"DFM Mimir is a model. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":19.352726049856567,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-portuguese:overall","benchmarkName":"EuroEval Portuguese","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":40.709999999999994,"unit":"percent","family":"euroeval","index":27.45433835437481,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/portuguese","measuredAt":null,"stderr":0.5465173830721215,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-french:overall","benchmarkName":"EuroEval French","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":40.245000000000005,"unit":"percent","family":"euroeval","index":26.82344034782662,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/french","measuredAt":null,"stderr":0.44599432451545845,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-dutch:overall","benchmarkName":"EuroEval Dutch","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":36.52333333333333,"unit":"percent","family":"euroeval","index":21.773995012263356,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/dutch","measuredAt":null,"stderr":0.46979244169271706,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":35.838750000000005,"unit":"percent","family":"euroeval","index":20.845172947067383,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":0.7306653645821731,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-italian:overall","benchmarkName":"EuroEval Italian","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":33.63875,"unit":"percent","family":"euroeval","index":17.860279152645745,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/italian","measuredAt":null,"stderr":0.6133922073192648,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":29.323749999999997,"unit":"percent","family":"euroeval","index":12.005817005859683,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.6518770973120623,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-spanish:overall","benchmarkName":"EuroEval Spanish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":27.852500000000003,"unit":"percent","family":"euroeval","index":10.009669280840221,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/spanish","measuredAt":null,"stderr":0.6347674278820551,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":25.39428571428571,"unit":"percent","family":"euroeval","index":6.674438114118779,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.6071546217347584,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"dfmmimirv15","providerId":"unknown","slug":"unknown-dfm-mimir-v1-5","name":"DFM Mimir V1.5","provider":"Unknown","description":"DFM Mimir V1.5 is a model. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":21.782192163782117,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-french:overall","benchmarkName":"EuroEval French","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":45.118750000000006,"unit":"percent","family":"euroeval","index":33.435997679900446,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/french","measuredAt":null,"stderr":0.44867339178961796,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-portuguese:overall","benchmarkName":"EuroEval Portuguese","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":42.40875,"unit":"percent","family":"euroeval","index":29.759151233135608,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/portuguese","measuredAt":null,"stderr":0.4979018477571659,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":39.816250000000004,"unit":"percent","family":"euroeval","index":26.24172525039104,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":0.6307324611116825,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-italian:overall","benchmarkName":"EuroEval Italian","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":36.647499999999994,"unit":"percent","family":"euroeval","index":21.942460608993958,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/italian","measuredAt":null,"stderr":0.575296119402869,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-spanish:overall","benchmarkName":"EuroEval Spanish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":36.411249999999995,"unit":"percent","family":"euroeval","index":21.621923718570276,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/spanish","measuredAt":null,"stderr":0.5303079883426234,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-dutch:overall","benchmarkName":"EuroEval Dutch","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":35.91777777777777,"unit":"percent","family":"euroeval","index":20.9523954577382,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/dutch","measuredAt":null,"stderr":0.5318521097698745,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":35.2125,"unit":"percent","family":"euroeval","index":19.995495793087134,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.535752974793421,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":28.204285714285714,"unit":"percent","family":"euroeval","index":10.486961551539132,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.6691374283054145,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"decision-index-0-3-dinah-0","providerId":"lukitaduarte","slug":"decision-index-0-3-dinah-0","name":"Dinah-0","provider":"Lukitaduarte","description":"Dinah-0 is a model from Lukitaduarte. It has no scored result.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"distylbuttonagent","providerId":"unknown","slug":"unknown-distyl-buttonagent","name":"Distyl ButtonAgent","provider":"Unknown","description":"Distyl ButtonAgent is a model. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":23.486285506173758,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"agentic":1},"benchmarks":[{"benchmarkId":"tau2-bench:banking-knowledge:overall","benchmarkName":"τ²-bench Banking","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":31.19,"unit":"percent","family":"tau2-bench","index":23.486285506173758,"aboveFit":false,"run":"high effort · Distyl AI","source":"Sierra Research","sourceUrl":"https://github.com/sierra-research/tau2-bench","measuredAt":"2026-10-20","stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":20,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:sierra-research","name":"Sierra Research","url":"https://github.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"diva-llama-3-v0-8b","providerId":"diva-authors","slug":"diva-llama-3-v0-8b","name":"DiVA Llama 3 8B","provider":"DiVA authors","description":"DiVA Llama 3 8B is a standard model from DiVA authors. It has no scored result.","logoUrl":null,"releaseDate":"2024-10-03","contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"djev","providerId":"maisa","slug":"djev","name":"djev","provider":"Maisa","description":"djev is a standard model from Maisa. It has 2 scored results in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":90.70407295036175,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"reasoning":2},"benchmarks":[{"benchmarkId":"jevBench15","benchmarkName":"jevBench15","category":"reasoning","score":64.15569234453473,"unit":"percent","family":"jevBench15","index":90.32475059067613,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"jevBench14","benchmarkName":"jevBench14","category":"reasoning","score":52.228492110723586,"unit":"percent","family":"jevBench14","index":91.08339531004737,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":15,"benchmarkCount":2,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"djev-thinking","providerId":"maisa","slug":"djev-thinking","name":"djev (thinking)","provider":"Maisa","description":"djev (thinking) is a standard model from Maisa. It has 2 scored results in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":59.48548293881785,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"reasoning":2},"benchmarks":[{"benchmarkId":"jevBench15","benchmarkName":"jevBench15","category":"reasoning","score":17.403898604323025,"unit":"percent","family":"jevBench15","index":62.01322291670615,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"jevBench14","benchmarkName":"jevBench14","category":"reasoning","score":15.201198018956955,"unit":"percent","family":"jevBench14","index":56.95774296092955,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":15,"benchmarkCount":2,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"dna-1-0","providerId":"community","slug":"dna-1-0","name":"DNA 1.0 8B","provider":"Community","description":"DNA 1.0 8B is a standard model from Community in the DNA 1.0 family. It has no scored result.","logoUrl":null,"releaseDate":null,"contextWindowTokens":32000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"domynsmallv10","providerId":"unknown","slug":"unknown-domyn-small-v1-0","name":"Domyn Small V1.0","provider":"Unknown","description":"Domyn Small V1.0 is a model. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":30.5642488044719,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-italian:overall","benchmarkName":"EuroEval Italian","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":48.06125,"unit":"percent","family":"euroeval","index":37.42829312993937,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/italian","measuredAt":null,"stderr":0.5808614292583043,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-portuguese:overall","benchmarkName":"EuroEval Portuguese","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":46.451249999999995,"unit":"percent","family":"euroeval","index":35.24389358038535,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/portuguese","measuredAt":null,"stderr":0.6547435089407149,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":45.56875000000001,"unit":"percent","family":"euroeval","index":34.04654413785033,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":0.9370090381100922,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-dutch:overall","benchmarkName":"EuroEval Dutch","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":44.3,"unit":"percent","family":"euroeval","index":32.32514231890829,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/dutch","measuredAt":null,"stderr":0.6119882593887155,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":41.70428571428572,"unit":"percent","family":"euroeval","index":28.80335529003551,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.5268504802801537,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":40.6825,"unit":"percent","family":"euroeval","index":27.417027181944544,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.8031315661210185,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-spanish:overall","benchmarkName":"EuroEval Spanish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":36.432500000000005,"unit":"percent","family":"euroeval","index":21.650755079084586,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/spanish","measuredAt":null,"stderr":0.7434505699775877,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-french:overall","benchmarkName":"EuroEval French","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":35.9125,"unit":"percent","family":"euroeval","index":20.94523472767583,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/french","measuredAt":null,"stderr":1.2061100955136723,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"doubaoseed16","providerId":"bytedance","slug":"bytedance-doubao-seed-1-6","name":"Doubao Seed 1.6","provider":"ByteDance","description":"Doubao Seed 1.6 is a model from ByteDance. It has 4 scored results in 3 categories. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":37.779751700889406,"multimodal":null,"knowledge":43.39291731777432,"multilingual":null,"instruction":null,"math":49.3214551635846},"capabilityCounts":{"knowledge":2,"math":1,"reasoning":1},"benchmarks":[{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":77.58838383838383,"unit":"percent","family":"gpqa-diamond","index":48.37104628560305,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.5273066621077898,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":69.44444444444444,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":49.3214551635846,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":6.123151591268345,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-simpleqa-verified:overall","benchmarkName":"SimpleQA Verified","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":17.4,"unit":"percent","family":"epoch-ai:epoch-simpleqa-verified:overall","index":38.414788349945596,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.199449323097345,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-chess-puzzles:overall","benchmarkName":"Chess Puzzles","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":5,"unit":"percent","family":"epoch-ai:epoch-chess-puzzles:overall","index":37.779751700889406,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.1904291355759034,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":40,"benchmarkCount":4,"categoryCount":3,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"doubaoseed20liteapr2026","providerId":"bytedance","slug":"bytedance-doubao-seed-2-0-lite-apr-2026","name":"Doubao Seed 2.0 Lite (Apr 2026)","provider":"ByteDance","description":"Doubao Seed 2.0 Lite (Apr 2026) is a model from ByteDance. It has 4 scored results in 3 categories. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":49.164979628140856,"multimodal":null,"knowledge":54.10103301049665,"multilingual":null,"instruction":null,"math":61.9269928211348},"capabilityCounts":{"math":1,"knowledge":2,"reasoning":1},"benchmarks":[{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":91.11111111111111,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":61.9269928211348,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":3.6111111111111116,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":88.13131313131312,"unit":"percent","family":"gpqa-diamond","index":58.04073294668916,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.944079830983592,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-simpleqa-verified:overall","benchmarkName":"SimpleQA Verified","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":31.5,"unit":"percent","family":"epoch-ai:epoch-simpleqa-verified:overall","index":50.16133307430414,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.4696631960792619,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-chess-puzzles:overall","benchmarkName":"Chess Puzzles","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":20,"unit":"percent","family":"epoch-ai:epoch-chess-puzzles:overall","index":49.164979628140856,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":4.020151261036848,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":40,"benchmarkCount":4,"categoryCount":3,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"doubaoseed21pro","providerId":"bytedance","slug":"bytedance-doubao-seed-2-1-pro","name":"Doubao Seed 2.1 Pro","provider":"ByteDance","description":"Doubao Seed 2.1 Pro is a model from ByteDance. It has 4 scored results in 3 categories. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":49.923994823290954,"multimodal":null,"knowledge":60.14948764312097,"multilingual":null,"instruction":null,"math":64.51274413550408},"capabilityCounts":{"math":1,"knowledge":2,"reasoning":1},"benchmarks":[{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":95.55555555555556,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":64.51274413550408,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.9235728721796908,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":89.33080808080808,"unit":"percent","family":"gpqa-diamond","index":59.14087693807023,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.8417155503921614,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-simpleqa-verified:overall","benchmarkName":"SimpleQA Verified","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":44.7,"unit":"percent","family":"epoch-ai:epoch-simpleqa-verified:overall","index":61.158098348171706,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.5730176046009101,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-chess-puzzles:overall","benchmarkName":"Chess Puzzles","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":21,"unit":"percent","family":"epoch-ai:epoch-chess-puzzles:overall","index":49.923994823290954,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":4.093601807403323,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":40,"benchmarkCount":4,"categoryCount":3,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"doubaoseed21turbo","providerId":"bytedance","slug":"bytedance-doubao-seed-2-1-turbo","name":"Doubao Seed 2.1 Turbo","provider":"ByteDance","description":"Doubao Seed 2.1 Turbo is a model from ByteDance. It has 4 scored results in 3 categories. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":53.71907079904143,"multimodal":null,"knowledge":54.01713018636791,"multilingual":null,"instruction":null,"math":64.83596304980023},"capabilityCounts":{"math":1,"knowledge":2,"reasoning":1},"benchmarks":[{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":96.11111111111111,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":64.83596304980023,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.4753271842145104,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":89.58333333333334,"unit":"percent","family":"gpqa-diamond","index":59.372486199413615,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.7895899774362864,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-simpleqa-verified:overall","benchmarkName":"SimpleQA Verified","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":29.7,"unit":"percent","family":"epoch-ai:epoch-simpleqa-verified:overall","index":48.66177417332219,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.445683229480096,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-chess-puzzles:overall","benchmarkName":"Chess Puzzles","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":26,"unit":"percent","family":"epoch-ai:epoch-chess-puzzles:overall","index":53.71907079904143,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":4.4084400227680804,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":40,"benchmarkCount":4,"categoryCount":3,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"doubaoseedevolvingnov2026","providerId":"bytedance","slug":"bytedance-doubao-seed-evolving-nov-2026","name":"Doubao Seed Evolving (Nov 2026)","provider":"ByteDance","description":"Doubao Seed Evolving (Nov 2026) is a model from ByteDance. It has 4 scored results in 3 categories. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":64.34528353114278,"multimodal":null,"knowledge":60.720233513543704,"multilingual":null,"instruction":null,"math":66.6136670784291},"capabilityCounts":{"math":1,"knowledge":2,"reasoning":1},"benchmarks":[{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":99.16666666666667,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":66.6136670784291,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0.4700633956814717,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":93.11868686868688,"unit":"percent","family":"gpqa-diamond","index":62.615015858220936,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.5670135540928183,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-simpleqa-verified:overall","benchmarkName":"SimpleQA Verified","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":41.9,"unit":"percent","family":"epoch-ai:epoch-simpleqa-verified:overall","index":58.825451168866465,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.5610338967577952,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-chess-puzzles:overall","benchmarkName":"Chess Puzzles","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":40,"unit":"percent","family":"epoch-ai:epoch-chess-puzzles:overall","index":64.34528353114278,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":4.92365963917331,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":40,"benchmarkCount":4,"categoryCount":3,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"doubaoseed20pro","providerId":"bytedance","slug":"bytedance-doubao-seed-2-0-pro","name":"Doubao-Seed-2.0 Pro","provider":"ByteDance","description":"Doubao-Seed-2.0 Pro is a model from ByteDance. It has 4 scored results in 3 categories. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":49.164979628140856,"multimodal":null,"knowledge":57.420682867541615,"multilingual":null,"instruction":null,"math":63.54308739261559},"capabilityCounts":{"math":1,"knowledge":2,"reasoning":1},"benchmarks":[{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":93.88888888888889,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":63.54308739261559,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.7415567763225743,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":88.19444444444444,"unit":"percent","family":"gpqa-diamond","index":58.09863526202501,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.9429399729068986,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-simpleqa-verified:overall","benchmarkName":"SimpleQA Verified","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":39.4,"unit":"percent","family":"epoch-ai:epoch-simpleqa-verified:overall","index":56.74273047305821,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.5459721957493382,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-chess-puzzles:overall","benchmarkName":"Chess Puzzles","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":20,"unit":"percent","family":"epoch-ai:epoch-chess-puzzles:overall","index":49.164979628140856,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":4.020151261036848,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":40,"benchmarkCount":4,"categoryCount":3,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"drex-max","providerId":"nace-ai","slug":"drex-max","name":"Drex Max","provider":"Nace.AI","description":"Drex Max is a model from Nace.AI. It has no scored result.","logoUrl":null,"releaseDate":"2026-10-13","contextWindowTokens":262000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":0.03,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"drex-max-v1-1","providerId":"nace-ai","slug":"drex-max-v1-1","name":"Drex Max v1.1","provider":"Nace.AI","description":"Drex Max v1.1 is a model from Nace.AI in the Drex Max family. It has no scored result.","logoUrl":null,"releaseDate":"2026-10-28","contextWindowTokens":262000,"modalities":["text"],"openWeights":false,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":0.03,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"drex-v1-5","providerId":"nace-ai","slug":"drex-v1-5","name":"Drex v1.5","provider":"Nace.AI","description":"Drex v1.5 is a model from Nace.AI. It has no scored result.","logoUrl":null,"releaseDate":"2026-09-28","contextWindowTokens":131000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":0.05,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"drex-v2-0","providerId":"nace-ai","slug":"drex-v2-0","name":"Drex v2.0","provider":"Nace.AI","description":"Drex v2.0 is a model from Nace.AI. It has no scored result.","logoUrl":null,"releaseDate":"2026-10-06","contextWindowTokens":131000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":0.06,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"eikos-27b","providerId":"caiovicentino1","slug":"eikos-27b","name":"Eikos-27B","provider":"caiovicentino1","description":"Eikos-27B is a standard model from caiovicentino1. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":62.67882105561752,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"reasoning":1},"benchmarks":[{"benchmarkId":"jevBench15","benchmarkName":"jevBench15","category":"reasoning","score":18.503023570753697,"unit":"percent","family":"jevBench15","index":62.67882105561752,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":15,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"elephant-1","providerId":"baseten","slug":"elephant-1","name":"elephant-1","provider":"Baseten","description":"elephant-1 is a model from Baseten. It has no scored result.","logoUrl":null,"releaseDate":"2026-10-21","contextWindowTokens":262000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"embeddinggemma-2","providerId":"google","slug":"embeddinggemma-2","name":"EmbeddingGemma 2","provider":"Google","description":"EmbeddingGemma 2 is a standard model from Google in the EmbeddingGemma family. It has no scored result.","logoUrl":null,"releaseDate":"2026-10-06","contextWindowTokens":8192,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"ember-1","providerId":"fireworks","slug":"ember-1","name":"Ember-1","provider":"Fireworks","description":"Ember-1 is a reasoning model from Fireworks in the Fireworks Ember family. It has 8 scored results in 3 categories. It has no index.","logoUrl":null,"releaseDate":"2026-09-23","contextWindowTokens":1048576,"modalities":["text","image"],"openWeights":false,"details":{"apiId":"fireworks/ember-1","maxOutputTokens":943718,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.3,"weightsUrl":null},"reasoning":true,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":69.73092648333224,"coding":66.92782058826953,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":68.21654735116284},"capabilityCounts":{"coding":4,"agentic":3,"math":1},"benchmarks":[{"benchmarkId":"sweVerified","benchmarkName":"Software Engineering Benchmark Verified","category":"coding","score":92.2,"unit":"percent","family":"swe-bench-verified","index":69.23073620112804,"aboveFit":false,"run":null,"source":"Carlos E. Jimenez, John Yang, Alexander Wettig, Shunyu Yao, Kexin Pei, Ofir Press, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2310.06770","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:sage:overall","benchmarkName":"SAGE","category":null,"domain":"Education","eligible":false,"ranked":true,"score":86.376,"unit":"percent","family":"vals-ai:sage:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/sage","measuredAt":"2026-11-05","stderr":1.889,"costPerTest":0.205336,"latency":53.98},{"benchmarkId":"vals-ai:vibe-code:overall","benchmarkName":"Vibe Code Bench v1.1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":83.869,"unit":"percent","family":"vals-ai:vibe-code:overall","index":72.51077547177684,"aboveFit":false,"run":"OpenHands","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","measuredAt":"2026-11-09","stderr":3.151,"costPerTest":8.559507,"latency":2357.968},{"benchmarkId":"terminalBench21","benchmarkName":"Terminal-Bench 2.1","category":"agentic","score":82,"unit":"percent","family":"terminal-bench-2-1","index":69.09926109413918,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"deepSwe","benchmarkName":"DeepSWE","category":"agentic","score":75.2,"unit":"percent","family":"deepSwe","index":75.74502134921156,"aboveFit":false,"run":null,"source":"Datacurve AI","sourceUrl":"https://deepswe.datacurve.ai/blog","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:cyber:overall","benchmarkName":"CyberBench v1.1","category":null,"domain":"Other","eligible":false,"ranked":true,"score":69.524,"unit":"percent","family":"vals-ai:cyber:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/cyber","measuredAt":"2026-11-05","stderr":5.558,"costPerTest":2.426078,"latency":1322.414},{"benchmarkId":"vals-ai:biomysterybench:overall","benchmarkName":"BioMysteryBench","category":null,"domain":"Science","eligible":false,"ranked":true,"score":67.037,"unit":"percent","family":"vals-ai:biomysterybench:overall","index":null,"aboveFit":false,"run":"Terminus 2","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/biomysterybench","measuredAt":"2026-11-09","stderr":3.032,"costPerTest":0.907066,"latency":566.167},{"benchmarkId":"vals-ai:public-benefits-bench:overall","benchmarkName":"Public Benefits Bench v1.1","category":null,"domain":"Public benefits","eligible":false,"ranked":true,"score":66.171,"unit":"percent","family":"vals-ai:public-benefits-bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/public-benefits-bench","measuredAt":"2026-10-12","stderr":1.231,"costPerTest":1.023263,"latency":801.415},{"benchmarkId":"tau2Airline","benchmarkName":"τ²-Bench Airline Domain","category":"agentic","score":66,"unit":"percent","family":"tau2Airline","index":null,"aboveFit":false,"run":null,"source":"Victor Barres, Honghua Dong, Soham Ray, Xujie Si, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2506.07982","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:emb:overall","benchmarkName":"EMB","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":63.368,"unit":"percent","family":"vals-ai:emb:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/emb","measuredAt":"2026-10-20","stderr":2.949,"costPerTest":2.459539,"latency":1867.313},{"benchmarkId":"vals-ai:proof_bench:overall","benchmarkName":"ProofBench v1.1","category":"math","domain":null,"eligible":true,"ranked":true,"score":61,"unit":"percent","family":"vals-ai:proof_bench:overall","index":68.21654735116284,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/proof_bench","measuredAt":"2026-11-09","stderr":4.902,"costPerTest":1.737739,"latency":1269.194},{"benchmarkId":"vals-ai:fabv2:overall","benchmarkName":"Finance Agent (v2)","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":51.853,"unit":"percent","family":"vals-ai:fabv2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/fabv2","measuredAt":"2026-10-29","stderr":2.124,"costPerTest":1.95445,"latency":798.612},{"benchmarkId":"vals-ai:ioi:overall","benchmarkName":"IOI","category":"coding","domain":null,"eligible":true,"ranked":true,"score":46.333,"unit":"percent","family":"vals-ai:ioi:overall","index":63.51108026949298,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/ioi","measuredAt":"2026-10-29","stderr":9.163,"costPerTest":24.458496,"latency":6401.12},{"benchmarkId":"vals-ai:legal_research:overall","benchmarkName":"Legal Research Bench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":44.712,"unit":"percent","family":"vals-ai:legal_research:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_research","measuredAt":"2026-11-09","stderr":3.456,"costPerTest":3.674091,"latency":2313.552},{"benchmarkId":"vals-ai:code-migration:overall","benchmarkName":"Code Migration","category":"coding","domain":null,"eligible":true,"ranked":true,"score":32.508,"unit":"percent","family":"vals-ai:code-migration:overall","index":62.458690410680255,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/code-migration","measuredAt":"2026-10-29","stderr":4.626,"costPerTest":51.110833,"latency":14916.444},{"benchmarkId":"vals-ai:tax_agent_bench:overall","benchmarkName":"Tax Agent Bench","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":31.077,"unit":"percent","family":"vals-ai:tax_agent_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench","measuredAt":"2026-11-09","stderr":3.339,"costPerTest":2.972904,"latency":1845.939},{"benchmarkId":"vals-ai:mysterymechanism:overall","benchmarkName":"MysteryMechanism","category":null,"domain":"Science","eligible":false,"ranked":true,"score":22.523,"unit":"percent","family":"vals-ai:mysterymechanism:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mysterymechanism","measuredAt":"2026-11-11","stderr":2.81,"costPerTest":1.293505,"latency":1215.329},{"benchmarkId":"vals-ai:terminal-bench-4:overall","benchmarkName":"Terminal-Bench 4.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":20.707,"unit":"percent","family":"terminal-bench-4","index":64.34849700664597,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-4","measuredAt":"2026-11-09","stderr":3.642,"costPerTest":19.484094,"latency":6896.196},{"benchmarkId":"vals-ai:terminal-bench-science:overall","benchmarkName":"Terminal-Bench-Science","category":null,"domain":"Science","eligible":false,"ranked":true,"score":5.714,"unit":"percent","family":"vals-ai:terminal-bench-science:overall","index":null,"aboveFit":false,"run":"Mini-SWE-agent","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-science","measuredAt":"2026-11-09","stderr":2.794,"costPerTest":30.760832,"latency":12196.691}],"price":{"inputPerMillion":3,"outputPerMillion":15,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":62,"timeToFirstToken":0.447,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":50,"benchmarkCount":8,"categoryCount":3,"confidence":"unranked","state":"partial"},"hosts":[{"host":"fireworks-ai","name":"Fireworks AI","kind":"host","url":"https://fireworks.ai/docs/","inputPerMillion":3,"outputPerMillion":15,"cacheReadPerMillion":0.3,"contextTokens":1048576,"maxOutputTokens":131072,"deprecated":false},{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":3,"outputPerMillion":15,"cacheReadPerMillion":0.3,"contextTokens":1048576,"maxOutputTokens":943718,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"ernie45vl424ba47b","providerId":"baidu","slug":"baidu-ernie-4-5-vl-424b-a47b","name":"ERNIE 4.5 VL 424B A47B","provider":"Baidu","description":"ERNIE 4.5 VL 424B A47B is a model from Baidu. It has no scored result.","logoUrl":null,"releaseDate":"2025-06-30T16:28:23.000Z","contextWindowTokens":123000,"modalities":["image","text"],"openWeights":true,"details":{"apiId":"baidu/ernie-4.5-vl-424b-a47b","maxOutputTokens":16000,"knowledgeCutoff":"2025-03-31","retiresAt":"2026-10-08","tools":false,"structuredOutput":false,"cacheReadPerMillion":null,"weightsUrl":"https://huggingface.co/baidu/ERNIE-4.5-VL-424B-A47B-PT"},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":0.42,"outputPerMillion":1.25,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":35.5,"timeToFirstToken":1.702,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"ernie-5-5-flash-preview","providerId":"baidu","slug":"ernie-5-5-flash-preview","name":"ERNIE 5.5 Flash Preview","provider":"Baidu","description":"ERNIE 5.5 Flash Preview is a reasoning model from Baidu in the ERNIE 5.5 Flash family. It has 10 scored results in 3 categories. It has no index.","logoUrl":null,"releaseDate":"2026-11-11","contextWindowTokens":1000000,"modalities":["text"],"openWeights":null,"details":null,"reasoning":true,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":72.74072528602929,"coding":65.47408234928886,"reasoning":null,"multimodal":null,"knowledge":70.93942768269594,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"agentic":7,"coding":2,"knowledge":1},"benchmarks":[{"benchmarkId":"browseComp","benchmarkName":"BrowseComp","category":"agentic","score":87.4,"unit":"percent","family":"browseComp","index":68.9567250949382,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"draco","benchmarkName":"Data Research and Analysis with Complex Operations","category":"agentic","score":84.9,"unit":"percent","family":"draco","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mcpAtlas","benchmarkName":"MCP Atlas","category":"agentic","score":84.6,"unit":"percent","family":"mcpAtlas","index":68.03155073928174,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"toolathlonVerified","benchmarkName":"Toolathlon-Verified","category":"agentic","score":75,"unit":"percent","family":"toolathlonVerified","index":69.4435794246127,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"deepSwe","benchmarkName":"DeepSWE","category":"agentic","score":71.2,"unit":"percent","family":"deepSwe","index":72.83713301323574,"aboveFit":false,"run":null,"source":"Datacurve AI","sourceUrl":"https://deepswe.datacurve.ai/blog","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"officeQaProV2","benchmarkName":"OfficeQA Pro v2","category":"multimodal","score":67.9,"unit":"percent","family":"officeQaProV2","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"nl2Repo","benchmarkName":"NL2Repo","category":"coding","score":64.8,"unit":"percent","family":"nl2Repo","index":72.46931101627645,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"apexAgents","benchmarkName":"APEX-Agents","category":"agentic","score":64.2,"unit":"percent","family":"apexAgents","index":78.08479182127104,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"automationBench","benchmarkName":"AutomationBench","category":"agentic","score":61.9,"unit":"percent","family":"automationBench","index":95,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hleNoTools","benchmarkName":"Humanity's Last Exam without tools","category":"knowledge","score":46.4,"unit":"percent","family":"hle","index":70.93942768269594,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"spreadsheetBench2","benchmarkName":"SpreadsheetBench 2","category":"agentic","score":36.4,"unit":"percent","family":"spreadsheetBench2","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"programBench","benchmarkName":"ProgramBench: Can Language Models Rebuild Programs From Scratch?","category":"coding","score":29.5,"unit":"percent","family":"programBench","index":58.47885368230129,"aboveFit":false,"run":null,"source":"John Yang, Kilian Lieret, Jeffrey Ma, Parth Thakkar, Dmitrii Pedchenko, Sten Sootla, Emily McMilin, Pengcheng Yin, Rui Hou, Gabriel Synnaeve, Diyi Yang, Ofir Press","sourceUrl":"https://programbench.com/static/paper.pdf","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"agentsLastExamCli","benchmarkName":"Agents' Last Exam v1.1, CLI subset","category":"agentic","score":29,"unit":"percent","family":"agentsLastExamCli","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"terminalBenchScience","benchmarkName":"Terminal-Bench-Science 0.1","category":"agentic","score":21,"unit":"percent","family":"terminalBenchScience","index":56.831296908865625,"aboveFit":false,"run":null,"source":"Terminal-Bench-Science Team","sourceUrl":"https://www.terminal-bench-science.ai/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":55.00000000000001,"benchmarkCount":10,"categoryCount":3,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"ernielightdt10intl","providerId":"baidu","slug":"baidu-ernie-light-dt-1-0","name":"ERNIE Light DT 1.0","provider":"Baidu","description":"ERNIE Light DT 1.0 is a model from Baidu. It has no scored result.","logoUrl":null,"releaseDate":"2026-11-10T17:21:00.000Z","contextWindowTokens":1024000,"modalities":["text"],"openWeights":null,"details":{"apiId":"baidu/ernie-light-dt-1.0-intl","maxOutputTokens":131072,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.113,"weightsUrl":null},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":1.13,"outputPerMillion":3.9400000000000004,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":65,"timeToFirstToken":2.479,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"eurollm17b","providerId":"unknown","slug":"unknown-eurollm-1-7b","name":"EuroLLM 1.7B","provider":"Unknown","description":"EuroLLM 1.7B is a model. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":5,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":10.507142857142856,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.34327058405513133,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"eurollm17binstruct","providerId":"unknown","slug":"unknown-eurollm-1-7b-instruct","name":"EuroLLM 1.7B Instruct","provider":"Unknown","description":"EuroLLM 1.7B Instruct is a model. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":5,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":21.372500000000002,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":0.8385096525979889,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":18.985000000000003,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.7781127007060096,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":16.667142857142856,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":1.193919116666978,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"eurollm22b2512","providerId":"unknown","slug":"unknown-eurollm-22b-2512","name":"EuroLLM 22B 2512","provider":"Unknown","description":"EuroLLM 22B 2512 is a model. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":31.412714556749876,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-dutch:overall","benchmarkName":"EuroEval Dutch","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":46.74666666666667,"unit":"percent","family":"euroeval","index":35.644706023613566,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/dutch","measuredAt":null,"stderr":0.6338506659037811,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-portuguese:overall","benchmarkName":"EuroEval Portuguese","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":44.91375000000001,"unit":"percent","family":"euroeval","index":33.15785984905662,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/portuguese","measuredAt":null,"stderr":0.6667610610256121,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-italian:overall","benchmarkName":"EuroEval Italian","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":44.0275,"unit":"percent","family":"euroeval","index":31.955422519371986,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/italian","measuredAt":null,"stderr":0.5687692304441231,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-french:overall","benchmarkName":"EuroEval French","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":43.92625,"unit":"percent","family":"euroeval","index":31.81804956633326,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/french","measuredAt":null,"stderr":0.3927009835740165,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":43.32875000000001,"unit":"percent","family":"euroeval","index":31.007379547166487,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":0.5154260980004796,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":38.72749999999999,"unit":"percent","family":"euroeval","index":24.764542014628955,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.5559254671266645,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":34.95142857142857,"unit":"percent","family":"euroeval","index":19.64128193534002,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.5179334919087742,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-spanish:overall","benchmarkName":"EuroEval Spanish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":32.678749999999994,"unit":"percent","family":"euroeval","index":16.55778004235266,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/spanish","measuredAt":null,"stderr":0.3777068969452371,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"eurollm22binstruct2512","providerId":"unknown","slug":"unknown-eurollm-22b-instruct-2512","name":"EuroLLM 22B Instruct 2512","provider":"Unknown","description":"EuroLLM 22B Instruct 2512 is a model. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":37.72763049057313,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-french:overall","benchmarkName":"EuroEval French","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":54.045,"unit":"percent","family":"euroeval","index":45.54686505828957,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/french","measuredAt":null,"stderr":0.5104455774125191,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-dutch:overall","benchmarkName":"EuroEval Dutch","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":51.511111111111106,"unit":"percent","family":"euroeval","index":42.1089608672903,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/dutch","measuredAt":null,"stderr":0.3164424157238341,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-portuguese:overall","benchmarkName":"EuroEval Portuguese","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":48.675,"unit":"percent","family":"euroeval","index":38.26101066008768,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/portuguese","measuredAt":null,"stderr":0.39975578482368457,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-italian:overall","benchmarkName":"EuroEval Italian","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":48.36124999999999,"unit":"percent","family":"euroeval","index":37.83532410190594,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/italian","measuredAt":null,"stderr":0.4046680738580695,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":48.2025,"unit":"percent","family":"euroeval","index":37.619936879240306,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":0.34620035745215516,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":45.287142857142854,"unit":"percent","family":"euroeval","index":33.66446804095072,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.3770914467124301,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":43.985,"unit":"percent","family":"euroeval","index":31.89775979834338,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.5474557630530525,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-spanish:overall","benchmarkName":"EuroEval Spanish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":38.802499999999995,"unit":"percent","family":"euroeval","index":24.866299757620602,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/spanish","measuredAt":null,"stderr":0.3809322446315092,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"eurollm9b","providerId":"unknown","slug":"unknown-eurollm-9b","name":"EuroLLM 9B","provider":"Unknown","description":"EuroLLM 9B is a model. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":22.849315923321903,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-french:overall","benchmarkName":"EuroEval French","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":43.691250000000004,"unit":"percent","family":"euroeval","index":31.49920863829277,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/french","measuredAt":null,"stderr":0.42794567412231194,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-dutch:overall","benchmarkName":"EuroEval Dutch","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":42.96000000000001,"unit":"percent","family":"euroeval","index":30.507070644124227,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/dutch","measuredAt":null,"stderr":0.5099515832742856,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":41.356249999999996,"unit":"percent","family":"euroeval","index":28.331150906486165,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":0.4407929644402233,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-portuguese:overall","benchmarkName":"EuroEval Portuguese","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":38.05749999999999,"unit":"percent","family":"euroeval","index":23.855506177236908,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/portuguese","measuredAt":null,"stderr":0.3672192805395708,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":36.574285714285715,"unit":"percent","family":"euroeval","index":21.843125669406884,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.5804449384141551,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-italian:overall","benchmarkName":"EuroEval Italian","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":36.15375,"unit":"percent","family":"euroeval","index":21.272555467632298,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/italian","measuredAt":null,"stderr":0.41936708263763384,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":35.058749999999996,"unit":"percent","family":"euroeval","index":19.786892419954253,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.575225499260942,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-spanish:overall","benchmarkName":"EuroEval Spanish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":34.82124999999999,"unit":"percent","family":"euroeval","index":19.464659567147365,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/spanish","measuredAt":null,"stderr":0.39318411717667334,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"eurollm9binstruct","providerId":"unknown","slug":"unknown-eurollm-9b-instruct","name":"EuroLLM 9B Instruct","provider":"Unknown","description":"EuroLLM 9B Instruct is a model. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":27.71830278530195,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-dutch:overall","benchmarkName":"EuroEval Dutch","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":44.46444444444444,"unit":"percent","family":"euroeval","index":32.54825559243071,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/dutch","measuredAt":null,"stderr":0.37670271869420446,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-french:overall","benchmarkName":"EuroEval French","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":44.39625,"unit":"percent","family":"euroeval","index":32.455731422414246,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/french","measuredAt":null,"stderr":0.34529652112351206,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":43.54625,"unit":"percent","family":"euroeval","index":31.302477001842252,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":0.32909914919367383,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-portuguese:overall","benchmarkName":"EuroEval Portuguese","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":41.146249999999995,"unit":"percent","family":"euroeval","index":28.046229226109556,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/portuguese","measuredAt":null,"stderr":0.284308371315373,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":40.66285714285714,"unit":"percent","family":"euroeval","index":27.390376344494353,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.3211204872737627,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-italian:overall","benchmarkName":"EuroEval Italian","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":37.71875,"unit":"percent","family":"euroeval","index":23.39590037139132,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/italian","measuredAt":null,"stderr":0.35114099732158877,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-spanish:overall","benchmarkName":"EuroEval Spanish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":35.934999999999995,"unit":"percent","family":"euroeval","index":20.97576205057332,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/spanish","measuredAt":null,"stderr":0.3888544875657217,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":35.06,"unit":"percent","family":"euroeval","index":19.788588382337455,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.5704439060941926,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"eurollm9binstruct2512","providerId":"unknown","slug":"unknown-eurollm-9b-instruct-2512","name":"EuroLLM 9B Instruct 2512","provider":"Unknown","description":"EuroLLM 9B Instruct 2512 is a model. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":31.388123102193553,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-french:overall","benchmarkName":"EuroEval French","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":49.67625,"unit":"percent","family":"euroeval","index":39.61947652902616,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/french","measuredAt":null,"stderr":0.529237364043772,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-dutch:overall","benchmarkName":"EuroEval Dutch","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":47.138888888888886,"unit":"percent","family":"euroeval","index":36.17686133140691,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/dutch","measuredAt":null,"stderr":0.5838547933802433,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-portuguese:overall","benchmarkName":"EuroEval Portuguese","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":46.09125,"unit":"percent","family":"euroeval","index":34.75545641402546,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/portuguese","measuredAt":null,"stderr":0.6236335061556587,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-italian:overall","benchmarkName":"EuroEval Italian","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":45.03125,"unit":"percent","family":"euroeval","index":33.31728031307685,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/italian","measuredAt":null,"stderr":0.6753251531669763,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":42.1875,"unit":"percent","family":"euroeval","index":29.458965891310253,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":0.9133326680350375,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":39.134285714285724,"unit":"percent","family":"euroeval","index":25.316456630188434,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.6765865707764986,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":35.472500000000004,"unit":"percent","family":"euroeval","index":20.348255968791513,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.5990004695323702,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-spanish:overall","benchmarkName":"EuroEval Spanish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":35.364999999999995,"unit":"percent","family":"euroeval","index":20.202403203836806,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/spanish","measuredAt":null,"stderr":0.6541346860547911,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"euromoe26ba06binstructpreview","providerId":"unknown","slug":"unknown-euromoe-2-6b-a0-6b-instruct-preview","name":"EuroMoE 2.6B A0.6B Instruct Preview","provider":"Unknown","description":"EuroMoE 2.6B A0.6B Instruct Preview is a model. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":5,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-french:overall","benchmarkName":"EuroEval French","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":20.8925,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/french","measuredAt":null,"stderr":0.7032935908992772,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-dutch:overall","benchmarkName":"EuroEval Dutch","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":17.20111111111111,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/dutch","measuredAt":null,"stderr":0.3396185223758279,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":15.352500000000001,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":0.43789624912757596,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":15.157142857142857,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.555905218760265,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-portuguese:overall","benchmarkName":"EuroEval Portuguese","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":14.422500000000003,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/portuguese","measuredAt":null,"stderr":0.9915486624467806,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-spanish:overall","benchmarkName":"EuroEval Spanish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":13.34625,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/spanish","measuredAt":null,"stderr":0.4344338988845138,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-italian:overall","benchmarkName":"EuroEval Italian","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":12.752500000000001,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/italian","measuredAt":null,"stderr":0.5173761929196202,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":12.362499999999999,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.586269988998243,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"euromoe26ba06bpreview","providerId":"unknown","slug":"unknown-euromoe-2-6b-a0-6b-preview","name":"EuroMoE 2.6B A0.6B Preview","provider":"Unknown","description":"EuroMoE 2.6B A0.6B Preview is a model. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":5,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":11.387142857142859,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.5691293027645881,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"eurus27bprime","providerId":"unknown","slug":"unknown-eurus-2-7b-prime","name":"Eurus 2 7B PRIME","provider":"Unknown","description":"Eurus 2 7B PRIME is a model. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":8.302644073198294,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"knowledge":1},"benchmarks":[{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":33.90151515151515,"unit":"percent","family":"gpqa-diamond","index":8.302644073198294,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.107317439394748,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":15,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"exaone-4-0-1-2b","providerId":"lg-ai-research","slug":"exaone-4-0-1-2b","name":"Exaone 4.0 1.2B","provider":"LG AI Research","description":"Exaone 4.0 1.2B is a standard model from LG AI Research in the Exaone 4.0 family. It has 6 scored results in 4 categories. It has no index.","logoUrl":null,"releaseDate":"2025-07-15","contextWindowTokens":128000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":16.392688939731798,"coding":null,"reasoning":22.073582735971804,"multimodal":null,"knowledge":23.101317557215857,"multilingual":null,"instruction":12.666747947251316,"math":null},"capabilityCounts":{"knowledge":2,"instruction":1,"agentic":1,"reasoning":2},"benchmarks":[{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":42.4,"unit":"percent","family":"gpqa-diamond","index":16.097222154448417,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaIfBench","benchmarkName":"Artificial Analysis IFBench","category":"instruction","score":25.3,"unit":"percent","family":"ifbench-family","index":12.666747947251316,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"tau2Bench","benchmarkName":"τ²-Bench Tool-Agent-User Evaluation","category":"agentic","score":20.5,"unit":"percent","family":"tau2-bench","index":16.392688939731798,"aboveFit":false,"run":null,"source":"Victor Barres, Honghua Dong, Soham Ray, Xujie Si, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2506.07982","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":5.7,"unit":"percent","family":"hle","index":30.1054129599833,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":0,"unit":"percent","family":"lcr","index":7.298049459170706,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":0,"unit":"percent","family":"critpt","index":36.8491160127729,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":55.00000000000001,"benchmarkCount":6,"categoryCount":4,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"extract","providerId":"scaledown","slug":"scaledown-extract","name":"Extract","provider":"ScaleDown","description":"Extract is a model from ScaleDown. It has no scored result.","logoUrl":null,"releaseDate":"2026-10-13T12:09:11.000Z","contextWindowTokens":1000000,"modalities":["text","image","file"],"openWeights":null,"details":{"apiId":"scaledown/extract","maxOutputTokens":900000,"knowledgeCutoff":null,"retiresAt":null,"tools":false,"structuredOutput":false,"cacheReadPerMillion":null,"weightsUrl":null},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":0.04,"outputPerMillion":0,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":327,"timeToFirstToken":0.854,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"decision-index-0-3-ezjev-4b-s2","providerId":"everettjf","slug":"decision-index-0-3-ezjev-4b-s2","name":"ezjev 4B s2","provider":"everettjf","description":"ezjev 4B s2 is a model from everettjf. It has no scored result.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"falconh1tinymultilingual100mbase","providerId":"tii","slug":"tii-falcon-h1-tiny-multilingual-100m-base","name":"Falcon H1 Tiny Multilingual 100M Base","provider":"TII","description":"Falcon H1 Tiny Multilingual 100M Base is a model from TII. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":5,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-french:overall","benchmarkName":"EuroEval French","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":13.80375,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/french","measuredAt":null,"stderr":0.5886066704514994,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-dutch:overall","benchmarkName":"EuroEval Dutch","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":12.301111111111112,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/dutch","measuredAt":null,"stderr":0.37306372540073707,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":11.712500000000002,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.5273770591332164,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-portuguese:overall","benchmarkName":"EuroEval Portuguese","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":10.97625,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/portuguese","measuredAt":null,"stderr":0.5510019283087856,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":9.42125,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":0.7361863300143517,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-italian:overall","benchmarkName":"EuroEval Italian","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":9.420000000000002,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/italian","measuredAt":null,"stderr":0.2123050576411217,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-spanish:overall","benchmarkName":"EuroEval Spanish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":8.72875,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/spanish","measuredAt":null,"stderr":0.3340237641246503,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":5.952857142857142,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.4534606163849788,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"falconh1tinymultilingual100minstruct","providerId":"tii","slug":"tii-falcon-h1-tiny-multilingual-100m-instruct","name":"Falcon H1 Tiny Multilingual 100M Instruct","provider":"TII","description":"Falcon H1 Tiny Multilingual 100M Instruct is a model from TII. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":5,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-french:overall","benchmarkName":"EuroEval French","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":14.54125,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/french","measuredAt":null,"stderr":0.502548194206287,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-dutch:overall","benchmarkName":"EuroEval Dutch","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":12.768888888888888,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/dutch","measuredAt":null,"stderr":0.2815543278057946,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-italian:overall","benchmarkName":"EuroEval Italian","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":11.7625,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/italian","measuredAt":null,"stderr":0.24790055970086067,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-portuguese:overall","benchmarkName":"EuroEval Portuguese","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":11.196250000000001,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/portuguese","measuredAt":null,"stderr":0.3469149751740331,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-spanish:overall","benchmarkName":"EuroEval Spanish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":10.370000000000001,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/spanish","measuredAt":null,"stderr":0.2589461816671565,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":9.89625,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.4726157265686363,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":7.9075,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":0.33645439884180445,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":6.855714285714285,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.19719978474136532,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"fara-1-5-27b","providerId":"microsoft","slug":"fara-1-5-27b","name":"Fara1.5-27B","provider":"Microsoft","description":"Fara1.5-27B is a reasoning model from Microsoft in the Fara 1.5 family. It has no scored result.","logoUrl":null,"releaseDate":"2026-07-17","contextWindowTokens":262000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":true,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[{"benchmarkId":"webVoyager","benchmarkName":"WebVoyager","category":"agentic","score":89.3,"unit":"percent","family":"webVoyager","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"fara-1-5-4b","providerId":"microsoft","slug":"fara-1-5-4b","name":"Fara1.5-4B","provider":"Microsoft","description":"Fara1.5-4B is a reasoning model from Microsoft in the Fara 1.5 family. It has no scored result.","logoUrl":null,"releaseDate":"2026-07-17","contextWindowTokens":262000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":true,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[{"benchmarkId":"webVoyager","benchmarkName":"WebVoyager","category":"agentic","score":80.8,"unit":"percent","family":"webVoyager","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"fish-audio-s2-1-pro","providerId":"fish-audio","slug":"fish-audio-s2-1-pro","name":"Fish Audio S2.1 Pro","provider":"Fish Audio","description":"Fish Audio S2.1 Pro is a standard model from Fish Audio. It has no scored result.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":false,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"flint-40b-a2-5b","providerId":"recursive","slug":"flint-40b-a2-5b","name":"Flint","provider":"Recursive","description":"Flint is a reasoning model from Recursive. It has 5 scored results in 3 categories. It has no index.","logoUrl":null,"releaseDate":"2026-10-21","contextWindowTokens":256000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":true,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":53.24968071692149,"coding":53.15132990960811,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":63.54381943251798,"math":null},"capabilityCounts":{"instruction":2,"coding":2,"agentic":1},"benchmarks":[{"benchmarkId":"ifeval","benchmarkName":"Instruction-Following Eval","category":"instruction","score":95.1,"unit":"percent","family":"ifeval","index":54.70566263017937,"aboveFit":false,"run":null,"source":"Jeffrey Zhou, Tianjian Lu, Swaroop Mishra, Siddhartha Brahma, Sujoy Basu, Yi Luan, Denny Zhou, Le Hou","sourceUrl":"https://arxiv.org/abs/2311.07911","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"ifBench","benchmarkName":"Instruction Following Benchmark","category":"instruction","score":89.2,"unit":"percent","family":"ifbench-family","index":72.38197623485658,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"liveCodeBenchV6","benchmarkName":"LiveCodeBench v6","category":"coding","score":80.3,"unit":"percent","family":"liveCodeBenchV6","index":50.347960018194634,"aboveFit":false,"run":null,"source":"LiveCodeBench maintainers","sourceUrl":"https://github.com/LiveCodeBench/LiveCodeBench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"sweVerified","benchmarkName":"Software Engineering Benchmark Verified","category":"coding","score":74.2,"unit":"percent","family":"swe-bench-verified","index":55.95469980102159,"aboveFit":false,"run":null,"source":"Carlos E. Jimenez, John Yang, Alexander Wettig, Shunyu Yao, Kexin Pei, Ofir Press, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2310.06770","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"terminalBench21","benchmarkName":"Terminal-Bench 2.1","category":"agentic","score":54.9,"unit":"percent","family":"terminal-bench-2-1","index":53.24968071692149,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":0,"outputPerMillion":0,"batch":null,"freeTier":true,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":45,"benchmarkCount":5,"categoryCount":3,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"fluxgeneralen","providerId":"unknown","slug":"unknown-flux-general-en","name":"Flux General En","provider":"Unknown","description":"Flux General En is a model. It has no scored result.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"frogboss32b2510","providerId":"unknown","slug":"unknown-frogboss-32b","name":"FrogBoss 32B","provider":"Unknown","description":"FrogBoss 32B is a model. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":40.76101369867754,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"coding":1},"benchmarks":[{"benchmarkId":"swebench-experiments:swe-bench-verified:overall","benchmarkName":"SWE-bench Verified","category":"coding","domain":null,"eligible":true,"ranked":true,"score":53.6,"unit":"percent","family":"swe-bench-verified","index":40.76101369867754,"aboveFit":false,"run":"FrogBoss-32B-2510","source":"SWE-bench team","sourceUrl":"https://github.com/SWE-bench/experiments/tree/main/evaluation/verified","measuredAt":"2026-09-01","stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":20,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:swe-bench-team","name":"SWE-bench team","url":"https://github.com","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"frogmini14b2510","providerId":"unknown","slug":"unknown-frogmini-14b","name":"FrogMini 14B","provider":"Unknown","description":"FrogMini 14B is a model. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":34.418018529737786,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"coding":1},"benchmarks":[{"benchmarkId":"swebench-experiments:swe-bench-verified:overall","benchmarkName":"SWE-bench Verified","category":"coding","domain":null,"eligible":true,"ranked":true,"score":45,"unit":"percent","family":"swe-bench-verified","index":34.418018529737786,"aboveFit":false,"run":"FrogMini-14B-2510","source":"SWE-bench team","sourceUrl":"https://github.com/SWE-bench/experiments/tree/main/evaluation/verified","measuredAt":"2026-09-01","stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":20,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:swe-bench-team","name":"SWE-bench team","url":"https://github.com","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"sakana-fugu-cyber","providerId":"sakana","slug":"sakana-fugu-cyber","name":"Fugu Cyber","provider":"Sakana","description":"Fugu Cyber is a reasoning model from Sakana in the Sakana Fugu family. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":"2026-07-21","contextWindowTokens":1000000,"modalities":["text"],"openWeights":false,"details":null,"reasoning":true,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":70.9720189035278,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"agentic":1},"benchmarks":[{"benchmarkId":"cyberGym","benchmarkName":"CyberGym","category":"agentic","score":86.9,"unit":"percent","family":"cyberGym","index":70.9720189035278,"aboveFit":false,"run":null,"source":"Zhun Wang, Tianneng Shi, Jingxuan He, Matthew Cai, Jialin Zhang, Dawn Song","sourceUrl":"https://www.cybergym.io/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"ctiRealm","benchmarkName":"CTI-REALM","category":"agentic","score":72.1,"unit":"percent","family":"ctiRealm","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":6,"outputPerMillion":36,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":20,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"fugumax","providerId":"sakana","slug":"sakana-fugu-max","name":"Fugu Max","provider":"Sakana","description":"Fugu Max is a model from Sakana. It has no scored result.","logoUrl":null,"releaseDate":"2026-09-11T05:32:51.000Z","contextWindowTokens":1000000,"modalities":["text","image","file"],"openWeights":null,"details":{"apiId":"sakana/fugu-max","maxOutputTokens":128000,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.25,"weightsUrl":null},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":2,"outputPerMillion":6,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":29,"timeToFirstToken":2.8415,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":2,"outputPerMillion":6,"cacheReadPerMillion":0.25,"contextTokens":1000000,"maxOutputTokens":128000,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"fuguultra","providerId":"sakana","slug":"sakana-fugu-ultra","name":"Fugu Ultra","provider":"Sakana","description":"Fugu Ultra is a model from Sakana. It has no scored result.","logoUrl":null,"releaseDate":"2026-06-24T04:45:03.000Z","contextWindowTokens":1000000,"modalities":["text","image"],"openWeights":null,"details":{"apiId":"sakana/fugu-ultra","maxOutputTokens":128000,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.5,"weightsUrl":null},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":5,"outputPerMillion":30,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":33,"timeToFirstToken":7.209,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":5,"outputPerMillion":30,"cacheReadPerMillion":0.5,"contextTokens":1000000,"maxOutputTokens":128000,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"fuguultrav2","providerId":"sakana","slug":"sakana-fugu-ultra-v2","name":"Fugu Ultra v2","provider":"Sakana","description":"Fugu Ultra v2 is a model from Sakana. It has no scored result.","logoUrl":null,"releaseDate":"2026-09-11T05:43:03.000Z","contextWindowTokens":1000000,"modalities":["text","image","file"],"openWeights":null,"details":{"apiId":"sakana/fugu-ultra-v2","maxOutputTokens":128000,"knowledgeCutoff":"2026-08-28","retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.5,"weightsUrl":null},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":5,"outputPerMillion":30,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":44,"timeToFirstToken":4.532,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":5,"outputPerMillion":30,"cacheReadPerMillion":0.5,"contextTokens":1000000,"maxOutputTokens":128000,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gams312b","providerId":"unknown","slug":"unknown-gams3-12b","name":"GaMS3 12B","provider":"Unknown","description":"GaMS3 12B is a model. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":34.999917277545876,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":46.27142857142858,"unit":"percent","family":"euroeval","index":34.999917277545876,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.7607380948655009,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gams312binstruct","providerId":"unknown","slug":"unknown-gams3-12b-instruct","name":"GaMS3 12B Instruct","provider":"Unknown","description":"GaMS3 12B Instruct is a model. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":34.2323731589803,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":45.705714285714286,"unit":"percent","family":"euroeval","index":34.2323731589803,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.8175735628139188,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gemini-1-0-pro","providerId":"google","slug":"gemini-1-0-pro","name":"Gemini 1.0 Pro","provider":"Google","description":"Gemini 1.0 Pro is a standard model from Google. It has 4 scored results in 2 categories. It has no index.","logoUrl":null,"releaseDate":"2023-12-06","contextWindowTokens":32000,"modalities":["text"],"openWeights":false,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":17.04406947663474,"multilingual":null,"instruction":null,"math":7.282764352578525},"capabilityCounts":{"knowledge":2,"math":2},"benchmarks":[{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":33.964646464646464,"unit":"percent","family":"gpqa-diamond","index":8.360546388534146,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.011093340517674,"costPerTest":null,"latency":null},{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":27.7,"unit":"percent","family":"gpqa-diamond","index":5,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-math-level-5:overall","benchmarkName":"MATH level 5","category":"math","domain":null,"eligible":true,"ranked":true,"score":11.244335347432024,"unit":"percent","family":"epoch-ai:epoch-math-level-5:overall","index":5,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0.5990949270731778,"costPerTest":null,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":4.2,"unit":"percent","family":"hle","index":28.60047384243861,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":1.1111111111111112,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":9.565528705157057,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0.6054026310473157,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":25,"benchmarkCount":4,"categoryCount":2,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gemini10pro002","providerId":"google","slug":"google-gemini-1-0-pro-002","name":"Gemini 1.0 Pro 002","provider":"Google","description":"Gemini 1.0 Pro 002 is a model from Google. It has no scored result.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[{"benchmarkId":"vals-ai:legal_bench:overall","benchmarkName":"LegalBench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":70.247,"unit":"percent","family":"vals-ai:legal_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","measuredAt":"2026-10-13","stderr":0.765,"costPerTest":null,"latency":0.465}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gemini15flash001","providerId":"google","slug":"google-gemini-1-5-flash-001","name":"Gemini 1.5 Flash 001","provider":"Google","description":"Gemini 1.5 Flash 001 is a model from Google. It has no scored result.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[{"benchmarkId":"vals-ai:corp_fin_v2:overall","benchmarkName":"CorpFin v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":28.633,"unit":"percent","family":"vals-ai:corp_fin_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","measuredAt":"2026-08-12","stderr":0.88,"costPerTest":0.006413,"latency":1.156}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":{"elo":1239.3084864401344,"eloUpper":1243.7721824814016,"eloLower":1234.8447903988667,"votes":62833,"rank":302,"rankUpper":null,"rankLower":null,"arenaName":"Gemini 1.5 Flash-001","organization":"google","updatedAt":"2026-11-09"},"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gemini15flash002","providerId":"google","slug":"google-gemini-1-5-flash-002","name":"Gemini 1.5 Flash 002","provider":"Google","description":"Gemini 1.5 Flash 002 is a model from Google. It has 6 scored results in 4 categories. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":19.969515265989873,"reasoning":null,"multimodal":29.583185294530978,"knowledge":23.249396233885996,"multilingual":null,"instruction":null,"math":25.22964583774263},"capabilityCounts":{"math":2,"knowledge":2,"multimodal":1,"coding":1},"benchmarks":[{"benchmarkId":"vals-ai:mgsm:overall","benchmarkName":"MGSM","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":86.582,"unit":"percent","family":"vals-ai:mgsm:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mgsm","measuredAt":"2026-01-09","stderr":2.126,"costPerTest":0.000064,"latency":1.413},{"benchmarkId":"vals-ai:math500:overall","benchmarkName":"MATH 500","category":"math","domain":null,"eligible":true,"ranked":true,"score":78.8,"unit":"percent","family":"math-500","index":27.399330395851816,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/math500","measuredAt":"2026-01-09","stderr":3.576,"costPerTest":0.00012,"latency":2.645},{"benchmarkId":"vals-ai:mmlu_pro:overall","benchmarkName":"MMLU Pro","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":65.606,"unit":"percent","family":"mmlu-pro","index":27.136435590609146,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","measuredAt":"2026-09-01","stderr":0.456,"costPerTest":0.00017,"latency":1.682},{"benchmarkId":"vals-ai:legal_bench:overall","benchmarkName":"LegalBench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":63.238,"unit":"percent","family":"vals-ai:legal_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","measuredAt":"2026-10-13","stderr":0.736,"costPerTest":null,"latency":0.476},{"benchmarkId":"vals-ai:mmmu:overall","benchmarkName":"MMMU Pro","category":"multimodal","domain":null,"eligible":true,"ranked":true,"score":57.192,"unit":"percent","family":"mmmu-pro","index":29.583185294530978,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmmu","measuredAt":"2026-09-01","stderr":1.189,"costPerTest":0.000114,"latency":41.314},{"benchmarkId":"vals-ai:tax_eval_v2:overall","benchmarkName":"TaxEval v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":48.202,"unit":"percent","family":"vals-ai:tax_eval_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","measuredAt":"2026-09-01","stderr":0.98,"costPerTest":0.000155,"latency":3.701},{"benchmarkId":"vals-ai:gpqa:overall","benchmarkName":"GPQA Diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":45.96,"unit":"percent","family":"gpqa-diamond","index":19.362356877162853,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","measuredAt":"2026-09-01","stderr":2.504,"costPerTest":0.000233,"latency":3.39},{"benchmarkId":"vals-ai:mortgage_tax:overall","benchmarkName":"MortgageTax","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":42.766,"unit":"percent","family":"vals-ai:mortgage_tax:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mortgage_tax","measuredAt":"2026-09-01","stderr":1.835,"costPerTest":0.00012,"latency":2.1},{"benchmarkId":"vals-ai:corp_fin_v2:overall","benchmarkName":"CorpFin v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":38.19,"unit":"percent","family":"vals-ai:corp_fin_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","measuredAt":"2026-08-12","stderr":0.958,"costPerTest":0.007862,"latency":28.407},{"benchmarkId":"vals-ai:lcb:overall","benchmarkName":"LiveCodeBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":36.91,"unit":"percent","family":"livecodebench","index":19.969515265989873,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","measuredAt":"2026-09-01","stderr":1.079,"costPerTest":0.000147,"latency":2.512},{"benchmarkId":"vals-ai:aime:overall","benchmarkName":"AIME","category":"math","domain":null,"eligible":true,"ranked":true,"score":17.292,"unit":"percent","family":"vals-ai:aime:overall","index":23.05996127963344,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/aime","measuredAt":"2026-04-16","stderr":0.901,"costPerTest":0.000266,"latency":5.696}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":{"elo":1286.6613206037023,"eloUpper":1290.863153431074,"eloLower":1282.4594877763304,"votes":34902,"rank":267,"rankUpper":null,"rankLower":null,"arenaName":"Gemini 1.5 Flash-002","organization":"google","updatedAt":"2026-11-09"},"quality":{"coveragePercent":55.00000000000001,"benchmarkCount":6,"categoryCount":4,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gemini15flash8b","providerId":"google","slug":"google-gemini-1-5-flash-8b","name":"Gemini 1.5 Flash 8B","provider":"Google","description":"Gemini 1.5 Flash 8B is a model from Google. It has 2 scored results in 2 categories. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":7.434109343160621,"multilingual":null,"instruction":null,"math":11.585646919508044},"capabilityCounts":{"knowledge":1,"math":1},"benchmarks":[{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":32.95454545454545,"unit":"percent","family":"gpqa-diamond","index":7.434109343160621,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.4165511162117568,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":4.583333333333333,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":11.585646919508044,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.0100756305184246,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":25,"benchmarkCount":2,"categoryCount":2,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gemini-1-5-pro","providerId":"google","slug":"gemini-1-5-pro","name":"Gemini 1.5 Pro","provider":"Google","description":"Gemini 1.5 Pro is a standard model from Google. It has 6 scored results in 4 categories. It has no index.","logoUrl":null,"releaseDate":"2024-02-15","contextWindowTokens":2000000,"modalities":["text"],"openWeights":false,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":39.55582316302347,"multimodal":26.730489609389767,"knowledge":28.430123595273226,"multilingual":null,"instruction":null,"math":21.798527028436183},"capabilityCounts":{"knowledge":2,"math":2,"multimodal":1,"reasoning":1},"benchmarks":[{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":58.9,"unit":"percent","family":"gpqa-diamond","index":31.230571290624866,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-math-level-5:overall","benchmarkName":"MATH level 5","category":"math","domain":null,"eligible":true,"ranked":true,"score":55.570241691842895,"unit":"percent","family":"epoch-ai:epoch-math-level-5:overall","index":25.99145485859836,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.0574902459145803,"costPerTest":null,"latency":null},{"benchmarkId":"aaMmmuPro","benchmarkName":"Artificial Analysis MMMU-Pro","category":"multimodal","score":55,"unit":"percent","family":"mmmu-pro","index":26.730489609389767,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmmu-pro","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":51.54671717171717,"unit":"percent","family":"gpqa-diamond","index":24.486341209566984,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.617295639558329,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":14.930555555555557,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":17.605599198274007,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":5.186782011216038,"costPerTest":null,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":4.6,"unit":"percent","family":"hle","index":29.001790940450533,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"arc-prize:v2-semi-private:overall","benchmarkName":"ARC-AGI-2 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":0.8,"unit":"percent","family":"arc-prize:v2-semi-private:overall","index":39.55582316302347,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.04,"latency":null},{"benchmarkId":"arc-prize:v2-public-eval:overall","benchmarkName":"ARC-AGI-2 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":0,"unit":"percent","family":"arc-prize:v2-public-eval:overall","index":null,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.04,"latency":null}],"price":{"inputPerMillion":1.25,"outputPerMillion":5,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":50,"benchmarkCount":6,"categoryCount":4,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:arc-prize-foundation","name":"ARC Prize Foundation","url":"https://arcprize.org","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gemini15pro002","providerId":"google","slug":"google-gemini-1-5-pro-002","name":"Gemini 1.5 Pro 002","provider":"Google","description":"Gemini 1.5 Pro 002 is a model from Google. It has 6 scored results in 4 categories. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":23.95974083626405,"reasoning":null,"multimodal":40.4096339280573,"knowledge":35.02680376171908,"multilingual":null,"instruction":null,"math":27.661768621959315},"capabilityCounts":{"math":2,"knowledge":2,"multimodal":1,"coding":1},"benchmarks":[{"benchmarkId":"vals-ai:mgsm:overall","benchmarkName":"MGSM","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":89.2,"unit":"percent","family":"vals-ai:mgsm:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mgsm","measuredAt":"2026-01-09","stderr":1.921,"costPerTest":0.001014,"latency":2.798},{"benchmarkId":"vals-ai:math500:overall","benchmarkName":"MATH 500","category":"math","domain":null,"eligible":true,"ranked":true,"score":82.8,"unit":"percent","family":"math-500","index":31.626363245571337,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/math500","measuredAt":"2026-01-09","stderr":3.305,"costPerTest":0.002164,"latency":5.021},{"benchmarkId":"vals-ai:medqa:overall","benchmarkName":"MedQA","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":76.53,"unit":"percent","family":"vals-ai:medqa:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medqa","measuredAt":"2026-04-16","stderr":0.758,"costPerTest":0.002954,"latency":5.927},{"benchmarkId":"vals-ai:mmlu_pro:overall","benchmarkName":"MMLU Pro","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":75.294,"unit":"percent","family":"mmlu-pro","index":39.342155966817884,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","measuredAt":"2026-09-01","stderr":0.417,"costPerTest":0.002649,"latency":3.343},{"benchmarkId":"vals-ai:legal_bench:overall","benchmarkName":"LegalBench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":69.08,"unit":"percent","family":"vals-ai:legal_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","measuredAt":"2026-10-13","stderr":0.751,"costPerTest":null,"latency":0.898},{"benchmarkId":"vals-ai:mmmu:overall","benchmarkName":"MMMU Pro","category":"multimodal","domain":null,"eligible":true,"ranked":true,"score":65.511,"unit":"percent","family":"mmmu-pro","index":40.4096339280573,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmmu","measuredAt":"2026-09-01","stderr":1.142,"costPerTest":0.001846,"latency":43.736},{"benchmarkId":"vals-ai:tax_eval_v2:overall","benchmarkName":"TaxEval v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":59.485,"unit":"percent","family":"vals-ai:tax_eval_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","measuredAt":"2026-09-01","stderr":0.962,"costPerTest":0.002545,"latency":9.86},{"benchmarkId":"vals-ai:gpqa:overall","benchmarkName":"GPQA Diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":58.334,"unit":"percent","family":"gpqa-diamond","index":30.711451556620272,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","measuredAt":"2026-09-01","stderr":2.475,"costPerTest":0.003655,"latency":7.709},{"benchmarkId":"vals-ai:mortgage_tax:overall","benchmarkName":"MortgageTax","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":56.756,"unit":"percent","family":"vals-ai:mortgage_tax:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mortgage_tax","measuredAt":"2026-09-01","stderr":1.916,"costPerTest":0.002028,"latency":4.205},{"benchmarkId":"vals-ai:lcb:overall","benchmarkName":"LiveCodeBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":41.719,"unit":"percent","family":"livecodebench","index":23.95974083626405,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","measuredAt":"2026-09-01","stderr":1.089,"costPerTest":0.001975,"latency":3.954},{"benchmarkId":"vals-ai:corp_fin_v2:overall","benchmarkName":"CorpFin v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":40.521,"unit":"percent","family":"vals-ai:corp_fin_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","measuredAt":"2026-08-12","stderr":0.967,"costPerTest":0.131149,"latency":38.05},{"benchmarkId":"vals-ai:aime:overall","benchmarkName":"AIME","category":"math","domain":null,"eligible":true,"ranked":true,"score":18.75,"unit":"percent","family":"vals-ai:aime:overall","index":23.69717399834729,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/aime","measuredAt":"2026-04-16","stderr":0.534,"costPerTest":0.004203,"latency":10.642}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":{"elo":1319.1402905321447,"eloUpper":1322.5199059348727,"eloLower":1315.7606751294165,"votes":55606,"rank":241,"rankUpper":null,"rankLower":null,"arenaName":"Gemini 1.5 Pro-002","organization":"google","updatedAt":"2026-11-09"},"quality":{"coveragePercent":55.00000000000001,"benchmarkCount":6,"categoryCount":4,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gemini20flash001","providerId":"google","slug":"google-gemini-2-0-flash-001","name":"Gemini 2.0 Flash 001","provider":"Google","description":"Gemini 2.0 Flash 001 is a model from Google. It has 6 scored results in 4 categories. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":25.527122017850218,"reasoning":null,"multimodal":45.97317136144174,"knowledge":39.464980806023654,"multilingual":null,"instruction":null,"math":32.8222696176372},"capabilityCounts":{"math":2,"knowledge":2,"multimodal":1,"coding":1},"benchmarks":[{"benchmarkId":"vals-ai:mgsm:overall","benchmarkName":"MGSM","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":89.018,"unit":"percent","family":"vals-ai:mgsm:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mgsm","measuredAt":"2026-01-09","stderr":1.877,"costPerTest":0.000102,"latency":1.536},{"benchmarkId":"vals-ai:math500:overall","benchmarkName":"MATH 500","category":"math","domain":null,"eligible":true,"ranked":true,"score":88,"unit":"percent","family":"math-500","index":37.12150595020672,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/math500","measuredAt":"2026-01-09","stderr":2.852,"costPerTest":0.00026,"latency":3.367},{"benchmarkId":"vals-ai:medqa:overall","benchmarkName":"MedQA","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":81.467,"unit":"percent","family":"vals-ai:medqa:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medqa","measuredAt":"2026-04-16","stderr":0.695,"costPerTest":0.000114,"latency":1.941},{"benchmarkId":"vals-ai:legal_bench:overall","benchmarkName":"LegalBench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":78.364,"unit":"percent","family":"vals-ai:legal_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","measuredAt":"2026-10-13","stderr":0.782,"costPerTest":null,"latency":0.447},{"benchmarkId":"vals-ai:mmlu_pro:overall","benchmarkName":"MMLU Pro","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":77.376,"unit":"percent","family":"mmlu-pro","index":41.96522675782394,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","measuredAt":"2026-09-01","stderr":0.407,"costPerTest":0.000414,"latency":4.318},{"benchmarkId":"vals-ai:mmmu:overall","benchmarkName":"MMMU Pro","category":"multimodal","domain":null,"eligible":true,"ranked":true,"score":69.786,"unit":"percent","family":"mmmu-pro","index":45.97317136144174,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmmu","measuredAt":"2026-09-01","stderr":1.104,"costPerTest":0.000365,"latency":45.203},{"benchmarkId":"vals-ai:tax_eval_v2:overall","benchmarkName":"TaxEval v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":65.25,"unit":"percent","family":"vals-ai:tax_eval_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","measuredAt":"2026-09-01","stderr":0.933,"costPerTest":0.000296,"latency":5.702},{"benchmarkId":"vals-ai:gpqa:overall","benchmarkName":"GPQA Diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":65.152,"unit":"percent","family":"gpqa-diamond","index":36.964734854223366,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","measuredAt":"2026-09-01","stderr":2.394,"costPerTest":0.00049,"latency":5.803},{"benchmarkId":"vals-ai:mortgage_tax:overall","benchmarkName":"MortgageTax","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":59.658,"unit":"percent","family":"vals-ai:mortgage_tax:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mortgage_tax","measuredAt":"2026-09-01","stderr":1.91,"costPerTest":0.00027,"latency":3.794},{"benchmarkId":"vals-ai:lcb:overall","benchmarkName":"LiveCodeBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":43.608,"unit":"percent","family":"livecodebench","index":25.527122017850218,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","measuredAt":"2026-09-01","stderr":1.078,"costPerTest":0.000304,"latency":3.358},{"benchmarkId":"vals-ai:corp_fin_v2:overall","benchmarkName":"CorpFin v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":33.722,"unit":"percent","family":"vals-ai:corp_fin_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","measuredAt":"2026-08-12","stderr":0.93,"costPerTest":0.010481,"latency":30.038},{"benchmarkId":"vals-ai:aime:overall","benchmarkName":"AIME","category":"math","domain":null,"eligible":true,"ranked":true,"score":29.792,"unit":"percent","family":"vals-ai:aime:overall","index":28.523033285067676,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/aime","measuredAt":"2026-04-16","stderr":0.644,"costPerTest":0.001044,"latency":11.214}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":{"elo":1354.115156879162,"eloUpper":1357.8664937183348,"eloLower":1350.3638200399892,"votes":43428,"rank":208,"rankUpper":null,"rankLower":null,"arenaName":"Gemini 2.0 Flash-001","organization":"google","updatedAt":"2026-11-09"},"quality":{"coveragePercent":55.00000000000001,"benchmarkCount":6,"categoryCount":4,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gemini20flashexp","providerId":"google","slug":"google-gemini-2-0-flash-exp","name":"Gemini 2.0 Flash Exp","provider":"Google","description":"Gemini 2.0 Flash Exp is a model from Google. It has 2 scored results in 2 categories. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":33.82797246751083,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":38.1782641626366},"capabilityCounts":{"math":1,"coding":1},"benchmarks":[{"benchmarkId":"vals-ai:math500:overall","benchmarkName":"MATH 500","category":"math","domain":null,"eligible":true,"ranked":true,"score":89,"unit":"percent","family":"math-500","index":38.1782641626366,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/math500","measuredAt":"2026-01-09","stderr":2.748,"costPerTest":0.0002,"latency":3.352},{"benchmarkId":"vals-ai:tax_eval_v2:overall","benchmarkName":"TaxEval v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":67.744,"unit":"percent","family":"vals-ai:tax_eval_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","measuredAt":"2026-09-01","stderr":0.915,"costPerTest":0.000302,"latency":7.486},{"benchmarkId":"vals-ai:mortgage_tax:overall","benchmarkName":"MortgageTax","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":60.374,"unit":"percent","family":"vals-ai:mortgage_tax:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mortgage_tax","measuredAt":"2026-09-01","stderr":1.902,"costPerTest":0.000132,"latency":2.796},{"benchmarkId":"swebench-experiments:swe-bench-verified:overall","benchmarkName":"SWE-bench Verified","category":"coding","domain":null,"eligible":true,"ranked":true,"score":44.2,"unit":"percent","family":"swe-bench-verified","index":33.82797246751083,"aboveFit":false,"run":"CodeShellAgent","source":"SWE-bench team","sourceUrl":"https://github.com/SWE-bench/experiments/tree/main/evaluation/verified","measuredAt":"2026-09-01","stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":30,"benchmarkCount":2,"categoryCount":2,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:swe-bench-team","name":"SWE-bench team","url":"https://github.com","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gemini20flashthinking","providerId":"google","slug":"google-gemini-2-0-flash-thinking","name":"Gemini 2.0 Flash Thinking","provider":"Google","description":"Gemini 2.0 Flash Thinking is a model from Google. It has 2 scored results in 2 categories. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":29.552793801453426,"multilingual":null,"instruction":null,"math":42.533857963365264},"capabilityCounts":{"math":1,"knowledge":1},"benchmarks":[{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":57.77777777777777,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":42.533857963365264,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":7.446027270295804,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":57.07070707070707,"unit":"percent","family":"gpqa-diamond","index":29.552793801453426,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":3.526552724601199,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":25,"benchmarkCount":2,"categoryCount":2,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gemini20flashthinkingexp0121","providerId":"google","slug":"google-gemini-2-0-flash-thinking-exp-01-21","name":"Gemini 2.0 Flash Thinking Exp 01.21","provider":"Google","description":"Gemini 2.0 Flash Thinking Exp 01.21 is a model from Google. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":33.52852802794512},"capabilityCounts":{"math":1},"benchmarks":[{"benchmarkId":"vals-ai:math500:overall","benchmarkName":"MATH 500","category":"math","domain":null,"eligible":true,"ranked":true,"score":84.6,"unit":"percent","family":"math-500","index":33.52852802794512,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/math500","measuredAt":"2026-01-09","stderr":3.163,"costPerTest":0.000387,"latency":11.801},{"benchmarkId":"vals-ai:tax_eval_v2:overall","benchmarkName":"TaxEval v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":69.788,"unit":"percent","family":"vals-ai:tax_eval_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","measuredAt":"2026-09-01","stderr":0.899,"costPerTest":0.000827,"latency":12.807}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":10,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gemini-2-0-flash-lite","providerId":"google","slug":"gemini-2-0-flash-lite","name":"Gemini 2.0 Flash-Lite","provider":"Google","description":"Gemini 2.0 Flash-Lite is a standard model from Google in the Gemini 2.0 Flash family. It has 5 scored results in 4 categories. It has no index.","logoUrl":null,"releaseDate":"2025-02-25","contextWindowTokens":1000000,"modalities":["text"],"openWeights":false,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":10.447397251767804,"reasoning":null,"multimodal":32.44977580649874,"knowledge":24.999940847997266,"multilingual":null,"instruction":null,"math":11.921716188316843},"capabilityCounts":{"multimodal":1,"knowledge":2,"math":1,"coding":1},"benchmarks":[{"benchmarkId":"mmmu","benchmarkName":"Massive Multi-discipline Multimodal Understanding","category":"multimodal","score":65.1,"unit":"percent","family":"mmmu","index":32.44977580649874,"aboveFit":false,"run":null,"source":"MMMU authors","sourceUrl":"https://arxiv.org/abs/2401.05508","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gpqa","benchmarkName":"Graduate-Level Google-Proof Q&A","category":"knowledge","score":50.5,"unit":"percent","family":"gpqa-diamond","index":23.526320821298675,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aime2025","benchmarkName":"American Invitational Mathematics Examination 2025","category":"math","score":23.8,"unit":"percent","family":"aime2025","index":11.921716188316843,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"simpleQa","benchmarkName":"Measuring Short-Form Factuality in Large Language Models","category":"knowledge","score":16.5,"unit":"percent","family":"simpleQa","index":26.473560874695853,"aboveFit":false,"run":null,"source":"Jason Wei, Najoung Kim, Hyung Won Chung, Yu-An Chung, Siddhartha Papay, Yifeng Lu, Hannaneh Hajishirzi, Luke Zettlemoyer","sourceUrl":"https://arxiv.org/abs/2411.04368","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"sweVerified","benchmarkName":"Software Engineering Benchmark Verified","category":"coding","score":12.5,"unit":"percent","family":"swe-bench-verified","index":10.447397251767804,"aboveFit":false,"run":null,"source":"Carlos E. Jimenez, John Yang, Alexander Wettig, Shunyu Yao, Kexin Pei, Ofir Press, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2310.06770","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":55.00000000000001,"benchmarkCount":5,"categoryCount":4,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gemini20pro","providerId":"google","slug":"google-gemini-2-0-pro","name":"Gemini 2.0 Pro","provider":"Google","description":"Gemini 2.0 Pro is a model from Google. It has 2 scored results in 2 categories. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":37.42750868712835,"multilingual":null,"instruction":null,"math":39.53867339326304},"capabilityCounts":{"math":1,"knowledge":1},"benchmarks":[{"benchmarkId":"epoch-ai:epoch-math-level-5:overall","benchmarkName":"MATH level 5","category":"math","domain":null,"eligible":true,"ranked":true,"score":83.45921450151057,"unit":"percent","family":"epoch-ai:epoch-math-level-5:overall","index":39.53867339326304,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.021492751504924,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":65.65656565656566,"unit":"percent","family":"gpqa-diamond","index":37.42750868712835,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":3.3832012232444413,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-2025-02-28-public:overall","benchmarkName":"FrontierMath-2025-02-28-Public","category":"math","domain":null,"eligible":false,"ranked":true,"score":0,"unit":"percent","family":"epoch-ai:epoch-frontiermath-2025-02-28-public:overall","index":null,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":25,"benchmarkCount":2,"categoryCount":2,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gemini20proexp0205","providerId":"google","slug":"google-gemini-2-0-pro-exp-02-05","name":"Gemini 2.0 Pro Exp 02.05","provider":"Google","description":"Gemini 2.0 Pro Exp 02.05 is a model from Google. It has no scored result.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[{"benchmarkId":"vals-ai:tax_eval_v2:overall","benchmarkName":"TaxEval v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":68.152,"unit":"percent","family":"vals-ai:tax_eval_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","measuredAt":"2026-09-01","stderr":0.916,"costPerTest":0.004012,"latency":8.989},{"benchmarkId":"vals-ai:corp_fin_v2:overall","benchmarkName":"CorpFin v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":43.435,"unit":"percent","family":"vals-ai:corp_fin_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","measuredAt":"2026-08-12","stderr":0.971,"costPerTest":0.104585,"latency":18.879}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gemini25deepthink","providerId":"google","slug":"google-gemini-2-5-deep-think","name":"Gemini 2.5 Deep Think","provider":"Google","description":"Gemini 2.5 Deep Think is a model from Google. It has 2 scored results in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":54.64009106164595},"capabilityCounts":{"math":2},"benchmarks":[{"benchmarkId":"epoch-ai:epoch-frontiermath-2025-02-28-private:overall","benchmarkName":"FrontierMath-2025-02-28-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":28.999999999999996,"unit":"percent","family":"epoch-ai:epoch-frontiermath-2025-02-28-private:overall","index":55.514724521927285,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.7,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-private:overall","benchmarkName":"FrontierMath-Tier-4-2025-07-01-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":10.4,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-private:overall","index":53.7654576013646,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":4.3999999999999995,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":10,"benchmarkCount":2,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gemini25flashpreviewthinking16k","providerId":"google","slug":"google-gemini-2-5-flash-preview-thinking-16k","name":"Gemini 2.5 Flash (Preview) (Thinking 16K)","provider":"Google","description":"Gemini 2.5 Flash (Preview) (Thinking 16K) is a model from Google. It has 2 scored results in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":41.203982641251386,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"reasoning":2},"benchmarks":[{"benchmarkId":"arc-prize:v1-public-eval:overall","benchmarkName":"ARC-AGI-1 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":36,"unit":"percent","family":"arc-prize:v1-public-eval:overall","index":null,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.1846,"latency":null},{"benchmarkId":"arc-prize:v1-semi-private:overall","benchmarkName":"ARC-AGI-1 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":33.33,"unit":"percent","family":"arc-prize:v1-semi-private:overall","index":42.310363172327435,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.2134,"latency":null},{"benchmarkId":"arc-prize:v2-semi-private:overall","benchmarkName":"ARC-AGI-2 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":1.9800000000000002,"unit":"percent","family":"arc-prize:v2-semi-private:overall","index":40.09760211017534,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.3173,"latency":null},{"benchmarkId":"arc-prize:v2-public-eval:overall","benchmarkName":"ARC-AGI-2 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":1.11,"unit":"percent","family":"arc-prize:v2-public-eval:overall","index":null,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.324,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":15,"benchmarkCount":2,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:arc-prize-foundation","name":"ARC Prize Foundation","url":"https://arcprize.org","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gemini25flashpreviewthinking1k","providerId":"google","slug":"google-gemini-2-5-flash-preview-thinking-1k","name":"Gemini 2.5 Flash (Preview) (Thinking 1K)","provider":"Google","description":"Gemini 2.5 Flash (Preview) (Thinking 1K) is a model from Google. It has 2 scored results in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":37.2616914320661,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"reasoning":2},"benchmarks":[{"benchmarkId":"arc-prize:v1-public-eval:overall","benchmarkName":"ARC-AGI-1 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":17.96,"unit":"percent","family":"arc-prize:v1-public-eval:overall","index":null,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.0231,"latency":null},{"benchmarkId":"arc-prize:v1-semi-private:overall","benchmarkName":"ARC-AGI-1 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":16,"unit":"percent","family":"arc-prize:v1-semi-private:overall","index":34.34313650778115,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.0356,"latency":null},{"benchmarkId":"arc-prize:v2-semi-private:overall","benchmarkName":"ARC-AGI-2 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":2.16,"unit":"percent","family":"arc-prize:v2-semi-private:overall","index":40.18024635635104,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.0302,"latency":null},{"benchmarkId":"arc-prize:v2-public-eval:overall","benchmarkName":"ARC-AGI-2 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":0,"unit":"percent","family":"arc-prize:v2-public-eval:overall","index":null,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.0348,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":15,"benchmarkCount":2,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:arc-prize-foundation","name":"ARC Prize Foundation","url":"https://arcprize.org","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gemini25flashpreviewthinking24k","providerId":"google","slug":"google-gemini-2-5-flash-preview-thinking-24k","name":"Gemini 2.5 Flash (Preview) (Thinking 24K)","provider":"Google","description":"Gemini 2.5 Flash (Preview) (Thinking 24K) is a model from Google. It has 2 scored results in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":41.10267230603751,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"reasoning":2},"benchmarks":[{"benchmarkId":"arc-prize:v1-public-eval:overall","benchmarkName":"ARC-AGI-1 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":39.12,"unit":"percent","family":"arc-prize:v1-public-eval:overall","index":null,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.1858,"latency":null},{"benchmarkId":"arc-prize:v1-semi-private:overall","benchmarkName":"ARC-AGI-1 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":32.33,"unit":"percent","family":"arc-prize:v1-semi-private:overall","index":41.85062706935304,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.1971,"latency":null},{"benchmarkId":"arc-prize:v2-public-eval:overall","benchmarkName":"ARC-AGI-2 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":2.64,"unit":"percent","family":"arc-prize:v2-public-eval:overall","index":null,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.3305,"latency":null},{"benchmarkId":"arc-prize:v2-semi-private:overall","benchmarkName":"ARC-AGI-2 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":2.54,"unit":"percent","family":"arc-prize:v2-semi-private:overall","index":40.354717542721986,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.3191,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":15,"benchmarkCount":2,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:arc-prize-foundation","name":"ARC Prize Foundation","url":"https://arcprize.org","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gemini25flashpreviewthinking8k","providerId":"google","slug":"google-gemini-2-5-flash-preview-thinking-8k","name":"Gemini 2.5 Flash (Preview) (Thinking 8K)","provider":"Google","description":"Gemini 2.5 Flash (Preview) (Thinking 8K) is a model from Google. It has 2 scored results in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":39.51211168416573,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"reasoning":2},"benchmarks":[{"benchmarkId":"arc-prize:v1-public-eval:overall","benchmarkName":"ARC-AGI-1 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":34.53,"unit":"percent","family":"arc-prize:v1-public-eval:overall","index":null,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.1158,"latency":null},{"benchmarkId":"arc-prize:v1-semi-private:overall","benchmarkName":"ARC-AGI-1 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":25.83,"unit":"percent","family":"arc-prize:v1-semi-private:overall","index":38.862342400019465,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.1344,"latency":null},{"benchmarkId":"arc-prize:v2-semi-private:overall","benchmarkName":"ARC-AGI-2 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":2.12,"unit":"percent","family":"arc-prize:v2-semi-private:overall","index":40.161880968312,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.1994,"latency":null},{"benchmarkId":"arc-prize:v2-public-eval:overall","benchmarkName":"ARC-AGI-2 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":1.11,"unit":"percent","family":"arc-prize:v2-public-eval:overall","index":null,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.2047,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":15,"benchmarkCount":2,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:arc-prize-foundation","name":"ARC Prize Foundation","url":"https://arcprize.org","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gemini25flashlitenothinking","providerId":"google","slug":"google-gemini-2-5-flash-lite-no-thinking","name":"Gemini 2.5 Flash Lite No Thinking","provider":"Google","description":"Gemini 2.5 Flash Lite No Thinking is a model from Google. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":38.29916981370954,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-french:overall","benchmarkName":"EuroEval French","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":53.05624999999999,"unit":"percent","family":"euroeval","index":44.20535881318301,"aboveFit":false,"run":"no-thinking effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/french","measuredAt":null,"stderr":0.47248181181924875,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":52.98875,"unit":"percent","family":"euroeval","index":44.113776844490545,"aboveFit":false,"run":"no-thinking effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":0.6063865825527475,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-dutch:overall","benchmarkName":"EuroEval Dutch","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":52.39444444444445,"unit":"percent","family":"euroeval","index":43.30744095141416,"aboveFit":false,"run":"no-thinking effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/dutch","measuredAt":null,"stderr":0.504362450507055,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-portuguese:overall","benchmarkName":"EuroEval Portuguese","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":49.448750000000004,"unit":"percent","family":"euroeval","index":39.310811375284835,"aboveFit":false,"run":"no-thinking effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/portuguese","measuredAt":null,"stderr":0.579688763475712,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-italian:overall","benchmarkName":"EuroEval Italian","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":47.9575,"unit":"percent","family":"euroeval","index":37.287528252134265,"aboveFit":false,"run":"no-thinking effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/italian","measuredAt":null,"stderr":0.4459960762159237,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-spanish:overall","benchmarkName":"EuroEval Spanish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":44.6425,"unit":"percent","family":"euroeval","index":32.78983601190348,"aboveFit":false,"run":"no-thinking effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/spanish","measuredAt":null,"stderr":0.4420336525650507,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":41.638749999999995,"unit":"percent","family":"euroeval","index":28.714438405088035,"aboveFit":false,"run":"no-thinking effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.7239302452584779,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":33.269999999999996,"unit":"percent","family":"euroeval","index":17.359970249603485,"aboveFit":false,"run":"no-thinking effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.350148656477007,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gemini25flashlitepreview092025","providerId":"google","slug":"google-gemini-2-5-flash-lite-preview","name":"Gemini 2.5 Flash Lite Preview","provider":"Google","description":"Gemini 2.5 Flash Lite Preview is a model from Google. It has 5 scored results in 4 categories. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":45.49152572888513,"reasoning":null,"multimodal":49.56116497637529,"knowledge":39.7974230165822,"multilingual":null,"instruction":null,"math":26.975017201607834},"capabilityCounts":{"knowledge":2,"multimodal":1,"coding":1,"math":1},"benchmarks":[{"benchmarkId":"vals-ai:mgsm:overall","benchmarkName":"MGSM","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":89.527,"unit":"percent","family":"vals-ai:mgsm:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mgsm","measuredAt":"2026-01-09","stderr":1.894,"costPerTest":0.000212,"latency":1.465},{"benchmarkId":"vals-ai:medqa:overall","benchmarkName":"MedQA","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":80.325,"unit":"percent","family":"vals-ai:medqa:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medqa","measuredAt":"2026-04-16","stderr":0.363,"costPerTest":0.000281,"latency":5.145},{"benchmarkId":"vals-ai:legal_bench:overall","benchmarkName":"LegalBench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":79.008,"unit":"percent","family":"vals-ai:legal_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","measuredAt":"2026-10-13","stderr":0.443,"costPerTest":null,"latency":0.456},{"benchmarkId":"vals-ai:mmlu_pro:overall","benchmarkName":"MMLU Pro","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":78.639,"unit":"percent","family":"mmlu-pro","index":43.55645558061002,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","measuredAt":"2026-09-01","stderr":0.4,"costPerTest":0.000556,"latency":3.19},{"benchmarkId":"vals-ai:medscribe:overall","benchmarkName":"MedScribe","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":75.824,"unit":"percent","family":"vals-ai:medscribe:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medscribe","measuredAt":"2026-10-29","stderr":1.851,"costPerTest":0.001332,"latency":4.491},{"benchmarkId":"vals-ai:mmmu:overall","benchmarkName":"MMMU Pro","category":"multimodal","domain":null,"eligible":true,"ranked":true,"score":72.543,"unit":"percent","family":"mmmu-pro","index":49.56116497637529,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmmu","measuredAt":"2026-09-01","stderr":1.073,"costPerTest":0.000899,"latency":20.069},{"benchmarkId":"vals-ai:lcb:overall","benchmarkName":"LiveCodeBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":67.669,"unit":"percent","family":"livecodebench","index":45.49152572888513,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","measuredAt":"2026-09-01","stderr":1.172,"costPerTest":0.002003,"latency":17.442},{"benchmarkId":"vals-ai:tax_eval_v2:overall","benchmarkName":"TaxEval v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":66.231,"unit":"percent","family":"vals-ai:tax_eval_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","measuredAt":"2026-09-01","stderr":0.923,"costPerTest":0.000477,"latency":3.597},{"benchmarkId":"vals-ai:gpqa:overall","benchmarkName":"GPQA Diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":64.142,"unit":"percent","family":"gpqa-diamond","index":36.03839045255438,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","measuredAt":"2026-09-01","stderr":2.412,"costPerTest":0.001236,"latency":7.274},{"benchmarkId":"vals-ai:corp_fin_v2:overall","benchmarkName":"CorpFin v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":56.294,"unit":"percent","family":"vals-ai:corp_fin_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","measuredAt":"2026-08-12","stderr":0.976,"costPerTest":0.008646,"latency":54.75},{"benchmarkId":"vals-ai:mortgage_tax:overall","benchmarkName":"MortgageTax","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":55.803,"unit":"percent","family":"vals-ai:mortgage_tax:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mortgage_tax","measuredAt":"2026-09-01","stderr":0.99,"costPerTest":0.000479,"latency":8.042},{"benchmarkId":"vals-ai:medcode:overall","benchmarkName":"MedCode","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":27.079,"unit":"percent","family":"vals-ai:medcode:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medcode","measuredAt":"2026-10-29","stderr":1.911,"costPerTest":0.00144,"latency":6.239},{"benchmarkId":"vals-ai:aime:overall","benchmarkName":"AIME","category":"math","domain":null,"eligible":true,"ranked":true,"score":26.25,"unit":"percent","family":"vals-ai:aime:overall","index":26.975017201607834,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/aime","measuredAt":"2026-04-16","stderr":1.989,"costPerTest":0.003224,"latency":18.704}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":55.00000000000001,"benchmarkCount":5,"categoryCount":4,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gemini25flashlitepreview092025thinking","providerId":"google","slug":"google-gemini-2-5-flash-lite-preview-thinking","name":"Gemini 2.5 Flash Lite Preview Thinking","provider":"Google","description":"Gemini 2.5 Flash Lite Preview Thinking is a model from Google. It has 5 scored results in 4 categories. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":48.574844135859315,"reasoning":null,"multimodal":53.32354783483478,"knowledge":42.87819755417606,"multilingual":null,"instruction":null,"math":33.89476272657105},"capabilityCounts":{"knowledge":2,"multimodal":1,"coding":1,"math":1},"benchmarks":[{"benchmarkId":"vals-ai:medqa:overall","benchmarkName":"MedQA","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":88.867,"unit":"percent","family":"vals-ai:medqa:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medqa","measuredAt":"2026-04-16","stderr":0.287,"costPerTest":0.000167,"latency":14.822},{"benchmarkId":"vals-ai:mgsm:overall","benchmarkName":"MGSM","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":88.4,"unit":"percent","family":"vals-ai:mgsm:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mgsm","measuredAt":"2026-01-09","stderr":1.928,"costPerTest":0.000147,"latency":4.568},{"benchmarkId":"vals-ai:legal_bench:overall","benchmarkName":"LegalBench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":82.044,"unit":"percent","family":"vals-ai:legal_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","measuredAt":"2026-10-13","stderr":0.438,"costPerTest":null,"latency":3.139},{"benchmarkId":"vals-ai:mmlu_pro:overall","benchmarkName":"MMLU Pro","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":79.118,"unit":"percent","family":"mmlu-pro","index":44.15993824578384,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","measuredAt":"2026-09-01","stderr":0.397,"costPerTest":0.000314,"latency":9.618},{"benchmarkId":"vals-ai:mmmu:overall","benchmarkName":"MMMU Pro","category":"multimodal","domain":null,"eligible":true,"ranked":true,"score":75.434,"unit":"percent","family":"mmmu-pro","index":53.32354783483478,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmmu","measuredAt":"2026-09-01","stderr":1.035,"costPerTest":0.000356,"latency":25.728},{"benchmarkId":"vals-ai:lcb:overall","benchmarkName":"LiveCodeBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":71.385,"unit":"percent","family":"livecodebench","index":48.574844135859315,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","measuredAt":"2026-09-01","stderr":1.138,"costPerTest":0.000408,"latency":38.697},{"benchmarkId":"vals-ai:gpqa:overall","benchmarkName":"GPQA Diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":70.202,"unit":"percent","family":"gpqa-diamond","index":41.596456862568274,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","measuredAt":"2026-09-01","stderr":2.298,"costPerTest":0.000481,"latency":20.61},{"benchmarkId":"vals-ai:medscribe:overall","benchmarkName":"MedScribe","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":66.877,"unit":"percent","family":"vals-ai:medscribe:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medscribe","measuredAt":"2026-10-29","stderr":1.923,"costPerTest":0.002567,"latency":11.518},{"benchmarkId":"vals-ai:tax_eval_v2:overall","benchmarkName":"TaxEval v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":64.718,"unit":"percent","family":"vals-ai:tax_eval_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","measuredAt":"2026-09-01","stderr":0.931,"costPerTest":0.000509,"latency":3.94},{"benchmarkId":"vals-ai:corp_fin_v2:overall","benchmarkName":"CorpFin v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":57.576,"unit":"percent","family":"vals-ai:corp_fin_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","measuredAt":"2026-08-12","stderr":0.973,"costPerTest":0.008596,"latency":54.225},{"benchmarkId":"vals-ai:mortgage_tax:overall","benchmarkName":"MortgageTax","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":57.552,"unit":"percent","family":"vals-ai:mortgage_tax:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mortgage_tax","measuredAt":"2026-09-01","stderr":0.985,"costPerTest":0.000286,"latency":28.206},{"benchmarkId":"vals-ai:aime:overall","benchmarkName":"AIME","category":"math","domain":null,"eligible":true,"ranked":true,"score":42.083,"unit":"percent","family":"vals-ai:aime:overall","index":33.89476272657105,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/aime","measuredAt":"2026-04-16","stderr":1.567,"costPerTest":0.000566,"latency":51.146},{"benchmarkId":"vals-ai:medcode:overall","benchmarkName":"MedCode","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":34.191,"unit":"percent","family":"vals-ai:medcode:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medcode","measuredAt":"2026-10-29","stderr":1.736,"costPerTest":0.001182,"latency":10.49}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":55.00000000000001,"benchmarkCount":5,"categoryCount":4,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gemini25flashlitethinking","providerId":"google","slug":"google-gemini-2-5-flash-lite-thinking","name":"Gemini 2.5 Flash Lite Thinking","provider":"Google","description":"Gemini 2.5 Flash Lite Thinking is a model from Google. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":48.092141174953234,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":58.584999999999994,"unit":"percent","family":"euroeval","index":51.70660043405056,"aboveFit":false,"run":"thinking effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":0.70475727204478,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-french:overall","benchmarkName":"EuroEval French","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":58.42000000000001,"unit":"percent","family":"euroeval","index":51.482733399468955,"aboveFit":false,"run":"thinking effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/french","measuredAt":null,"stderr":0.40918554776531396,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-dutch:overall","benchmarkName":"EuroEval Dutch","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":57.37444444444444,"unit":"percent","family":"euroeval","index":50.06415508605947,"aboveFit":false,"run":"thinking effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/dutch","measuredAt":null,"stderr":0.5169951166280818,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":56.445714285714295,"unit":"percent","family":"euroeval","index":48.80408195538409,"aboveFit":false,"run":"thinking effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.504338321820921,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-italian:overall","benchmarkName":"EuroEval Italian","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":55.396249999999995,"unit":"percent","family":"euroeval","index":47.38020039452239,"aboveFit":false,"run":"thinking effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/italian","measuredAt":null,"stderr":0.43237172953374275,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-spanish:overall","benchmarkName":"EuroEval Spanish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":54.30625,"unit":"percent","family":"euroeval","index":45.90132119637713,"aboveFit":false,"run":"thinking effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/spanish","measuredAt":null,"stderr":0.5290321351298047,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-portuguese:overall","benchmarkName":"EuroEval Portuguese","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":54.0375,"unit":"percent","family":"euroeval","index":45.5366892839904,"aboveFit":false,"run":"thinking effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/portuguese","measuredAt":null,"stderr":0.4833153732295301,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":49.28125,"unit":"percent","family":"euroeval","index":39.08355241593682,"aboveFit":false,"run":"thinking effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.6338572394474957,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gemini-2-5-flash-native-audio-preview-12-2025","providerId":"google","slug":"gemini-2-5-flash-native-audio-preview-12-2025","name":"Gemini 2.5 Flash Native Audio Preview (12-2025)","provider":"Google","description":"Gemini 2.5 Flash Native Audio Preview (12-2025) is a standard model from Google. It has no scored result.","logoUrl":null,"releaseDate":"2025-12-12","contextWindowTokens":null,"modalities":["text"],"openWeights":false,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gemini25flashpreview0417","providerId":"google","slug":"google-gemini-2-5-flash-preview-04-17","name":"Gemini 2.5 Flash Preview 04.17","provider":"Google","description":"Gemini 2.5 Flash Preview 04.17 is a model from Google. It has 4 scored results in 4 categories. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":36.58591311799873,"reasoning":null,"multimodal":43.64234199461332,"knowledge":30.016234669031075,"multilingual":null,"instruction":null,"math":40.92583551495429},"capabilityCounts":{"math":1,"multimodal":1,"knowledge":1,"coding":1},"benchmarks":[{"benchmarkId":"vals-ai:math500:overall","benchmarkName":"MATH 500","category":"math","domain":null,"eligible":true,"ranked":true,"score":91.6,"unit":"percent","family":"math-500","index":40.92583551495429,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/math500","measuredAt":"2026-01-09","stderr":1.24,"costPerTest":0.006461,"latency":9.496},{"benchmarkId":"vals-ai:medqa:overall","benchmarkName":"MedQA","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":86.733,"unit":"percent","family":"vals-ai:medqa:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medqa","measuredAt":"2026-04-16","stderr":0.31,"costPerTest":0.001068,"latency":2.254},{"benchmarkId":"vals-ai:legal_bench:overall","benchmarkName":"LegalBench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":83.796,"unit":"percent","family":"vals-ai:legal_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","measuredAt":"2026-10-13","stderr":0.399,"costPerTest":null,"latency":0.545},{"benchmarkId":"vals-ai:tax_eval_v2:overall","benchmarkName":"TaxEval v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":71.178,"unit":"percent","family":"vals-ai:tax_eval_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","measuredAt":"2026-09-01","stderr":0.885,"costPerTest":0.00261,"latency":5.388},{"benchmarkId":"vals-ai:mmmu:overall","benchmarkName":"MMMU Pro","category":"multimodal","domain":null,"eligible":true,"ranked":true,"score":67.995,"unit":"percent","family":"mmmu-pro","index":43.64234199461332,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmmu","measuredAt":"2026-09-01","stderr":1.121,"costPerTest":0.02536,"latency":37.103},{"benchmarkId":"vals-ai:mortgage_tax:overall","benchmarkName":"MortgageTax","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":58.426,"unit":"percent","family":"vals-ai:mortgage_tax:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mortgage_tax","measuredAt":"2026-09-01","stderr":0.98,"costPerTest":0.001974,"latency":3.556},{"benchmarkId":"vals-ai:gpqa:overall","benchmarkName":"GPQA Diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":57.576,"unit":"percent","family":"gpqa-diamond","index":30.016234669031075,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","measuredAt":"2026-09-01","stderr":2.411,"costPerTest":0.030117,"latency":43.452},{"benchmarkId":"vals-ai:lcb:overall","benchmarkName":"LiveCodeBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":56.936,"unit":"percent","family":"livecodebench","index":36.58591311799873,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","measuredAt":"2026-09-01","stderr":1.176,"costPerTest":0.011721,"latency":16.407},{"benchmarkId":"vals-ai:corp_fin_v2:overall","benchmarkName":"CorpFin v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":54.157,"unit":"percent","family":"vals-ai:corp_fin_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","measuredAt":"2026-08-12","stderr":0.982,"costPerTest":0.020528,"latency":8.992}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":55.00000000000001,"benchmarkCount":4,"categoryCount":4,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gemini25flashpreview0417thinking","providerId":"google","slug":"google-gemini-2-5-flash-preview-04-17-thinking","name":"Gemini 2.5 Flash Preview 04.17 Thinking","provider":"Google","description":"Gemini 2.5 Flash Preview 04.17 Thinking is a model from Google. It has 3 scored results in 3 categories. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":28.234567648086163,"reasoning":null,"multimodal":48.75559078426652,"knowledge":null,"multilingual":null,"instruction":null,"math":41.137187157440266},"capabilityCounts":{"math":1,"multimodal":1,"coding":1},"benchmarks":[{"benchmarkId":"vals-ai:math500:overall","benchmarkName":"MATH 500","category":"math","domain":null,"eligible":true,"ranked":true,"score":91.8,"unit":"percent","family":"math-500","index":41.137187157440266,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/math500","measuredAt":"2026-01-09","stderr":1.227,"costPerTest":0.011241,"latency":23.657},{"benchmarkId":"vals-ai:medqa:overall","benchmarkName":"MedQA","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":91.017,"unit":"percent","family":"vals-ai:medqa:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medqa","measuredAt":"2026-04-16","stderr":0.261,"costPerTest":0.003784,"latency":8.867},{"benchmarkId":"vals-ai:mmmu:overall","benchmarkName":"MMMU Pro","category":"multimodal","domain":null,"eligible":true,"ranked":true,"score":71.924,"unit":"percent","family":"mmmu-pro","index":48.75559078426652,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmmu","measuredAt":"2026-09-01","stderr":1.08,"costPerTest":0.002641,"latency":471.866},{"benchmarkId":"vals-ai:tax_eval_v2:overall","benchmarkName":"TaxEval v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":70.524,"unit":"percent","family":"vals-ai:tax_eval_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","measuredAt":"2026-09-01","stderr":0.891,"costPerTest":0.007461,"latency":15.355},{"benchmarkId":"vals-ai:mortgage_tax:overall","benchmarkName":"MortgageTax","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":57.512,"unit":"percent","family":"vals-ai:mortgage_tax:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mortgage_tax","measuredAt":"2026-09-01","stderr":0.986,"costPerTest":0.0041,"latency":8.4},{"benchmarkId":"vals-ai:lcb:overall","benchmarkName":"LiveCodeBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":46.871,"unit":"percent","family":"livecodebench","index":28.234567648086163,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","measuredAt":"2026-09-01","stderr":1.441,"costPerTest":0.001987,"latency":124.947}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":40,"benchmarkCount":3,"categoryCount":3,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gemini25flashpreview092025thinking","providerId":"google","slug":"google-gemini-2-5-flash-preview-thinking","name":"Gemini 2.5 Flash Preview Thinking","provider":"Google","description":"Gemini 2.5 Flash Preview Thinking is a model from Google. It has 7 scored results in 5 categories. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":38.30985755088647,"coding":52.58166453071354,"reasoning":null,"multimodal":60.24315684847339,"knowledge":48.63741961779156,"multilingual":null,"instruction":null,"math":37.99206673064673},"capabilityCounts":{"knowledge":2,"multimodal":1,"coding":1,"math":1,"agentic":2},"benchmarks":[{"benchmarkId":"vals-ai:medqa:overall","benchmarkName":"MedQA","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":91.167,"unit":"percent","family":"vals-ai:medqa:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medqa","measuredAt":"2026-04-16","stderr":0.259,"costPerTest":0.001265,"latency":18.007},{"benchmarkId":"vals-ai:mgsm:overall","benchmarkName":"MGSM","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":89.818,"unit":"percent","family":"vals-ai:mgsm:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mgsm","measuredAt":"2026-01-09","stderr":1.787,"costPerTest":0.001019,"latency":5.733},{"benchmarkId":"vals-ai:mmlu_pro:overall","benchmarkName":"MMLU Pro","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":83.664,"unit":"percent","family":"mmlu-pro","index":49.88735410357131,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","measuredAt":"2026-09-01","stderr":0.361,"costPerTest":0.00146,"latency":10.445},{"benchmarkId":"vals-ai:legal_bench:overall","benchmarkName":"LegalBench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":82.625,"unit":"percent","family":"vals-ai:legal_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","measuredAt":"2026-10-13","stderr":0.414,"costPerTest":null,"latency":1.965},{"benchmarkId":"vals-ai:mmmu:overall","benchmarkName":"MMMU Pro","category":"multimodal","domain":null,"eligible":true,"ranked":true,"score":80.751,"unit":"percent","family":"mmmu-pro","index":60.24315684847339,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmmu","measuredAt":"2026-09-01","stderr":0.948,"costPerTest":0.001858,"latency":27.866},{"benchmarkId":"vals-ai:medscribe:overall","benchmarkName":"MedScribe","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":78.497,"unit":"percent","family":"vals-ai:medscribe:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medscribe","measuredAt":"2026-10-29","stderr":1.993,"costPerTest":0.014526,"latency":31.385},{"benchmarkId":"vals-ai:gpqa:overall","benchmarkName":"GPQA Diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":76.516,"unit":"percent","family":"gpqa-diamond","index":47.387485132011804,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","measuredAt":"2026-09-01","stderr":2.154,"costPerTest":0.002833,"latency":23.53},{"benchmarkId":"vals-ai:lcb:overall","benchmarkName":"LiveCodeBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":76.214,"unit":"percent","family":"livecodebench","index":52.58166453071354,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","measuredAt":"2026-09-01","stderr":1.136,"costPerTest":0.002383,"latency":48.671},{"benchmarkId":"vals-ai:tax_eval_v2:overall","benchmarkName":"TaxEval v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":72.404,"unit":"percent","family":"vals-ai:tax_eval_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","measuredAt":"2026-09-01","stderr":0.872,"costPerTest":0.003099,"latency":10.635},{"benchmarkId":"vals-ai:mortgage_tax:overall","benchmarkName":"MortgageTax","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":61.924,"unit":"percent","family":"vals-ai:mortgage_tax:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mortgage_tax","measuredAt":"2026-09-01","stderr":0.958,"costPerTest":0.001154,"latency":18.331},{"benchmarkId":"vals-ai:corp_fin_v2:overall","benchmarkName":"CorpFin v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":59.751,"unit":"percent","family":"vals-ai:corp_fin_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","measuredAt":"2026-08-12","stderr":0.966,"costPerTest":0.026119,"latency":51.757},{"benchmarkId":"vals-ai:aime:overall","benchmarkName":"AIME","category":"math","domain":null,"eligible":true,"ranked":true,"score":51.458,"unit":"percent","family":"vals-ai:aime:overall","index":37.99206673064673,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/aime","measuredAt":"2026-04-16","stderr":1.94,"costPerTest":0.004986,"latency":61.601},{"benchmarkId":"vals-ai:medcode:overall","benchmarkName":"MedCode","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":40.33,"unit":"percent","family":"vals-ai:medcode:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medcode","measuredAt":"2026-10-29","stderr":1.915,"costPerTest":0.003653,"latency":16.106},{"benchmarkId":"vals-ai:terminal-bench:overall","benchmarkName":"Terminal-Bench 1.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":26.25,"unit":"percent","family":"vals-ai:terminal-bench:overall","index":37.214579501211844,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench","measuredAt":"2026-01-12","stderr":4.919,"costPerTest":0.132934,"latency":95.3},{"benchmarkId":"vals-ai:terminal-bench-2:overall","benchmarkName":"Terminal-Bench 2.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":21.348,"unit":"percent","family":"vals-ai:terminal-bench-2:overall","index":39.405135600561096,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2","measuredAt":"2026-06-04","stderr":4.368,"costPerTest":0.143966,"latency":417.621}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":75,"benchmarkCount":7,"categoryCount":5,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gemini25flashthinking","providerId":"google","slug":"google-gemini-2-5-flash-thinking","name":"Gemini 2.5 Flash Thinking","provider":"Google","description":"Gemini 2.5 Flash Thinking is a model from Google. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":33.59616319989902,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"agentic":1},"benchmarks":[{"benchmarkId":"vals-ai:medscribe:overall","benchmarkName":"MedScribe","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":82.983,"unit":"percent","family":"vals-ai:medscribe:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medscribe","measuredAt":"2026-10-29","stderr":1.908,"costPerTest":0.014824,"latency":22.529},{"benchmarkId":"vals-ai:medcode:overall","benchmarkName":"MedCode","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":40.357,"unit":"percent","family":"vals-ai:medcode:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medcode","measuredAt":"2026-10-29","stderr":1.952,"costPerTest":0.003661,"latency":22.833},{"benchmarkId":"vals-ai:terminal-bench:overall","benchmarkName":"Terminal-Bench 1.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":21.25,"unit":"percent","family":"vals-ai:terminal-bench:overall","index":33.59616319989902,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench","measuredAt":"2026-01-12","stderr":4.574,"costPerTest":0.447777,"latency":244.663}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":20,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gemini25projun2025","providerId":"google","slug":"google-gemini-2-5-pro-jun-2025","name":"Gemini 2.5 Pro (Jun 2025)","provider":"Google","description":"Gemini 2.5 Pro (Jun 2025) is a model from Google. It has 7 scored results in 3 categories. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":43.67948169590361,"reasoning":null,"multimodal":null,"knowledge":55.23247065290069,"multilingual":null,"instruction":null,"math":47.52357362180221},"capabilityCounts":{"knowledge":1,"math":5,"coding":1},"benchmarks":[{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":85.06944444444444,"unit":"percent","family":"gpqa-diamond","index":55.23247065290069,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.117010842348584,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":84.72222222222221,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":58.20997530672897,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":4.814319605471135,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-swe-bench-verified:overall","benchmarkName":"SWE-Bench verified","category":"coding","domain":null,"eligible":true,"ranked":true,"score":57.55693581780539,"unit":"percent","family":"swe-bench-verified","index":43.67948169590361,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.251275673889269,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-2025-02-28-public:overall","benchmarkName":"FrontierMath-2025-02-28-Public","category":"math","domain":null,"eligible":false,"ranked":true,"score":35,"unit":"percent","family":"epoch-ai:epoch-frontiermath-2025-02-28-public:overall","index":null,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":16.32993161855452,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tiers-1-3-v2-private:overall","benchmarkName":"FrontierMath-Tiers-1-3-v2-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":24.561403508771928,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tiers-1-3-v2-private:overall","index":43.822633074063226,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.55425481026507,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-2025-02-28-private:overall","benchmarkName":"FrontierMath-2025-02-28-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":12.24137931034483,"unit":"percent","family":"epoch-ai:epoch-frontiermath-2025-02-28-private:overall","index":40.72058912814187,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.0494847099131452,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-private:overall","benchmarkName":"FrontierMath-Tier-4-2025-07-01-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":3.1249999999999996,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-private:overall","index":46.37636610282637,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.914766351555603,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tier-4-v2-private:overall","benchmarkName":"FrontierMath-Tier-4-v2-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":0,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tier-4-v2-private:overall","index":48.48830449725058,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-public:overall","benchmarkName":"FrontierMath-Tier-4-2025-07-01-Public","category":"math","domain":null,"eligible":false,"ranked":true,"score":0,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-public:overall","index":null,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":45,"benchmarkCount":7,"categoryCount":3,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gemini25promar2025","providerId":"google","slug":"google-gemini-2-5-pro-mar-2025","name":"Gemini 2.5 Pro (Mar 2025)","provider":"Google","description":"Gemini 2.5 Pro (Mar 2025) is a model from Google. It has 2 scored results in 2 categories. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":54.10337550385171,"multilingual":null,"instruction":null,"math":45.418001139180475},"capabilityCounts":{"math":1,"knowledge":1},"benchmarks":[{"benchmarkId":"epoch-ai:epoch-math-level-5:overall","benchmarkName":"MATH level 5","category":"math","domain":null,"eligible":true,"ranked":true,"score":95.56268882175226,"unit":"percent","family":"epoch-ai:epoch-math-level-5:overall","index":45.418001139180475,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0.4475169260814502,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":83.83838383838383,"unit":"percent","family":"gpqa-diamond","index":54.10337550385171,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.622591986362927,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":25,"benchmarkCount":2,"categoryCount":2,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gemini25promay2025","providerId":"google","slug":"google-gemini-2-5-pro-may-2025","name":"Gemini 2.5 Pro (May 2025)","provider":"Google","description":"Gemini 2.5 Pro (May 2025) is a model from Google. It has 2 scored results in 2 categories. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":38.353945732501856,"multilingual":null,"instruction":null,"math":45.58309926621091},"capabilityCounts":{"math":1,"knowledge":1},"benchmarks":[{"benchmarkId":"epoch-ai:epoch-math-level-5:overall","benchmarkName":"MATH level 5","category":"math","domain":null,"eligible":true,"ranked":true,"score":95.9025679758308,"unit":"percent","family":"epoch-ai:epoch-math-level-5:overall","index":45.58309926621091,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0.4221388011004008,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":66.66666666666666,"unit":"percent","family":"gpqa-diamond","index":38.353945732501856,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":3.358618145732523,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":25,"benchmarkCount":2,"categoryCount":2,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gemini25prothinking16k","providerId":"google","slug":"google-gemini-2-5-pro-thinking-16k","name":"Gemini 2.5 Pro (Thinking 16K)","provider":"Google","description":"Gemini 2.5 Pro (Thinking 16K) is a model from Google. It has 2 scored results in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":43.437683664658756,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"reasoning":2},"benchmarks":[{"benchmarkId":"arc-prize:v1-public-eval:overall","benchmarkName":"ARC-AGI-1 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":56.43,"unit":"percent","family":"arc-prize:v1-public-eval:overall","index":null,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.3964,"latency":null},{"benchmarkId":"arc-prize:v1-semi-private:overall","benchmarkName":"ARC-AGI-1 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":41,"unit":"percent","family":"arc-prize:v1-semi-private:overall","index":45.83653908214106,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.4839,"latency":null},{"benchmarkId":"arc-prize:v2-public-eval:overall","benchmarkName":"ARC-AGI-2 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":5.140000000000001,"unit":"percent","family":"arc-prize:v2-public-eval:overall","index":null,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.7222,"latency":null},{"benchmarkId":"arc-prize:v2-semi-private:overall","benchmarkName":"ARC-AGI-2 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":4.03,"unit":"percent","family":"arc-prize:v2-semi-private:overall","index":41.038828247176454,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.7145,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":15,"benchmarkCount":2,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:arc-prize-foundation","name":"ARC Prize Foundation","url":"https://arcprize.org","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gemini25prothinking1k","providerId":"google","slug":"google-gemini-2-5-pro-thinking-1k","name":"Gemini 2.5 Pro (Thinking 1K)","provider":"Google","description":"Gemini 2.5 Pro (Thinking 1K) is a model from Google. It has 2 scored results in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":36.76582595501185,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"reasoning":2},"benchmarks":[{"benchmarkId":"arc-prize:v1-public-eval:overall","benchmarkName":"ARC-AGI-1 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":17.5,"unit":"percent","family":"arc-prize:v1-public-eval:overall","index":null,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.048,"latency":null},{"benchmarkId":"arc-prize:v1-semi-private:overall","benchmarkName":"ARC-AGI-1 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":16,"unit":"percent","family":"arc-prize:v1-semi-private:overall","index":34.34313650778115,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.0573,"latency":null},{"benchmarkId":"arc-prize:v2-public-eval:overall","benchmarkName":"ARC-AGI-2 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":0.42,"unit":"percent","family":"arc-prize:v2-public-eval:overall","index":null,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.0879,"latency":null},{"benchmarkId":"arc-prize:v2-semi-private:overall","benchmarkName":"ARC-AGI-2 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":0,"unit":"percent","family":"arc-prize:v2-semi-private:overall","index":39.188515402242544,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.0885,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":15,"benchmarkCount":2,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:arc-prize-foundation","name":"ARC Prize Foundation","url":"https://arcprize.org","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gemini25prothinking32k","providerId":"google","slug":"google-gemini-2-5-pro-thinking-32k","name":"Gemini 2.5 Pro (Thinking 32K)","provider":"Google","description":"Gemini 2.5 Pro (Thinking 32K) is a model from Google. It has 2 scored results in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":42.70875235961507,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"reasoning":2},"benchmarks":[{"benchmarkId":"arc-prize:v1-public-eval:overall","benchmarkName":"ARC-AGI-1 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":55.879999999999995,"unit":"percent","family":"arc-prize:v1-public-eval:overall","index":null,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.4164,"latency":null},{"benchmarkId":"arc-prize:v1-semi-private:overall","benchmarkName":"ARC-AGI-1 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":37,"unit":"percent","family":"arc-prize:v1-semi-private:overall","index":43.99759467024347,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.5123,"latency":null},{"benchmarkId":"arc-prize:v2-semi-private:overall","benchmarkName":"ARC-AGI-2 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":4.859999999999999,"unit":"percent","family":"arc-prize:v2-semi-private:overall","index":41.419910048986665,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.757,"latency":null},{"benchmarkId":"arc-prize:v2-public-eval:overall","benchmarkName":"ARC-AGI-2 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":4.58,"unit":"percent","family":"arc-prize:v2-public-eval:overall","index":null,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.7628,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":15,"benchmarkCount":2,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:arc-prize-foundation","name":"ARC Prize Foundation","url":"https://arcprize.org","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gemini25prothinking8k","providerId":"google","slug":"google-gemini-2-5-pro-thinking-8k","name":"Gemini 2.5 Pro (Thinking 8K)","provider":"Google","description":"Gemini 2.5 Pro (Thinking 8K) is a model from Google. It has 2 scored results in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":40.53938131351421,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"reasoning":2},"benchmarks":[{"benchmarkId":"arc-prize:v1-public-eval:overall","benchmarkName":"ARC-AGI-1 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":44.190000000000005,"unit":"percent","family":"arc-prize:v1-public-eval:overall","index":null,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.2605,"latency":null},{"benchmarkId":"arc-prize:v1-semi-private:overall","benchmarkName":"ARC-AGI-1 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":29.5,"unit":"percent","family":"arc-prize:v1-semi-private:overall","index":40.5495738979355,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.2947,"latency":null},{"benchmarkId":"arc-prize:v2-semi-private:overall","benchmarkName":"ARC-AGI-2 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":2.92,"unit":"percent","family":"arc-prize:v2-semi-private:overall","index":40.52918872909292,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.4439,"latency":null},{"benchmarkId":"arc-prize:v2-public-eval:overall","benchmarkName":"ARC-AGI-2 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":2.64,"unit":"percent","family":"arc-prize:v2-public-eval:overall","index":null,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.4493,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":15,"benchmarkCount":2,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:arc-prize-foundation","name":"ARC Prize Foundation","url":"https://arcprize.org","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gemini25proexp0325","providerId":"google","slug":"google-gemini-2-5-pro-exp-03-25","name":"Gemini 2.5 Pro Exp 03.25","provider":"Google","description":"Gemini 2.5 Pro Exp 03.25 is a model from Google. It has 5 scored results in 3 categories. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":61.00968867262858,"knowledge":50.85386859787557,"multilingual":null,"instruction":null,"math":48.87283991264637},"capabilityCounts":{"math":2,"knowledge":2,"multimodal":1},"benchmarks":[{"benchmarkId":"vals-ai:math500:overall","benchmarkName":"MATH 500","category":"math","domain":null,"eligible":true,"ranked":true,"score":95.2,"unit":"percent","family":"math-500","index":44.73016507970186,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/math500","measuredAt":"2026-01-09","stderr":0.956,"costPerTest":0.007941,"latency":25.828},{"benchmarkId":"vals-ai:medqa:overall","benchmarkName":"MedQA","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":93.142,"unit":"percent","family":"vals-ai:medqa:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medqa","measuredAt":"2026-04-16","stderr":0.231,"costPerTest":0.002896,"latency":10.704},{"benchmarkId":"vals-ai:aime:overall","benchmarkName":"AIME","category":"math","domain":null,"eligible":true,"ranked":true,"score":85.833,"unit":"percent","family":"vals-ai:aime:overall","index":53.01551474559088,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/aime","measuredAt":"2026-04-16","stderr":1.318,"costPerTest":0.027139,"latency":143.907},{"benchmarkId":"vals-ai:legal_bench:overall","benchmarkName":"LegalBench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":84.322,"unit":"percent","family":"vals-ai:legal_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","measuredAt":"2026-10-13","stderr":0.408,"costPerTest":null,"latency":4.022},{"benchmarkId":"vals-ai:mmlu_pro:overall","benchmarkName":"MMLU Pro","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":84.058,"unit":"percent","family":"mmlu-pro","index":50.383746942983606,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","measuredAt":"2026-09-01","stderr":0.358,"costPerTest":0.009006,"latency":18.46},{"benchmarkId":"vals-ai:mmmu:overall","benchmarkName":"MMMU Pro","category":"multimodal","domain":null,"eligible":true,"ranked":true,"score":81.34,"unit":"percent","family":"mmmu-pro","index":61.00968867262858,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmmu","measuredAt":"2026-09-01","stderr":0.936,"costPerTest":0.007771,"latency":24.145},{"benchmarkId":"vals-ai:gpqa:overall","benchmarkName":"GPQA Diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":80.808,"unit":"percent","family":"gpqa-diamond","index":51.32399025276752,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","measuredAt":"2026-09-01","stderr":1.998,"costPerTest":0.012944,"latency":41.102},{"benchmarkId":"vals-ai:tax_eval_v2:overall","benchmarkName":"TaxEval v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":72.894,"unit":"percent","family":"vals-ai:tax_eval_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","measuredAt":"2026-09-01","stderr":0.869,"costPerTest":0.010628,"latency":21.775},{"benchmarkId":"vals-ai:mortgage_tax:overall","benchmarkName":"MortgageTax","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":67.17,"unit":"percent","family":"vals-ai:mortgage_tax:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mortgage_tax","measuredAt":"2026-09-01","stderr":0.924,"costPerTest":0.003638,"latency":8.91},{"benchmarkId":"vals-ai:corp_fin_v2:overall","benchmarkName":"CorpFin v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":59.829,"unit":"percent","family":"vals-ai:corp_fin_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","measuredAt":"2026-08-12","stderr":0.966,"costPerTest":0.08579,"latency":16.302}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":35,"benchmarkCount":5,"categoryCount":3,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gemini25propreview0325","providerId":"google","slug":"google-gemini-2-5-pro-preview-03-25","name":"Gemini 2.5 Pro Preview 03.25","provider":"Google","description":"Gemini 2.5 Pro Preview 03.25 is a model from Google. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":55.02940115627162,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"coding":1},"benchmarks":[{"benchmarkId":"vals-ai:lcb:overall","benchmarkName":"LiveCodeBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":79.164,"unit":"percent","family":"livecodebench","index":55.02940115627162,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","measuredAt":"2026-09-01","stderr":1.086,"costPerTest":0.02004,"latency":164.657}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":20,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gemini25propreview","providerId":"google","slug":"google-gemini-2-5-pro-preview-06-05","name":"Gemini 2.5 Pro Preview 06-05","provider":"Google","description":"Gemini 2.5 Pro Preview 06-05 is a model from Google. It has no scored result.","logoUrl":null,"releaseDate":"2025-06-05T15:27:37.000Z","contextWindowTokens":1048576,"modalities":["file","image","text","audio"],"openWeights":null,"details":{"apiId":"google/gemini-2.5-pro-preview","maxOutputTokens":65536,"knowledgeCutoff":"2025-01-31","retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.125,"weightsUrl":null},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":1.25,"outputPerMillion":10,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":91,"timeToFirstToken":2.917,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":1.25,"outputPerMillion":10,"cacheReadPerMillion":0.125,"contextTokens":1048576,"maxOutputTokens":65536,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gemini3deepthink226","providerId":"google","slug":"google-gemini-3-deep-think-2-26","name":"Gemini 3 Deep Think (2/26)","provider":"Google","description":"Gemini 3 Deep Think (2/26) is a model from Google. It has 2 scored results in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":74.57207657826933,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"reasoning":2},"benchmarks":[{"benchmarkId":"arc-prize:v1-semi-private:overall","benchmarkName":"ARC-AGI-1 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":96,"unit":"percent","family":"arc-prize:v1-semi-private:overall","index":71.12202474573284,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":7.171,"latency":null},{"benchmarkId":"arc-prize:v2-semi-private:overall","benchmarkName":"ARC-AGI-2 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":84.58,"unit":"percent","family":"arc-prize:v2-semi-private:overall","index":78.02212841080582,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":13.6213,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":15,"benchmarkCount":2,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:arc-prize-foundation","name":"ARC Prize Foundation","url":"https://arcprize.org","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gemini3deepthinkpreview","providerId":"google","slug":"google-gemini-3-deep-think-preview","name":"Gemini 3 Deep Think (Preview) ²","provider":"Google","description":"Gemini 3 Deep Think (Preview) ² is a model from Google. It has 2 scored results in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":63.564061837378354,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"reasoning":2},"benchmarks":[{"benchmarkId":"arc-prize:v1-semi-private:overall","benchmarkName":"ARC-AGI-1 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":87.5,"unit":"percent","family":"arc-prize:v1-semi-private:overall","index":67.21426787045047,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":44.25900135,"latency":null},{"benchmarkId":"arc-prize:v2-semi-private:overall","benchmarkName":"ARC-AGI-2 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":45.14,"unit":"percent","family":"arc-prize:v2-semi-private:overall","index":59.91385580430623,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":77.16309638,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":15,"benchmarkCount":2,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:arc-prize-foundation","name":"ARC Prize Foundation","url":"https://arcprize.org","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gemini3flashpreviewnothinking","providerId":"google","slug":"google-gemini-3-flash-preview-no-thinking","name":"Gemini 3 Flash Preview No Thinking","provider":"Google","description":"Gemini 3 Flash Preview No Thinking is a model from Google. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":31.90539162906775,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":62.27857142857142,"unit":"percent","family":"euroeval","index":56.71792699604868,"aboveFit":false,"run":"no-thinking effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.5014285714285714,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-french:overall","benchmarkName":"EuroEval French","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":58.405,"unit":"percent","family":"euroeval","index":51.462381850870614,"aboveFit":false,"run":"no-thinking effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/french","measuredAt":null,"stderr":0.36733627305236277,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-italian:overall","benchmarkName":"EuroEval Italian","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":45.572500000000005,"unit":"percent","family":"euroeval","index":34.051632024999904,"aboveFit":false,"run":"no-thinking effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/italian","measuredAt":null,"stderr":0.5184622696011736,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":44.083749999999995,"unit":"percent","family":"euroeval","index":32.03174082661571,"aboveFit":false,"run":"no-thinking effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":0.6741754222752413,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-portuguese:overall","benchmarkName":"EuroEval Portuguese","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":43.89749999999999,"unit":"percent","family":"euroeval","index":31.779042431519777,"aboveFit":false,"run":"no-thinking effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/portuguese","measuredAt":null,"stderr":0.5016597452457193,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-dutch:overall","benchmarkName":"EuroEval Dutch","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":41.37222222222222,"unit":"percent","family":"euroeval","index":28.352821536938094,"aboveFit":false,"run":"no-thinking effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/dutch","measuredAt":null,"stderr":0.47562434601113357,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-spanish:overall","benchmarkName":"EuroEval Spanish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":40.9725,"unit":"percent","family":"euroeval","index":27.810490454845574,"aboveFit":false,"run":"no-thinking effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/spanish","measuredAt":null,"stderr":0.6001718503895364,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":37.83625,"unit":"percent","family":"euroeval","index":23.555320835411564,"aboveFit":false,"run":"no-thinking effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.5638109723125296,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gemini3flashpreviewthinking","providerId":"google","slug":"google-gemini-3-flash-preview-thinking","name":"Gemini 3 Flash Preview Thinking","provider":"Google","description":"Gemini 3 Flash Preview Thinking is a model from Google. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":66.3811976150348,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-french:overall","benchmarkName":"EuroEval French","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":74.3975,"unit":"percent","family":"euroeval","index":73.16052458145603,"aboveFit":false,"run":"thinking effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/french","measuredAt":null,"stderr":0.43324538370304655,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-italian:overall","benchmarkName":"EuroEval Italian","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":72.58125,"unit":"percent","family":"euroeval","index":70.69629123867499,"aboveFit":false,"run":"thinking effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/italian","measuredAt":null,"stderr":0.2935105406965821,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":70.93375,"unit":"percent","family":"euroeval","index":68.46101281762517,"aboveFit":false,"run":"thinking effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":0.5835157452888482,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-portuguese:overall","benchmarkName":"EuroEval Portuguese","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":69.445,"unit":"percent","family":"euroeval","index":66.44112161924097,"aboveFit":false,"run":"thinking effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/portuguese","measuredAt":null,"stderr":0.29748949561287036,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-dutch:overall","benchmarkName":"EuroEval Dutch","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":69.35666666666667,"unit":"percent","family":"euroeval","index":66.3212736108286,"aboveFit":false,"run":"thinking effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/dutch","measuredAt":null,"stderr":0.4341985663975969,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-spanish:overall","benchmarkName":"EuroEval Spanish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":66.53375,"unit":"percent","family":"euroeval","index":62.4912252287819,"aboveFit":false,"run":"thinking effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/spanish","measuredAt":null,"stderr":0.3676594042044893,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":62.10249999999999,"unit":"percent","family":"euroeval","index":56.479038580358775,"aboveFit":false,"run":"thinking effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.5389123885196925,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":61.59285714285714,"unit":"percent","family":"euroeval","index":55.7875704886965,"aboveFit":false,"run":"thinking effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.41011944502019587,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gemini3prorefinement","providerId":"google","slug":"google-gemini-3-pro-refinement","name":"Gemini 3 Pro (Refinement)","provider":"Google","description":"Gemini 3 Pro (Refinement) is a model from Google. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":63.981789254954975,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"reasoning":1},"benchmarks":[{"benchmarkId":"arc-prize:v2-semi-private:overall","benchmarkName":"ARC-AGI-2 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":54,"unit":"percent","family":"arc-prize:v2-semi-private:overall","index":63.981789254954975,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":30.568,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":15,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:arc-prize-foundation","name":"ARC Prize Foundation","url":"https://arcprize.org","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gemini-3-pro-deep-think","providerId":"google","slug":"gemini-3-pro-deep-think","name":"Gemini 3 Pro Deep Think","provider":"Google","description":"Gemini 3 Pro Deep Think is a reasoning model from Google in the Gemini 3 Pro family. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":"2026-02-12","contextWindowTokens":2000000,"modalities":["text"],"openWeights":false,"details":null,"reasoning":true,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":86.60131725193432,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"reasoning":1},"benchmarks":[{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":25.7,"unit":"percent","family":"critpt","index":86.60131725193432,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":15,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gemini30pro","providerId":"google","slug":"google-gemini-3-0-pro","name":"Gemini 3.0 Pro","provider":"Google","description":"Gemini 3.0 Pro is a model from Google. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":59.39222659225185,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"agentic":1},"benchmarks":[{"benchmarkId":"tau2-bench:telecom:overall","benchmarkName":"τ²-bench Telecom","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":98,"unit":"percent","family":"tau2-bench","index":67.81960511365625,"aboveFit":false,"run":"Sierra","source":"Sierra Research","sourceUrl":"https://github.com/sierra-research/tau2-bench","measuredAt":"2026-03-02","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"tau2-bench:retail:overall","benchmarkName":"τ²-bench Retail","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":85.3,"unit":"percent","family":"tau2-bench","index":59.39222659225185,"aboveFit":false,"run":"Sierra","source":"Sierra Research","sourceUrl":"https://github.com/sierra-research/tau2-bench","measuredAt":"2026-04-30","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"tau2-bench:airline:overall","benchmarkName":"τ²-bench Airline","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":73,"unit":"percent","family":"tau2-bench","index":51.2302773156161,"aboveFit":false,"run":"Sierra","source":"Sierra Research","sourceUrl":"https://github.com/sierra-research/tau2-bench","measuredAt":"2026-03-02","stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":20,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:sierra-research","name":"Sierra Research","url":"https://github.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gemini-3-1-flash-live-preview","providerId":"google","slug":"gemini-3-1-flash-live-preview","name":"Gemini 3.1 Flash Live Preview","provider":"Google","description":"Gemini 3.1 Flash Live Preview is a standard model from Google. It has no scored result.","logoUrl":null,"releaseDate":"2026-03-26","contextWindowTokens":null,"modalities":["text"],"openWeights":false,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gemini31propreviewcustomtools","providerId":"google","slug":"google-gemini-3-1-pro-preview-custom-tools","name":"Gemini 3.1 Pro Preview Custom Tools","provider":"Google","description":"Gemini 3.1 Pro Preview Custom Tools is a model from Google. It has no scored result.","logoUrl":null,"releaseDate":"2026-02-25T18:58:43.000Z","contextWindowTokens":1048576,"modalities":["text","audio","image","video","file"],"openWeights":null,"details":{"apiId":"google/gemini-3.1-pro-preview-customtools","maxOutputTokens":65536,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.19999999999999998,"weightsUrl":null},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":2,"outputPerMillion":12,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":80.5,"timeToFirstToken":3.542,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[{"host":"google-vertex","name":"Google Vertex AI","kind":"maker","url":"https://cloud.google.com/vertex-ai/generative-ai/docs/models","inputPerMillion":2,"outputPerMillion":12,"cacheReadPerMillion":0.2,"contextTokens":1048576,"maxOutputTokens":65536,"deprecated":false},{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":2,"outputPerMillion":12,"cacheReadPerMillion":0.2,"contextTokens":1048576,"maxOutputTokens":65536,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gemini-3-5-flash-cyber","providerId":"google","slug":"gemini-3-5-flash-cyber","name":"Gemini 3.5 Flash Cyber","provider":"Google","description":"Gemini 3.5 Flash Cyber is a reasoning model from Google in the Gemini 3.5 Flash family. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":"2026-07-21","contextWindowTokens":null,"modalities":["text"],"openWeights":false,"details":null,"reasoning":true,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":68.68461301423166,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"agentic":1},"benchmarks":[{"benchmarkId":"cyberGym","benchmarkName":"CyberGym","category":"agentic","score":83.2,"unit":"percent","family":"cyberGym","index":68.68461301423166,"aboveFit":false,"run":null,"source":"Zhun Wang, Tianneng Shi, Jingxuan He, Matthew Cai, Jialin Zhang, Dawn Song","sourceUrl":"https://www.cybergym.io/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":20,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gemini35flashexa","providerId":"google","slug":"google-gemini-3-5-flash-exa","name":"Gemini 3.5 Flash Exa","provider":"Google","description":"Gemini 3.5 Flash Exa is a model from Google. It has no scored result.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[{"benchmarkId":"vals-ai:web_search:overall","benchmarkName":"Vals Web Search Index","category":null,"domain":"Other","eligible":false,"ranked":true,"score":41.361000000000004,"unit":"percent","family":"vals-ai:web_search:overall","index":null,"aboveFit":false,"run":"Exa","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/web_search","measuredAt":"2026-11-04","stderr":1.930917139599729,"costPerTest":1.177103,"latency":197.4605}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gemini-3-8-flash-cyber","providerId":"google","slug":"gemini-3-8-flash-cyber","name":"Gemini 3.8 Flash Cyber","provider":"Google","description":"Gemini 3.8 Flash Cyber is a reasoning model from Google in the Gemini 3.8 Flash family. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":"2026-09-02","contextWindowTokens":null,"modalities":["text"],"openWeights":false,"details":null,"reasoning":true,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":70.53926643798528,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"agentic":1},"benchmarks":[{"benchmarkId":"cyberGym","benchmarkName":"CyberGym","category":"agentic","score":86.2,"unit":"percent","family":"cyberGym","index":70.53926643798528,"aboveFit":false,"run":null,"source":"Zhun Wang, Tianneng Shi, Jingxuan He, Matthew Cai, Jialin Zhang, Dawn Song","sourceUrl":"https://www.cybergym.io/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"cweBench","benchmarkName":"CWE-Bench","category":"agentic","score":47.2,"unit":"percent","family":"cweBench","index":null,"aboveFit":false,"run":null,"source":"Collinear","sourceUrl":"https://cwe-bench.com/#leaderboard","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":20,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gemini-3-8-live","providerId":"google","slug":"gemini-3-8-live","name":"Gemini 3.8 Live","provider":"Google","description":"Gemini 3.8 Live is a standard model from Google. It has no scored result.","logoUrl":null,"releaseDate":"2026-09-15","contextWindowTokens":128000,"modalities":["text"],"openWeights":false,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":0.75,"outputPerMillion":4.5,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gemini-3-8-live-extended-thinking","providerId":"google","slug":"gemini-3-8-live-extended-thinking","name":"Gemini 3.8 Live Extended Thinking","provider":"Google","description":"Gemini 3.8 Live Extended Thinking is a reasoning model from Google in the Gemini 3.8 Live family. It has no scored result.","logoUrl":null,"releaseDate":"2026-09-15","contextWindowTokens":128000,"modalities":["text"],"openWeights":false,"details":null,"reasoning":true,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":0.75,"outputPerMillion":4.5,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gemini-decision-1","providerId":"google","slug":"gemini-decision-1","name":"Gemini Decision 1","provider":"Google","description":"Gemini Decision 1 is a model from Google. It has no scored result.","logoUrl":null,"releaseDate":"2026-11-11","contextWindowTokens":262000,"modalities":["text"],"openWeights":false,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":0.1,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"geminiflashlatest","providerId":"google","slug":"google-gemini-flash-latest","name":"Gemini Flash Latest","provider":"Google","description":"Gemini Flash Latest is a model from Google. It has no scored result.","logoUrl":null,"releaseDate":"2026-04-27T19:33:18.000Z","contextWindowTokens":1048576,"modalities":["text","image","video","file","audio"],"openWeights":null,"details":{"apiId":"~google/gemini-flash-latest","maxOutputTokens":65536,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.075,"weightsUrl":null},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":0.75,"outputPerMillion":3.75,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":68,"timeToFirstToken":2.9665,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"geminiomni11flash","providerId":"google","slug":"google-gemini-omni-1-1-flash","name":"Gemini Omni 1.1 Flash","provider":"Google","description":"Gemini Omni 1.1 Flash is a model from Google. It has no scored result.","logoUrl":null,"releaseDate":"2026-11-04T20:54:28.000Z","contextWindowTokens":131072,"modalities":["text","image","video"],"openWeights":null,"details":{"apiId":"google/gemini-omni-1.1-flash","maxOutputTokens":65536,"knowledgeCutoff":null,"retiresAt":null,"tools":false,"structuredOutput":false,"cacheReadPerMillion":null,"weightsUrl":null},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":1.5,"outputPerMillion":9,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":1.5,"outputPerMillion":9,"cacheReadPerMillion":null,"contextTokens":131072,"maxOutputTokens":65536,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"geminiomni12flash","providerId":"google","slug":"google-gemini-omni-1-2-flash","name":"Gemini Omni 1.2 Flash","provider":"Google","description":"Gemini Omni 1.2 Flash is a model from Google. It has no scored result.","logoUrl":null,"releaseDate":"2026-11-05T17:16:01.000Z","contextWindowTokens":131072,"modalities":["text","image","video"],"openWeights":null,"details":{"apiId":"google/gemini-omni-1.2-flash","maxOutputTokens":65536,"knowledgeCutoff":null,"retiresAt":null,"tools":false,"structuredOutput":false,"cacheReadPerMillion":null,"weightsUrl":null},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":1.5,"outputPerMillion":9,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":1.5,"outputPerMillion":9,"cacheReadPerMillion":null,"contextTokens":131072,"maxOutputTokens":65536,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"geminiprolatest","providerId":"google","slug":"google-gemini-pro-latest","name":"Gemini Pro Latest","provider":"Google","description":"Gemini Pro Latest is a model from Google. It has no scored result.","logoUrl":null,"releaseDate":"2026-04-27T19:34:11.000Z","contextWindowTokens":1048576,"modalities":["audio","file","image","text","video"],"openWeights":null,"details":{"apiId":"~google/gemini-pro-latest","maxOutputTokens":943718,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.09999999999999999,"weightsUrl":null},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":2,"outputPerMillion":10,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":32,"timeToFirstToken":8.153,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gemma-2-27b","providerId":"google","slug":"gemma-2-27b","name":"Gemma 2 27B","provider":"Google","description":"Gemma 2 27B is a standard model from Google. It has 4 scored results in 2 categories. It has no index.","logoUrl":null,"releaseDate":"2024-06-27","contextWindowTokens":8000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":16.595593398081178,"multilingual":null,"instruction":null,"math":11.136133893045383},"capabilityCounts":{"knowledge":2,"math":2},"benchmarks":[{"benchmarkId":"mmlu","benchmarkName":"Massive Multitask Language Understanding","category":"knowledge","score":76.2,"unit":"percent","family":"mmlu","index":22.514547794194396,"aboveFit":false,"run":null,"source":"Dan Hendrycks, Collin Burns, Steven Basart, Andy Zou, Mantas Mazeika, Dawn Song, Jacob Steinhardt","sourceUrl":"https://arxiv.org/abs/2009.03300","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:legal_bench:overall","benchmarkName":"LegalBench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":67.978,"unit":"percent","family":"vals-ai:legal_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","measuredAt":"2026-10-13","stderr":0.651,"costPerTest":null,"latency":0.887},{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":36.48989898989899,"unit":"percent","family":"gpqa-diamond","index":10.67663900196795,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.3897884428691634,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-math-level-5:overall","benchmarkName":"MATH level 5","category":"math","domain":null,"eligible":true,"ranked":true,"score":27.888972809667674,"unit":"percent","family":"epoch-ai:epoch-math-level-5:overall","index":12.545129623785627,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0.9939474306804167,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":1.3888888888888888,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":9.727138162305138,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0.9078625258710881,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":25,"benchmarkCount":4,"categoryCount":2,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gemma29b","providerId":"google","slug":"google-gemma-2-9b","name":"Gemma 2 9B","provider":"Google","description":"Gemma 2 9B is a model from Google. It has 3 scored results in 2 categories. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":5,"multilingual":null,"instruction":null,"math":9.222101171140025},"capabilityCounts":{"knowledge":1,"math":2},"benchmarks":[{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":27.46212121212121,"unit":"percent","family":"gpqa-diamond","index":5,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.3032467954778797,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-math-level-5:overall","benchmarkName":"MATH level 5","category":"math","domain":null,"eligible":true,"ranked":true,"score":21.00641993957704,"unit":"percent","family":"epoch-ai:epoch-math-level-5:overall","index":9.201892551419157,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0.8990724846256172,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":0.5555555555555556,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":9.242309790860894,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0.3883473863441843,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":25,"benchmarkCount":3,"categoryCount":2,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gemma29bit","providerId":"google","slug":"google-gemma-2-9b-it","name":"Gemma 2 9B IT","provider":"Google","description":"Gemma 2 9B IT is a model from Google. It has no scored result.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[{"benchmarkId":"vals-ai:legal_bench:overall","benchmarkName":"LegalBench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":68.138,"unit":"percent","family":"vals-ai:legal_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","measuredAt":"2026-10-13","stderr":0.735,"costPerTest":null,"latency":0.579}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":{"elo":1207.4011335745363,"eloUpper":1211.2041981303628,"eloLower":1203.5980690187102,"votes":54611,"rank":321,"rankUpper":null,"rankLower":null,"arenaName":"Gemma 2 9B IT","organization":"google","updatedAt":"2026-11-09"},"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gemma312b","providerId":"google","slug":"google-gemma-3-12b","name":"Gemma 3 12B","provider":"Google","description":"Gemma 3 12B is a model from Google. It has 3 scored results in 3 categories. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":13.398047822752666,"multilingual":39.024193732525035,"instruction":null,"math":18.615658305449504},"capabilityCounts":{"multilingual":1,"knowledge":1,"math":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-french:overall","benchmarkName":"EuroEval French","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":55.375,"unit":"percent","family":"euroeval","index":47.351369034008094,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/french","measuredAt":null,"stderr":0.3367422048095546,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":53.7675,"unit":"percent","family":"euroeval","index":45.17036140922047,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":0.2665256788754134,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-dutch:overall","benchmarkName":"EuroEval Dutch","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":52.44777777777777,"unit":"percent","family":"euroeval","index":43.37980201309709,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/dutch","measuredAt":null,"stderr":0.26934992614821596,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-portuguese:overall","benchmarkName":"EuroEval Portuguese","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":49.2775,"unit":"percent","family":"euroeval","index":39.078464528787244,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/portuguese","measuredAt":null,"stderr":0.3169138289819489,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-italian:overall","benchmarkName":"EuroEval Italian","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":49.197500000000005,"unit":"percent","family":"euroeval","index":38.96992293626282,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/italian","measuredAt":null,"stderr":0.3401654009448933,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":47.73142857142857,"unit":"percent","family":"euroeval","index":36.98080134111658,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.26552988993516263,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-spanish:overall","benchmarkName":"EuroEval Spanish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":45.09125,"unit":"percent","family":"euroeval","index":33.398686507470174,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/spanish","measuredAt":null,"stderr":0.4248713040439422,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":42.885000000000005,"unit":"percent","family":"euroeval","index":30.405312901132575,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.4908681722214224,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":39.457070707070706,"unit":"percent","family":"gpqa-diamond","index":13.398047822752666,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.525432779548305,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":16.666666666666664,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":18.615658305449504,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":4.24174778156649,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":30,"benchmarkCount":3,"categoryCount":3,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gemma312bpt","providerId":"google","slug":"google-gemma-3-12b-pt","name":"Gemma 3 12B Pt","provider":"Google","description":"Gemma 3 12B Pt is a model from Google. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":41.61756249676927,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":55.00375,"unit":"percent","family":"euroeval","index":46.847668206199444,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":0.34072532926097526,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-french:overall","benchmarkName":"EuroEval French","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":54.51875,"unit":"percent","family":"euroeval","index":46.18963480152013,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/french","measuredAt":null,"stderr":0.4443815927780988,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-dutch:overall","benchmarkName":"EuroEval Dutch","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":53.361111111111114,"unit":"percent","family":"euroeval","index":44.61898519441759,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/dutch","measuredAt":null,"stderr":0.43294712133994284,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-portuguese:overall","benchmarkName":"EuroEval Portuguese","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":52.585,"unit":"percent","family":"euroeval","index":43.56598099471885,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/portuguese","measuredAt":null,"stderr":0.3742805598745412,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":49.71285714285714,"unit":"percent","family":"euroeval","index":39.66914399881969,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.46485459653354766,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-italian:overall","benchmarkName":"EuroEval Italian","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":46.81625,"unit":"percent","family":"euroeval","index":35.73911459627803,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/italian","measuredAt":null,"stderr":0.5297935918827256,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":45.27625,"unit":"percent","family":"euroeval","index":33.64968894018289,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.7202202354141405,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-spanish:overall","benchmarkName":"EuroEval Spanish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":39.107499999999995,"unit":"percent","family":"euroeval","index":25.280114579119964,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/spanish","measuredAt":null,"stderr":0.44661049864507213,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gemma31b","providerId":"google","slug":"google-gemma-3-1b","name":"Gemma 3 1B","provider":"Google","description":"Gemma 3 1B is a model from Google. It has 2 scored results in 2 categories. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":5,"multilingual":null,"instruction":null,"math":9.565528705157057},"capabilityCounts":{"knowledge":1,"math":1},"benchmarks":[{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":19.94949494949495,"unit":"percent","family":"gpqa-diamond","index":5,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.521201537793133,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":1.1111111111111112,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":9.565528705157057,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0.5362818328685681,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":25,"benchmarkCount":2,"categoryCount":2,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gemma31bit","providerId":"google","slug":"google-gemma-3-1b-it","name":"Gemma 3 1B IT","provider":"Google","description":"Gemma 3 1B IT is a model from Google. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":5,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-french:overall","benchmarkName":"EuroEval French","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":24.020000000000003,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/french","measuredAt":null,"stderr":0.41988837802444595,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-portuguese:overall","benchmarkName":"EuroEval Portuguese","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":23.381249999999998,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/portuguese","measuredAt":null,"stderr":0.44315629297122705,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-dutch:overall","benchmarkName":"EuroEval Dutch","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":21.45555555555556,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/dutch","measuredAt":null,"stderr":0.3439602331306755,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":21.394999999999996,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":0.40252717299581153,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":21.374285714285715,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.38603161549705395,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-italian:overall","benchmarkName":"EuroEval Italian","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":20.733750000000004,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/italian","measuredAt":null,"stderr":0.3588545143926714,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":20.0925,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.5791143561508383,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-spanish:overall","benchmarkName":"EuroEval Spanish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":16.224999999999998,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/spanish","measuredAt":null,"stderr":0.3969532403193102,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gemma31bpt","providerId":"google","slug":"google-gemma-3-1b-pt","name":"Gemma 3 1B Pt","provider":"Google","description":"Gemma 3 1B Pt is a model from Google. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":5,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-dutch:overall","benchmarkName":"EuroEval Dutch","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":24.354444444444443,"unit":"percent","family":"euroeval","index":5.263612771603903,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/dutch","measuredAt":null,"stderr":0.6607272461667981,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-french:overall","benchmarkName":"EuroEval French","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":24.313750000000002,"unit":"percent","family":"euroeval","index":5.2083997740177,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/french","measuredAt":null,"stderr":0.390956359457165,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":22.8375,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":0.43474309942769646,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-portuguese:overall","benchmarkName":"EuroEval Portuguese","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":22.662499999999994,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/portuguese","measuredAt":null,"stderr":0.45177255062697197,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":20.91142857142857,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.4960065009448887,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-italian:overall","benchmarkName":"EuroEval Italian","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":19.31875,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/italian","measuredAt":null,"stderr":0.3938591118661596,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-spanish:overall","benchmarkName":"EuroEval Spanish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":17.118750000000002,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/spanish","measuredAt":null,"stderr":0.3859485069280616,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":16.709999999999994,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.7210322461582421,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gemma3270m","providerId":"google","slug":"google-gemma-3-270m","name":"Gemma 3 270M","provider":"Google","description":"Gemma 3 270M is a model from Google. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":5,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-french:overall","benchmarkName":"EuroEval French","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":16.42375,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/french","measuredAt":null,"stderr":0.80228871829286,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-dutch:overall","benchmarkName":"EuroEval Dutch","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":13.614444444444445,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/dutch","measuredAt":null,"stderr":0.6440218921208067,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":13.55625,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":1.1722107852259336,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-portuguese:overall","benchmarkName":"EuroEval Portuguese","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":11.855000000000002,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/portuguese","measuredAt":null,"stderr":1.0318710251286254,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":11.345000000000002,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.7282202190958447,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-spanish:overall","benchmarkName":"EuroEval Spanish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":10.78125,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/spanish","measuredAt":null,"stderr":0.5534423524631992,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":10.285714285714286,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.8518515248610895,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-italian:overall","benchmarkName":"EuroEval Italian","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":9.85375,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/italian","measuredAt":null,"stderr":0.6696675947811721,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gemma3270mit","providerId":"google","slug":"google-gemma-3-270m-it","name":"Gemma 3 270M IT","provider":"Google","description":"Gemma 3 270M IT is a model from Google. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":5,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-french:overall","benchmarkName":"EuroEval French","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":10.85875,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/french","measuredAt":null,"stderr":0.7535914012248282,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-dutch:overall","benchmarkName":"EuroEval Dutch","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":9.867777777777777,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/dutch","measuredAt":null,"stderr":0.4882167096234576,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":9.78875,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.6834494311944374,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":9.64625,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":0.7128277754829704,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-portuguese:overall","benchmarkName":"EuroEval Portuguese","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":9.1375,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/portuguese","measuredAt":null,"stderr":0.8707548808361628,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":8.105714285714285,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.5178113280226413,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-spanish:overall","benchmarkName":"EuroEval Spanish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":7.64,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/spanish","measuredAt":null,"stderr":0.47403355630166105,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-italian:overall","benchmarkName":"EuroEval Italian","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":7.528749999999999,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/italian","measuredAt":null,"stderr":0.5130165080189916,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gemma327bpt","providerId":"google","slug":"google-gemma-3-27b-pt","name":"Gemma 3 27B Pt","provider":"Google","description":"Gemma 3 27B Pt is a model from Google. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":48.323613120221395,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":58.35,"unit":"percent","family":"euroeval","index":51.38775950601008,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":0.35760706438771594,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-french:overall","benchmarkName":"EuroEval French","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":58.01,"unit":"percent","family":"euroeval","index":50.92645773778127,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/french","measuredAt":null,"stderr":0.49454050642995867,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-portuguese:overall","benchmarkName":"EuroEval Portuguese","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":57.739999999999995,"unit":"percent","family":"euroeval","index":50.56012986301134,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/portuguese","measuredAt":null,"stderr":0.39573902385284165,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-dutch:overall","benchmarkName":"EuroEval Dutch","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":56.51888888888888,"unit":"percent","family":"euroeval","index":48.90336305489549,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/dutch","measuredAt":null,"stderr":0.3759054911431927,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":55.66428571428572,"unit":"percent","family":"euroeval","index":47.74386318554731,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.36764834573604294,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-italian:overall","benchmarkName":"EuroEval Italian","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":53.75625000000001,"unit":"percent","family":"euroeval","index":45.155097747771734,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/italian","measuredAt":null,"stderr":0.6134648013537533,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":50.68875000000001,"unit":"percent","family":"euroeval","index":40.9932060594134,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.6099129036182134,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-spanish:overall","benchmarkName":"EuroEval Spanish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":43.81625,"unit":"percent","family":"euroeval","index":31.668804876612175,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/spanish","measuredAt":null,"stderr":0.3594461886012982,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gemma34b","providerId":"google","slug":"google-gemma-3-4b","name":"Gemma 3 4B","provider":"Google","description":"Gemma 3 4B is a model from Google. It has 3 scored results in 3 categories. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":5,"multilingual":22.69631588832374,"instruction":null,"math":13.282546219562882},"capabilityCounts":{"multilingual":1,"knowledge":1,"math":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-french:overall","benchmarkName":"EuroEval French","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":44.3575,"unit":"percent","family":"euroeval","index":32.403156588535225,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/french","measuredAt":null,"stderr":0.6076543013918357,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-dutch:overall","benchmarkName":"EuroEval Dutch","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":39.75444444444444,"unit":"percent","family":"euroeval","index":26.157869332555315,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/dutch","measuredAt":null,"stderr":0.5911591459179745,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":37.77375000000001,"unit":"percent","family":"euroeval","index":23.470522716251867,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":0.46248479704742723,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-portuguese:overall","benchmarkName":"EuroEval Portuguese","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":37.741249999999994,"unit":"percent","family":"euroeval","index":23.426427694288805,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/portuguese","measuredAt":null,"stderr":0.45533812985516603,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-italian:overall","benchmarkName":"EuroEval Italian","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":36.66499999999999,"unit":"percent","family":"euroeval","index":21.966204082358676,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/italian","measuredAt":null,"stderr":0.3548305053120433,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":34.76,"unit":"percent","family":"euroeval","index":19.381557410370863,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.2554228187271658,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":33.98375,"unit":"percent","family":"euroeval","index":18.328364770407322,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.49198100827166086,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-spanish:overall","benchmarkName":"EuroEval Spanish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":31.86875,"unit":"percent","family":"euroeval","index":15.458796418042887,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/spanish","measuredAt":null,"stderr":0.43931409606339744,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":23.232323232323232,"unit":"percent","family":"gpqa-diamond","index":5,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.8690142324718149,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":7.5,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":13.282546219562882,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.763853991962833,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":30,"benchmarkCount":3,"categoryCount":3,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gemma34bpt","providerId":"google","slug":"google-gemma-3-4b-pt","name":"Gemma 3 4B Pt","provider":"Google","description":"Gemma 3 4B Pt is a model from Google. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":22.485410851956537,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-french:overall","benchmarkName":"EuroEval French","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":41.92375,"unit":"percent","family":"euroeval","index":29.101117828456296,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/french","measuredAt":null,"stderr":0.4631127697440441,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-dutch:overall","benchmarkName":"EuroEval Dutch","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":41.638888888888886,"unit":"percent","family":"euroeval","index":28.71462684535284,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/dutch","measuredAt":null,"stderr":0.4173194885882464,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-portuguese:overall","benchmarkName":"EuroEval Portuguese","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":37.487500000000004,"unit":"percent","family":"euroeval","index":23.082147330500412,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/portuguese","measuredAt":null,"stderr":0.44991665894918803,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":37.0925,"unit":"percent","family":"euroeval","index":22.54622321741107,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":0.38042739123254515,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":37.00285714285714,"unit":"percent","family":"euroeval","index":22.424598486502,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.3359057204949396,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-italian:overall","benchmarkName":"EuroEval Italian","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":35.42,"unit":"percent","family":"euroeval","index":20.277025548697353,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/italian","measuredAt":null,"stderr":0.4455053310567675,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":33.92000000000001,"unit":"percent","family":"euroeval","index":18.241870688864434,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.5617425455669172,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-spanish:overall","benchmarkName":"EuroEval Spanish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":30.09375,"unit":"percent","family":"euroeval","index":13.050529833907262,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/spanish","measuredAt":null,"stderr":0.44429192261394984,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gemma3ne4bit","providerId":"google","slug":"google-gemma-3n-e4b-it","name":"Gemma 3N E4B IT","provider":"Google","description":"Gemma 3N E4B IT is a model from Google. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":32.321265833460984,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":44.29714285714285,"unit":"percent","family":"euroeval","index":32.321265833460984,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.3153359077141034,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":{"elo":1305.5630577194884,"eloUpper":1310.7263651892208,"eloLower":1300.3997502497562,"votes":22106,"rank":249,"rankUpper":null,"rankLower":null,"arenaName":"Gemma 3n E4B IT","organization":"google","updatedAt":"2026-11-09"},"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gemma412bit","providerId":"google","slug":"google-gemma-4-12b-it","name":"Gemma 4 12B IT","provider":"Google","description":"Gemma 4 12B IT is a model from Google. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":49.96107826121423,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-french:overall","benchmarkName":"EuroEval French","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":61.9025,"unit":"percent","family":"euroeval","index":56.20768459904773,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/french","measuredAt":null,"stderr":0.4993370605112343,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-portuguese:overall","benchmarkName":"EuroEval Portuguese","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":60.74375,"unit":"percent","family":"euroeval","index":54.63552746982679,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/portuguese","measuredAt":null,"stderr":0.6189772310028859,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-italian:overall","benchmarkName":"EuroEval Italian","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":59.465,"unit":"percent","family":"euroeval","index":52.90055795181922,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/italian","measuredAt":null,"stderr":0.5719894011255803,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-dutch:overall","benchmarkName":"EuroEval Dutch","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":57.31444444444445,"unit":"percent","family":"euroeval","index":49.982748891666155,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/dutch","measuredAt":null,"stderr":0.4839676552808305,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":57.2825,"unit":"percent","family":"euroeval","index":49.939407630762304,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":0.8275472040917061,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":55.22571428571428,"unit":"percent","family":"euroeval","index":47.148822669386625,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.5976877213435943,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":49.92875,"unit":"percent","family":"euroeval","index":39.96206093043137,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.7599825244701354,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-spanish:overall","benchmarkName":"EuroEval Spanish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":45.03375,"unit":"percent","family":"euroeval","index":33.32067223784324,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/spanish","measuredAt":null,"stderr":0.5545366872444059,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gemma4e2bit","providerId":"google","slug":"google-gemma-4-e2b-it","name":"Gemma 4 E2B IT","provider":"Google","description":"Gemma 4 E2B IT is a model from Google. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":28.80771633616371,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-french:overall","benchmarkName":"EuroEval French","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":47.52625000000001,"unit":"percent","family":"euroeval","index":36.702421229932305,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/french","measuredAt":null,"stderr":0.6172886784155368,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-portuguese:overall","benchmarkName":"EuroEval Portuguese","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":43.544999999999995,"unit":"percent","family":"euroeval","index":31.30078103945905,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/portuguese","measuredAt":null,"stderr":0.574230463315906,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-dutch:overall","benchmarkName":"EuroEval Dutch","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":43.05333333333333,"unit":"percent","family":"euroeval","index":30.633702502069365,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/dutch","measuredAt":null,"stderr":0.5261847302710051,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-italian:overall","benchmarkName":"EuroEval Italian","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":42.292500000000004,"unit":"percent","family":"euroeval","index":29.601426731498567,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/italian","measuredAt":null,"stderr":0.775879541230467,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":41.122499999999995,"unit":"percent","family":"euroeval","index":28.01400594082887,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":0.7999277311107548,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":38.91285714285714,"unit":"percent","family":"euroeval","index":25.016029008022603,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.7823121147585272,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-spanish:overall","benchmarkName":"EuroEval Spanish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":37.368750000000006,"unit":"percent","family":"euroeval","index":22.921030904096977,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/spanish","measuredAt":null,"stderr":0.6671780590666931,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":35.83625,"unit":"percent","family":"euroeval","index":20.84178102230099,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.6965730848231217,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gemma4e4bit","providerId":"google","slug":"google-gemma-4-e4b-it","name":"Gemma 4 E4B IT","provider":"Google","description":"Gemma 4 E4B IT is a model from Google. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":38.614807257248444,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-french:overall","benchmarkName":"EuroEval French","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":53.709999999999994,"unit":"percent","family":"euroeval","index":45.092347139593535,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/french","measuredAt":null,"stderr":0.5067636283317895,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-portuguese:overall","benchmarkName":"EuroEval Portuguese","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":51.49875,"unit":"percent","family":"euroeval","index":42.09218968372317,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/portuguese","measuredAt":null,"stderr":0.6527872930748576,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-italian:overall","benchmarkName":"EuroEval Italian","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":50.90375,"unit":"percent","family":"euroeval","index":41.284911589322775,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/italian","measuredAt":null,"stderr":0.551656596081294,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-dutch:overall","benchmarkName":"EuroEval Dutch","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":49.77777777777778,"unit":"percent","family":"euroeval","index":39.75722636259448,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/dutch","measuredAt":null,"stderr":0.5377513767624185,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":48.09374999999999,"unit":"percent","family":"euroeval","index":37.47238815190241,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":0.8017666041311524,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":46.231428571428566,"unit":"percent","family":"euroeval","index":34.945646481283646,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.7418812270856036,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":41.37625,"unit":"percent","family":"euroeval","index":28.358286304617277,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.6668805646440747,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-spanish:overall","benchmarkName":"EuroEval Spanish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":37.513749999999995,"unit":"percent","family":"euroeval","index":23.117762540547478,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/spanish","measuredAt":null,"stderr":0.6180918216575916,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"decision-index-0-3-gev-26b-decide","providerId":"autotrust","slug":"decision-index-0-3-gev-26b-decide","name":"GEV-26B-Decide","provider":"autotrust","description":"GEV-26B-Decide is a model from autotrust. It has no scored result.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"decision-index-0-3-gevva-e2b","providerId":"davidburhans","slug":"decision-index-0-3-gevva-e2b","name":"Gevva E2B","provider":"davidburhans","description":"Gevva E2B is a model from davidburhans. It has no scored result.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"decision-index-0-3-gevva-e4b","providerId":"davidburhans","slug":"decision-index-0-3-gevva-e4b","name":"Gevva E4B","provider":"davidburhans","description":"Gevva E4B is a model from davidburhans. It has no scored result.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"fastino-glide","providerId":"fastino","slug":"fastino-glide","name":"GLiDE","provider":"Fastino","description":"GLiDE is a reasoning model from Fastino. It has no scored result.","logoUrl":null,"releaseDate":"2026-09-30","contextWindowTokens":40000,"modalities":["text"],"openWeights":false,"details":null,"reasoning":true,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":0.15,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"fastino-glide-v1-2","providerId":"fastino","slug":"fastino-glide-v1-2","name":"GLiDE v1.2","provider":"Fastino","description":"GLiDE v1.2 is a standard model from Fastino in the GLiDE family. It has no scored result.","logoUrl":null,"releaseDate":"2026-10-22","contextWindowTokens":64000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":0.04,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"glimpse","providerId":"elorian","slug":"elorian-glimpse","name":"Glimpse","provider":"Elorian","description":"Glimpse is a model from Elorian. It has no scored result.","logoUrl":null,"releaseDate":"2026-10-17T03:10:29.000Z","contextWindowTokens":262144,"modalities":["text","image","video"],"openWeights":null,"details":{"apiId":"elorian/glimpse","maxOutputTokens":256000,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.06,"weightsUrl":null},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":0.6,"outputPerMillion":3,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":191,"timeToFirstToken":0.565,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gliner2-large","providerId":"fastino","slug":"gliner2-large","name":"GLiNER2 large","provider":"Fastino","description":"GLiNER2 large is a standard model from Fastino. It has 2 scored results in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":56.73002981490743,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"reasoning":2},"benchmarks":[{"benchmarkId":"jevBench14","benchmarkName":"jevBench14","category":"reasoning","score":15.138054851149672,"unit":"percent","family":"jevBench14","index":56.899548003785966,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"jevBench15","benchmarkName":"jevBench15","category":"reasoning","score":8.399649998514048,"unit":"percent","family":"jevBench15","index":56.5605116260289,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":15,"benchmarkCount":2,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gliner2-5-base","providerId":"fastino","slug":"gliner2-5-base","name":"GLiNER2.5 base","provider":"Fastino","description":"GLiNER2.5 base is a standard model from Fastino. It has 2 scored results in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":53.324966161899084,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"reasoning":2},"benchmarks":[{"benchmarkId":"jevBench14","benchmarkName":"jevBench14","category":"reasoning","score":11.777645154812003,"unit":"percent","family":"jevBench14","index":53.80247661737528,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"jevBench15","benchmarkName":"jevBench15","category":"reasoning","score":2.2681537932105402,"unit":"percent","family":"jevBench15","index":52.847455706422885,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":15,"benchmarkCount":2,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gliner2-5-multi","providerId":"fastino","slug":"gliner2-5-multi","name":"GLiNER2.5 multi","provider":"Fastino","description":"GLiNER2.5 multi is a standard model from Fastino. It has 2 scored results in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":52.531203436278226,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"reasoning":2},"benchmarks":[{"benchmarkId":"jevBench14","benchmarkName":"jevBench14","category":"reasoning","score":9.759108258784872,"unit":"percent","family":"jevBench14","index":51.942122247515826,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"jevBench15","benchmarkName":"jevBench15","category":"reasoning","score":2.7186855157112975,"unit":"percent","family":"jevBench15","index":53.120284625040625,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":15,"benchmarkCount":2,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gliner2-5-small","providerId":"fastino","slug":"gliner2-5-small","name":"GLiNER2.5 small","provider":"Fastino","description":"GLiNER2.5 small is a standard model from Fastino. It has 2 scored results in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":50.79356082159474,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"reasoning":2},"benchmarks":[{"benchmarkId":"jevBench14","benchmarkName":"jevBench14","category":"reasoning","score":7.187800408767937,"unit":"percent","family":"jevBench14","index":49.572314787247024,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"jevBench15","benchmarkName":"jevBench15","category":"reasoning","score":0.893172274587778,"unit":"percent","family":"jevBench15","index":52.01480685594245,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":15,"benchmarkCount":2,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"decision-index-0-3-gliner2-5-decide","providerId":"fastino","slug":"decision-index-0-3-gliner2-5-decide","name":"GLiNER2.5-Decide","provider":"Fastino","description":"GLiNER2.5-Decide is a model from Fastino. It has no scored result.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"glm45v","providerId":"z-ai","slug":"z-ai-glm-4-5v","name":"GLM 4.5V","provider":"Z.AI","description":"GLM 4.5V is a model from Z.AI. It has no scored result.","logoUrl":null,"releaseDate":"2025-08-11T14:24:48.000Z","contextWindowTokens":65536,"modalities":["text","image"],"openWeights":true,"details":{"apiId":"z-ai/glm-4.5v","maxOutputTokens":16384,"knowledgeCutoff":"2024-12-31","retiresAt":"2026-12-31","tools":true,"structuredOutput":false,"cacheReadPerMillion":0.11,"weightsUrl":"https://huggingface.co/zai-org/GLM-4.5V"},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":0.6,"outputPerMillion":1.7999999999999998,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":28,"timeToFirstToken":1.0255,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":{"elo":1332.5384132485751,"eloUpper":1340.9667933250075,"eloLower":1324.1100331721423,"votes":4813,"rank":228,"rankUpper":null,"rankLower":null,"arenaName":"GLM-4.5V","organization":"zai","updatedAt":"2026-11-09"},"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[{"host":"zhipu-ai","name":"Zhipu AI","kind":"maker","url":"https://docs.bigmodel.cn/cn/guide/start/pricing","inputPerMillion":0.29879,"outputPerMillion":0.89637,"cacheReadPerMillion":0.05976,"contextTokens":64000,"maxOutputTokens":16384,"deprecated":false},{"host":"novita-ai","name":"Novita AI","kind":"host","url":"https://novita.ai/docs/guides/introduction","inputPerMillion":0.6,"outputPerMillion":1.8,"cacheReadPerMillion":0.11,"contextTokens":65536,"maxOutputTokens":16384,"deprecated":false},{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":0.6,"outputPerMillion":1.8,"cacheReadPerMillion":0.11,"contextTokens":65536,"maxOutputTokens":16384,"deprecated":false},{"host":"z-ai","name":"Z.AI","kind":"maker","url":"https://docs.z.ai/guides/overview/pricing","inputPerMillion":0.6,"outputPerMillion":1.8,"cacheReadPerMillion":0.11,"contextTokens":64000,"maxOutputTokens":16384,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"glm46v","providerId":"z-ai","slug":"z-ai-glm-4-6v","name":"GLM 4.6V","provider":"Z.AI","description":"GLM 4.6V is a model from Z.AI. It has no scored result.","logoUrl":null,"releaseDate":"2025-12-08T15:24:22.000Z","contextWindowTokens":131072,"modalities":["image","text","video"],"openWeights":true,"details":{"apiId":"z-ai/glm-4.6v","maxOutputTokens":32768,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":false,"cacheReadPerMillion":0.055,"weightsUrl":"https://huggingface.co/zai-org/GLM-4.6V"},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":0.3,"outputPerMillion":0.8999999999999999,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":14,"timeToFirstToken":4.874,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":{"elo":1376.480261938766,"eloUpper":1387.635068113072,"eloLower":1365.3254557644598,"votes":2825,"rank":183,"rankUpper":null,"rankLower":null,"arenaName":"GLM-4.6V","organization":"zai","updatedAt":"2026-11-09"},"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[{"host":"zhipu-ai","name":"Zhipu AI","kind":"maker","url":"https://docs.bigmodel.cn/cn/guide/start/pricing","inputPerMillion":0.1494,"outputPerMillion":0.44819,"cacheReadPerMillion":0.02988,"contextTokens":128000,"maxOutputTokens":32768,"deprecated":false},{"host":"novita-ai","name":"Novita AI","kind":"host","url":"https://novita.ai/docs/guides/introduction","inputPerMillion":0.3,"outputPerMillion":0.9,"cacheReadPerMillion":0.055,"contextTokens":131072,"maxOutputTokens":32768,"deprecated":false},{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":0.3,"outputPerMillion":0.9,"cacheReadPerMillion":0.055,"contextTokens":131072,"maxOutputTokens":32768,"deprecated":false},{"host":"z-ai","name":"Z.AI","kind":"maker","url":"https://docs.z.ai/guides/overview/pricing","inputPerMillion":0.3,"outputPerMillion":0.9,"cacheReadPerMillion":0.05,"contextTokens":128000,"maxOutputTokens":32768,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"glmflashlatest","providerId":"z-ai","slug":"z-ai-glm-flash-latest","name":"GLM Flash Latest","provider":"Z.AI","description":"GLM Flash Latest is a model from Z.AI. It has no scored result.","logoUrl":null,"releaseDate":"2026-08-27T08:00:33.000Z","contextWindowTokens":1048576,"modalities":["text","image","video"],"openWeights":null,"details":{"apiId":"~z-ai/glm-flash-latest","maxOutputTokens":131072,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.03,"weightsUrl":null},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":0.0399,"outputPerMillion":0.0784,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":121.5,"timeToFirstToken":0.897,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"glmlatest","providerId":"z-ai","slug":"z-ai-glm-latest","name":"GLM Latest","provider":"Z.AI","description":"GLM Latest is a model from Z.AI. It has no scored result.","logoUrl":null,"releaseDate":"2026-08-19T14:50:53.000Z","contextWindowTokens":1048576,"modalities":["text"],"openWeights":null,"details":{"apiId":"~z-ai/glm-latest","maxOutputTokens":943718,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.0689,"weightsUrl":null},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":0.0699,"outputPerMillion":1.4,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":86,"timeToFirstToken":0.32,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"glmlegal","providerId":"z-ai","slug":"z-ai-glm-legal","name":"GLM Legal","provider":"Z.AI","description":"GLM Legal is a model from Z.AI. It has no scored result.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[{"benchmarkId":"vals-ai:legal_bench:overall","benchmarkName":"LegalBench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":86.235,"unit":"percent","family":"vals-ai:legal_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","measuredAt":"2026-10-13","stderr":0.41,"costPerTest":0.000135,"latency":6.005},{"benchmarkId":"vals-ai:legal_research:overall","benchmarkName":"Legal Research Bench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":41.827,"unit":"percent","family":"vals-ai:legal_research:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_research","measuredAt":"2026-11-09","stderr":3.428,"costPerTest":0.04878,"latency":2388.741}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"glm-realtime-air","providerId":"z-ai","slug":"glm-realtime-air","name":"GLM Realtime Air","provider":"Z.AI","description":"GLM Realtime Air is a standard model from Z.AI. It has no scored result.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":false,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"glm-realtime-flash","providerId":"z-ai","slug":"glm-realtime-flash","name":"GLM Realtime Flash","provider":"Z.AI","description":"GLM Realtime Flash is a standard model from Z.AI. It has no scored result.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":false,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"glm-4-5-air","providerId":"z-ai","slug":"glm-4-5-air","name":"GLM-4.5-Air","provider":"Z.AI","description":"GLM-4.5-Air is a standard model from Z.AI. It has 7 scored results in 4 categories. It has no index.","logoUrl":null,"releaseDate":"2025-07-28","contextWindowTokens":131072,"modalities":["text"],"openWeights":true,"details":{"apiId":"z-ai/glm-4.5-air","maxOutputTokens":98304,"knowledgeCutoff":"2024-12-31","retiresAt":null,"tools":true,"structuredOutput":false,"cacheReadPerMillion":0.03,"weightsUrl":"https://huggingface.co/zai-org/GLM-4.5-Air"},"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":33.64558984969354,"coding":null,"reasoning":38.09356300839697,"multimodal":null,"knowledge":35.69322478159561,"multilingual":null,"instruction":24.16122850965421,"math":null},"capabilityCounts":{"knowledge":3,"reasoning":2,"agentic":1,"instruction":1},"benchmarks":[{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":73.3,"unit":"percent","family":"gpqa-diamond","index":44.43785780946977,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":46.7,"unit":"percent","family":"lcr","index":39.33801000402104,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"tau2Bench","benchmarkName":"τ²-Bench Tool-Agent-User Evaluation","category":"agentic","score":46.5,"unit":"percent","family":"tau2-bench","index":33.64558984969354,"aboveFit":false,"run":null,"source":"Victor Barres, Honghua Dong, Soham Ray, Xujie Si, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2506.07982","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaIfBench","benchmarkName":"Artificial Analysis IFBench","category":"instruction","score":37.6,"unit":"percent","family":"ifbench-family","index":24.16122850965421,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"omniscienceAccuracy","benchmarkName":"Artificial Analysis Omniscience Accuracy","category":"knowledge","score":16.3,"unit":"percent","family":"omniscienceAccuracy","index":31.232123006795025,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":7,"unit":"percent","family":"hle","index":31.409693528522038,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":0,"unit":"percent","family":"critpt","index":36.8491160127729,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":0.19999999999999998,"outputPerMillion":1.1,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":64,"timeToFirstToken":0.635,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":{"elo":1383.6745877603603,"eloUpper":1387.9047073952142,"eloLower":1379.4444681255063,"votes":30591,"rank":176,"rankUpper":null,"rankLower":null,"arenaName":"GLM-4.5-Air","organization":"zai","updatedAt":"2026-11-09"},"quality":{"coveragePercent":55.00000000000001,"benchmarkCount":7,"categoryCount":4,"confidence":"unranked","state":"partial"},"hosts":[{"host":"novita-ai","name":"Novita AI","kind":"host","url":"https://novita.ai/docs/guides/introduction","inputPerMillion":0.13,"outputPerMillion":0.85,"cacheReadPerMillion":0.025,"contextTokens":131072,"maxOutputTokens":98304,"deprecated":false},{"host":"siliconflow","name":"SiliconFlow","kind":"host","url":"https://cloud.siliconflow.com/models","inputPerMillion":0.14,"outputPerMillion":0.86,"cacheReadPerMillion":null,"contextTokens":131000,"maxOutputTokens":131000,"deprecated":false},{"host":"zhipu-ai","name":"Zhipu AI","kind":"maker","url":"https://docs.bigmodel.cn/cn/guide/start/pricing","inputPerMillion":0.11952,"outputPerMillion":0.89637,"cacheReadPerMillion":0.0239,"contextTokens":131072,"maxOutputTokens":98304,"deprecated":false},{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":0.2,"outputPerMillion":1.1,"cacheReadPerMillion":0.03,"contextTokens":131072,"maxOutputTokens":98304,"deprecated":false},{"host":"z-ai","name":"Z.AI","kind":"maker","url":"https://docs.z.ai/guides/overview/pricing","inputPerMillion":0.2,"outputPerMillion":1.1,"cacheReadPerMillion":0.03,"contextTokens":131072,"maxOutputTokens":98304,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"glm47flashnvfp4","providerId":"z-ai","slug":"z-ai-glm-4-7-flash-nvfp4","name":"GLM-4.7 Flash NVFP4","provider":"Z.AI","description":"GLM-4.7 Flash NVFP4 is a model from Z.AI. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":36.77704357479283,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-spanish:overall","benchmarkName":"EuroEval Spanish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":49.40749999999999,"unit":"percent","family":"euroeval","index":39.25484461663942,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/spanish","measuredAt":null,"stderr":0.6479740060372793,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":45.755,"unit":"percent","family":"euroeval","index":34.299242532946245,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.6594138021758416,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"glm-4-7-flash","providerId":"z-ai","slug":"glm-4-7-flash","name":"GLM-4.7-Flash","provider":"Z.AI","description":"GLM-4.7-Flash is a reasoning model from Z.AI. It has 6 scored results in 4 categories. It has no index.","logoUrl":null,"releaseDate":"2026-01-19","contextWindowTokens":200000,"modalities":["text"],"openWeights":true,"details":{"apiId":"z-ai/glm-4.7-flash","maxOutputTokens":117964,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":null,"weightsUrl":"https://huggingface.co/zai-org/GLM-4.7-Flash"},"reasoning":true,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":40.92803232774245,"reasoning":null,"multimodal":null,"knowledge":32.39487104739805,"multilingual":46.734038726525455,"instruction":null,"math":36.08784427801159},"capabilityCounts":{"math":2,"knowledge":1,"coding":2,"multilingual":1},"benchmarks":[{"benchmarkId":"aime2025","benchmarkName":"American Invitational Mathematics Examination 2025","category":"math","score":91.6,"unit":"percent","family":"aime2025","index":48.711746536131294,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gpqa","benchmarkName":"Graduate-Level Google-Proof Q&A","category":"knowledge","score":75.2,"unit":"percent","family":"gpqa-diamond","index":46.18048589181737,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"liveCodeBenchV6","benchmarkName":"LiveCodeBench v6","category":"coding","score":64,"unit":"percent","family":"liveCodeBenchV6","index":36.96472852121869,"aboveFit":false,"run":null,"source":"LiveCodeBench maintainers","sourceUrl":"https://github.com/LiveCodeBench/LiveCodeBench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"sweVerified","benchmarkName":"Software Engineering Benchmark Verified","category":"coding","score":59.2,"unit":"percent","family":"swe-bench-verified","index":44.89133613426621,"aboveFit":false,"run":null,"source":"Carlos E. Jimenez, John Yang, Alexander Wettig, Shunyu Yao, Kexin Pei, Ofir Press, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2310.06770","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-french:overall","benchmarkName":"EuroEval French","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":57.133750000000006,"unit":"percent","family":"euroeval","index":49.73758810716222,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/french","measuredAt":null,"stderr":0.6312202963308452,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-italian:overall","benchmarkName":"EuroEval Italian","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":57.12875,"unit":"percent","family":"euroeval","index":49.73080425762942,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/italian","measuredAt":null,"stderr":0.537251395996325,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-portuguese:overall","benchmarkName":"EuroEval Portuguese","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":54.92000000000001,"unit":"percent","family":"euroeval","index":46.734038726525455,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/portuguese","measuredAt":null,"stderr":0.756699246728844,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":53.45571428571429,"unit":"percent","family":"euroeval","index":44.74733993478378,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.649701658377104,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-dutch:overall","benchmarkName":"EuroEval Dutch","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":53.33444444444444,"unit":"percent","family":"euroeval","index":44.58280466357611,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/dutch","measuredAt":null,"stderr":0.631722122064894,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":45.13888888888889,"unit":"percent","family":"gpqa-diamond","index":18.60925620297872,"aboveFit":false,"run":"none effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.1751012010888355,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":25,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":23.46394201989189,"aboveFit":false,"run":"none effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":4.946047294394164,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":0.060500000000000005,"outputPerMillion":0.39999999999999997,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":50,"timeToFirstToken":0.273,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":{"elo":1350.5695470423655,"eloUpper":1356.257709286916,"eloLower":1344.881384797815,"votes":11937,"rank":212,"rankUpper":null,"rankLower":null,"arenaName":"GLM-4.7-Flash","organization":"zai","updatedAt":"2026-11-09"},"quality":{"coveragePercent":50,"benchmarkCount":6,"categoryCount":4,"confidence":"unranked","state":"partial"},"hosts":[{"host":"deepinfra","name":"Deep Infra","kind":"host","url":"https://deepinfra.com/models","inputPerMillion":0.06,"outputPerMillion":0.4,"cacheReadPerMillion":0.01,"contextTokens":202752,"maxOutputTokens":16384,"deprecated":true},{"host":"cloudflare","name":"Cloudflare Workers AI","kind":"cloud","url":"https://developers.cloudflare.com/workers-ai/models/","inputPerMillion":0.0605,"outputPerMillion":0.4,"cacheReadPerMillion":null,"contextTokens":131072,"maxOutputTokens":131072,"deprecated":false},{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":0.0605,"outputPerMillion":0.4,"cacheReadPerMillion":null,"contextTokens":200000,"maxOutputTokens":117964,"deprecated":false},{"host":"amazon-bedrock","name":"Amazon Bedrock","kind":"cloud","url":"https://docs.aws.amazon.com/bedrock/latest/userguide/models-supported.html","inputPerMillion":0.07,"outputPerMillion":0.4,"cacheReadPerMillion":null,"contextTokens":202752,"maxOutputTokens":131072,"deprecated":true},{"host":"novita-ai","name":"Novita AI","kind":"host","url":"https://novita.ai/docs/guides/introduction","inputPerMillion":0.07,"outputPerMillion":0.4,"cacheReadPerMillion":0.01,"contextTokens":200000,"maxOutputTokens":128000,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"glm-5-reasoning","providerId":"z-ai","slug":"glm-5-reasoning","name":"GLM-5 (Reasoning)","provider":"Z.AI","description":"GLM-5 (Reasoning) is a reasoning model from Z.AI in the GLM-5 family. It has no scored result.","logoUrl":null,"releaseDate":"2026-02-12","contextWindowTokens":200000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":true,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":1,"outputPerMillion":3.2,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"glm53flashx","providerId":"z-ai","slug":"z-ai-glm-5-3-flashx","name":"GLM-5.3 FlashX","provider":"Z.AI","description":"GLM-5.3 FlashX is a model from Z.AI. It has no scored result.","logoUrl":null,"releaseDate":"2026-09-18T15:07:00.000Z","contextWindowTokens":1048576,"modalities":["text","image","video"],"openWeights":null,"details":{"apiId":"z-ai/glm-5.3-flashx","maxOutputTokens":131072,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":false,"cacheReadPerMillion":0.075,"weightsUrl":null},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":0.37,"outputPerMillion":1.25,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":84,"timeToFirstToken":2.153,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"glm53flashxlegal","providerId":"z-ai","slug":"z-ai-glm-5-3-flashx-legal","name":"GLM-5.3 FlashX Legal","provider":"Z.AI","description":"GLM-5.3 FlashX Legal is a model from Z.AI. It has no scored result.","logoUrl":null,"releaseDate":"2026-11-02T13:29:14.000Z","contextWindowTokens":1048576,"modalities":["text","image","video"],"openWeights":null,"details":{"apiId":"z-ai/glm-5.3-flashx-legal","maxOutputTokens":131072,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.075,"weightsUrl":null},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":0.37,"outputPerMillion":1.25,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":104,"timeToFirstToken":1.222,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"glm53prime","providerId":"z-ai","slug":"z-ai-glm-5-3-prime","name":"GLM-5.3 Prime","provider":"Z.AI","description":"GLM-5.3 Prime is a model from Z.AI. It has no scored result.","logoUrl":null,"releaseDate":"2026-09-23T21:40:51.000Z","contextWindowTokens":1000000,"modalities":["text"],"openWeights":null,"details":{"apiId":"z-ai/glm-5.3-prime","maxOutputTokens":131072,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":false,"cacheReadPerMillion":0.56,"weightsUrl":null},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":2.8,"outputPerMillion":8.8,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":39,"timeToFirstToken":0.628,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gptastralatest","providerId":"openai","slug":"openai-gpt-astra-latest","name":"GPT Astra Latest","provider":"OpenAI","description":"GPT Astra Latest is a model from OpenAI. It has no scored result.","logoUrl":null,"releaseDate":"2026-09-11T12:48:52.000Z","contextWindowTokens":1050000,"modalities":["file","image","text"],"openWeights":null,"details":{"apiId":"~openai/gpt-astra-latest","maxOutputTokens":128000,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":1,"weightsUrl":null},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":10,"outputPerMillion":50,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":32,"timeToFirstToken":3.968,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gptaudio","providerId":"openai","slug":"openai-gpt-audio","name":"GPT Audio","provider":"OpenAI","description":"GPT Audio is a model from OpenAI. It has no scored result.","logoUrl":null,"releaseDate":"2026-01-19T22:42:49.000Z","contextWindowTokens":128000,"modalities":["text","audio"],"openWeights":null,"details":{"apiId":"openai/gpt-audio","maxOutputTokens":16384,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":null,"weightsUrl":null},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":2.5,"outputPerMillion":10,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":79,"timeToFirstToken":0.5425,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":2.5,"outputPerMillion":10,"cacheReadPerMillion":null,"contextTokens":128000,"maxOutputTokens":16384,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gpt-audio-1-5","providerId":"openai","slug":"gpt-audio-1-5","name":"GPT Audio 1.5","provider":"OpenAI","description":"GPT Audio 1.5 is a standard model from OpenAI. It has no scored result.","logoUrl":null,"releaseDate":"2026-04-23","contextWindowTokens":128000,"modalities":["text"],"openWeights":false,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":2.5,"outputPerMillion":10,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gptaudiomini","providerId":"openai","slug":"openai-gpt-audio-mini","name":"GPT Audio Mini","provider":"OpenAI","description":"GPT Audio Mini is a model from OpenAI. It has no scored result.","logoUrl":null,"releaseDate":"2026-01-19T21:50:19.000Z","contextWindowTokens":128000,"modalities":["text","audio"],"openWeights":null,"details":{"apiId":"openai/gpt-audio-mini","maxOutputTokens":16384,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":null,"weightsUrl":null},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":0.6,"outputPerMillion":2.4,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":98,"timeToFirstToken":0.505,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":0.6,"outputPerMillion":2.4,"cacheReadPerMillion":null,"contextTokens":128000,"maxOutputTokens":16384,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gptchatlatest","providerId":"openai","slug":"openai-gpt-chat-latest","name":"GPT Chat Latest","provider":"OpenAI","description":"GPT Chat Latest is a model from OpenAI. It has no scored result.","logoUrl":null,"releaseDate":"2026-05-05T16:56:52.000Z","contextWindowTokens":400000,"modalities":["text","image","file"],"openWeights":null,"details":{"apiId":"openai/gpt-chat-latest","maxOutputTokens":128000,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.5,"weightsUrl":null},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":5,"outputPerMillion":30,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":89,"timeToFirstToken":1.285,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gptdaybreakblue","providerId":"openai","slug":"openai-gpt-daybreak-blue","name":"GPT Daybreak Blue","provider":"OpenAI","description":"GPT Daybreak Blue is a model from OpenAI. It has no scored result.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[{"benchmarkId":"vals-ai:cyber:overall","benchmarkName":"CyberBench v1.1","category":null,"domain":"Other","eligible":false,"ranked":true,"score":72.976,"unit":"percent","family":"vals-ai:cyber:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/cyber","measuredAt":"2026-11-05","stderr":5.265,"costPerTest":3.97239,"latency":710.02},{"benchmarkId":"vals-ai:srebench:overall","benchmarkName":"SRE Bench","category":null,"domain":"Other","eligible":false,"ranked":true,"score":31.679,"unit":"percent","family":"vals-ai:srebench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/srebench","measuredAt":"2026-11-05","stderr":2.88,"costPerTest":54.338843,"latency":10424.242}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gptlunalatest","providerId":"openai","slug":"openai-gpt-luna-latest","name":"GPT Luna Latest","provider":"OpenAI","description":"GPT Luna Latest is a model from OpenAI. It has no scored result.","logoUrl":null,"releaseDate":"2026-09-11T12:48:42.000Z","contextWindowTokens":1050000,"modalities":["file","image","text"],"openWeights":null,"details":{"apiId":"~openai/gpt-luna-latest","maxOutputTokens":128000,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.01,"weightsUrl":null},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":0.09999999999999999,"outputPerMillion":0.5,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":65,"timeToFirstToken":2.755,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gptminilatest","providerId":"openai","slug":"openai-gpt-mini-latest","name":"GPT Mini Latest","provider":"OpenAI","description":"GPT Mini Latest is a model from OpenAI. It has no scored result.","logoUrl":null,"releaseDate":"2026-04-27T19:34:31.000Z","contextWindowTokens":400000,"modalities":["file","image","text"],"openWeights":null,"details":{"apiId":"~openai/gpt-mini-latest","maxOutputTokens":128000,"knowledgeCutoff":"2025-08-31","retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.075,"weightsUrl":null},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":0.75,"outputPerMillion":4.5,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":97,"timeToFirstToken":1.252,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gptosssafeguard20b","providerId":"openai","slug":"openai-gpt-oss-safeguard-20b","name":"GPT OSS Safeguard 20B","provider":"OpenAI","description":"GPT OSS Safeguard 20B is a model from OpenAI. It has no scored result.","logoUrl":null,"releaseDate":"2025-10-29T15:47:16.000Z","contextWindowTokens":131072,"modalities":["text"],"openWeights":true,"details":{"apiId":"openai/gpt-oss-safeguard-20b","maxOutputTokens":65536,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.0375,"weightsUrl":"https://huggingface.co/openai/gpt-oss-safeguard-20b"},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":0.075,"outputPerMillion":0.3,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":399,"timeToFirstToken":0.269,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[{"host":"amazon-bedrock","name":"Amazon Bedrock","kind":"cloud","url":"https://docs.aws.amazon.com/bedrock/latest/userguide/models-supported.html","inputPerMillion":0.07,"outputPerMillion":0.2,"cacheReadPerMillion":null,"contextTokens":128000,"maxOutputTokens":16384,"deprecated":false},{"host":"groq","name":"Groq","kind":"host","url":"https://console.groq.com/docs/models","inputPerMillion":0.075,"outputPerMillion":0.3,"cacheReadPerMillion":0.0375,"contextTokens":131072,"maxOutputTokens":65536,"deprecated":false},{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":0.075,"outputPerMillion":0.3,"cacheReadPerMillion":0.0375,"contextTokens":131072,"maxOutputTokens":65536,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gpt-realtime","providerId":"openai","slug":"gpt-realtime","name":"GPT Realtime","provider":"OpenAI","description":"GPT Realtime is a standard model from OpenAI. It has no scored result.","logoUrl":null,"releaseDate":"2025-08-28","contextWindowTokens":32000,"modalities":["text"],"openWeights":false,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":4,"outputPerMillion":16,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gpt-realtime-1-5","providerId":"openai","slug":"gpt-realtime-1-5","name":"GPT Realtime 1.5","provider":"OpenAI","description":"GPT Realtime 1.5 is a standard model from OpenAI. It has no scored result.","logoUrl":null,"releaseDate":"2026-02-23","contextWindowTokens":32000,"modalities":["text"],"openWeights":false,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":4,"outputPerMillion":16,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gpt-realtime-2","providerId":"openai","slug":"gpt-realtime-2","name":"GPT Realtime 2","provider":"OpenAI","description":"GPT Realtime 2 is a standard model from OpenAI. It has no scored result.","logoUrl":null,"releaseDate":null,"contextWindowTokens":128000,"modalities":["text"],"openWeights":false,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":4,"outputPerMillion":24,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gpt-realtime-2-1","providerId":"openai","slug":"gpt-realtime-2-1","name":"GPT Realtime 2.1","provider":"OpenAI","description":"GPT Realtime 2.1 is a standard model from OpenAI. It has no scored result.","logoUrl":null,"releaseDate":"2026-07-06","contextWindowTokens":128000,"modalities":["text"],"openWeights":false,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":4,"outputPerMillion":24,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gpt-realtime-2-1-mini","providerId":"openai","slug":"gpt-realtime-2-1-mini","name":"GPT Realtime 2.1 Mini","provider":"OpenAI","description":"GPT Realtime 2.1 Mini is a standard model from OpenAI. It has no scored result.","logoUrl":null,"releaseDate":"2026-07-06","contextWindowTokens":128000,"modalities":["text"],"openWeights":false,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":0.6,"outputPerMillion":2.4,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gpt-realtime-mini","providerId":"openai","slug":"gpt-realtime-mini","name":"GPT Realtime mini","provider":"OpenAI","description":"GPT Realtime mini is a standard model from OpenAI. It has no scored result.","logoUrl":null,"releaseDate":"2025-10-06","contextWindowTokens":32000,"modalities":["text"],"openWeights":false,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":0.6,"outputPerMillion":2.4,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gptsollatest","providerId":"openai","slug":"openai-gpt-sol-latest","name":"GPT Sol Latest","provider":"OpenAI","description":"GPT Sol Latest is a model from OpenAI. It has no scored result.","logoUrl":null,"releaseDate":"2026-09-11T12:48:48.000Z","contextWindowTokens":1050000,"modalities":["file","image","text"],"openWeights":null,"details":{"apiId":"~openai/gpt-sol-latest","maxOutputTokens":128000,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.09999999999999999,"weightsUrl":null},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":2,"outputPerMillion":10,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":34,"timeToFirstToken":4.441,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gptsw313binstruct","providerId":"openai","slug":"openai-gpt-sw3-1-3b-instruct","name":"GPT Sw3 1.3B Instruct","provider":"OpenAI","description":"GPT Sw3 1.3B Instruct is a model from OpenAI. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":5,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":4.93125,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":1.1045028576241893,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gptterralatest","providerId":"openai","slug":"openai-gpt-terra-latest","name":"GPT Terra Latest","provider":"OpenAI","description":"GPT Terra Latest is a model from OpenAI. It has no scored result.","logoUrl":null,"releaseDate":"2026-09-11T12:48:45.000Z","contextWindowTokens":1050000,"modalities":["file","image","text"],"openWeights":null,"details":{"apiId":"~openai/gpt-terra-latest","maxOutputTokens":128000,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.049999999999999996,"weightsUrl":null},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":1,"outputPerMillion":5,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":49,"timeToFirstToken":3.214,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gpt35","providerId":"openai","slug":"openai-gpt-3-5","name":"GPT-3.5","provider":"OpenAI","description":"GPT-3.5 is a model from OpenAI. It has 2 scored results in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":5,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"coding":2},"benchmarks":[{"benchmarkId":"swe-bench:swe-bench-verified:overall","benchmarkName":"SWE-bench Verified","category":"coding","domain":null,"eligible":true,"ranked":true,"score":0.4,"unit":"percent","family":"swe-bench-verified","index":5,"aboveFit":false,"run":"RAG baseline","source":"SWE-bench team","sourceUrl":"https://www.swebench.com/","measuredAt":"2026-02-26","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"swe-bench:swe-bench-lite:overall","benchmarkName":"SWE-bench Lite","category":"coding","domain":null,"eligible":true,"ranked":true,"score":0.33,"unit":"percent","family":"swe-bench-lite","index":5,"aboveFit":false,"run":"RAG baseline","source":"SWE-bench team","sourceUrl":"https://www.swebench.com/","measuredAt":"2025-09-11","stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":20,"benchmarkCount":2,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:swe-bench-team","name":"SWE-bench team","url":"https://www.swebench.com","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gpt35turbo","providerId":"openai","slug":"openai-gpt-3-5-turbo","name":"GPT-3.5 Turbo","provider":"OpenAI","description":"GPT-3.5 Turbo is a model from OpenAI. It has 5 scored results in 3 categories. It has no index.","logoUrl":null,"releaseDate":"2023-05-28T00:00:00.000Z","contextWindowTokens":16385,"modalities":["text"],"openWeights":null,"details":{"apiId":"openai/gpt-3.5-turbo","maxOutputTokens":4096,"knowledgeCutoff":"2021-09-30","retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":null,"weightsUrl":null},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":41.004242892185715,"multimodal":null,"knowledge":5,"multilingual":null,"instruction":null,"math":15.596251681127967},"capabilityCounts":{"knowledge":1,"math":3,"reasoning":1},"benchmarks":[{"benchmarkId":"vals-ai:legal_bench:overall","benchmarkName":"LegalBench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":64.372,"unit":"percent","family":"vals-ai:legal_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","measuredAt":"2026-10-13","stderr":0.628,"costPerTest":null,"latency":0.542},{"benchmarkId":"vals-ai:medqa:overall","benchmarkName":"MedQA","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":58.471,"unit":"percent","family":"vals-ai:medqa:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medqa","measuredAt":"2026-04-16","stderr":0.881,"costPerTest":0.000424,"latency":1.589},{"benchmarkId":"vals-ai:gpqa:overall","benchmarkName":"GPQA Diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":30.556,"unit":"percent","family":"gpqa-diamond","index":5.234228992698476,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","measuredAt":"2026-09-01","stderr":2.287,"costPerTest":0.000862,"latency":2.628},{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":27.604166666666664,"unit":"percent","family":"gpqa-diamond","index":5,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.9535799687887372,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-math-level-5:overall","benchmarkName":"MATH level 5","category":"math","domain":null,"eligible":true,"ranked":true,"score":13.76038519637462,"unit":"percent","family":"epoch-ai:epoch-math-level-5:overall","index":5.6820922043118,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0.5543945913851348,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-mystery-game-puzzles:overall","benchmarkName":"Mystery Game Puzzles","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":3,"unit":"percent","family":"epoch-ai:epoch-mystery-game-puzzles:overall","index":41.004242892185715,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.7144660799776528,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":2.2222222222222223,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":10.211966533749376,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.2222222222222223,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tiers-1-3-v2-private:overall","benchmarkName":"FrontierMath-Tiers-1-3-v2-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":0,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tiers-1-3-v2-private:overall","index":30.89469630532274,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":0.5,"outputPerMillion":1.5,"batch":{"inputPerMillion":0.25,"outputPerMillion":0.75},"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":121,"timeToFirstToken":0.517,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":{"elo":1124.926855400359,"eloUpper":1129.5906045639904,"eloLower":1120.2631062367277,"votes":66207,"rank":359,"rankUpper":null,"rankLower":null,"arenaName":"GPT-3.5 Turbo-0125","organization":"openai","updatedAt":"2026-11-09"},"quality":{"coveragePercent":40,"benchmarkCount":5,"categoryCount":3,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gpt35turbo0613","providerId":"openai","slug":"openai-gpt-3-5-turbo-older-v0613","name":"GPT-3.5 Turbo (older v0613)","provider":"OpenAI","description":"GPT-3.5 Turbo (older v0613) is a model from OpenAI. It has no scored result.","logoUrl":null,"releaseDate":"2024-01-25T00:00:00.000Z","contextWindowTokens":4095,"modalities":["text"],"openWeights":null,"details":{"apiId":"openai/gpt-3.5-turbo-0613","maxOutputTokens":3685,"knowledgeCutoff":"2021-09-30","retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":null,"weightsUrl":null},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":1,"outputPerMillion":2,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":93,"timeToFirstToken":1.127,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gpt35turbo16k","providerId":"openai","slug":"openai-gpt-3-5-turbo-16k","name":"GPT-3.5 Turbo 16k","provider":"OpenAI","description":"GPT-3.5 Turbo 16k is a model from OpenAI. It has no scored result.","logoUrl":null,"releaseDate":"2023-08-28T00:00:00.000Z","contextWindowTokens":16385,"modalities":["text"],"openWeights":null,"details":{"apiId":"openai/gpt-3.5-turbo-16k","maxOutputTokens":4096,"knowledgeCutoff":"2021-09-30","retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":null,"weightsUrl":null},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":3,"outputPerMillion":4,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":91,"timeToFirstToken":0.796,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gpt35turboinstruct","providerId":"openai","slug":"openai-gpt-3-5-turbo-instruct","name":"GPT-3.5 Turbo Instruct","provider":"OpenAI","description":"GPT-3.5 Turbo Instruct is a model from OpenAI. It has no scored result.","logoUrl":null,"releaseDate":"2023-09-28T00:00:00.000Z","contextWindowTokens":4095,"modalities":["text"],"openWeights":null,"details":{"apiId":"openai/gpt-3.5-turbo-instruct","maxOutputTokens":3685,"knowledgeCutoff":"2021-09-30","retiresAt":null,"tools":false,"structuredOutput":true,"cacheReadPerMillion":null,"weightsUrl":null},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":1.5,"outputPerMillion":2,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gpt4","providerId":"openai","slug":"openai-gpt-4","name":"GPT-4","provider":"OpenAI","description":"GPT-4 is a model from OpenAI. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":"2023-05-28T00:00:00.000Z","contextWindowTokens":8191,"modalities":["text"],"openWeights":null,"details":{"apiId":"openai/gpt-4","maxOutputTokens":4096,"knowledgeCutoff":"2021-09-30","retiresAt":null,"tools":true,"structuredOutput":false,"cacheReadPerMillion":null,"weightsUrl":null},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":21.68383706406152,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"coding":1},"benchmarks":[{"benchmarkId":"swebench-experiments:swe-bench-lite:overall","benchmarkName":"SWE-bench Lite","category":"coding","domain":null,"eligible":true,"ranked":true,"score":23.666666666666668,"unit":"percent","family":"swe-bench-lite","index":23.5813641790751,"aboveFit":false,"run":"OpenCSG StarShip CodeGenAgent","source":"SWE-bench team","sourceUrl":"https://github.com/SWE-bench/experiments/tree/main/evaluation/lite","measuredAt":"2025-09-11","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"swebench-experiments:swe-bench-lite:overall","benchmarkName":"SWE-bench Lite","category":"coding","domain":null,"eligible":true,"ranked":true,"score":19,"unit":"percent","family":"swe-bench-lite","index":19.786309949047943,"aboveFit":false,"run":"AutoCodeRover","source":"SWE-bench team","sourceUrl":"https://github.com/SWE-bench/experiments/tree/main/evaluation/lite","measuredAt":"2025-09-11","stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":30,"outputPerMillion":60,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":94,"timeToFirstToken":0.591,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":{"elo":1186.0132611775641,"eloUpper":1190.0403462460208,"eloLower":1181.9861761091074,"votes":88723,"rank":333,"rankUpper":null,"rankLower":null,"arenaName":"GPT-4-0613","organization":"openai","updatedAt":"2026-11-09"},"quality":{"coveragePercent":20,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":30,"outputPerMillion":60,"cacheReadPerMillion":null,"contextTokens":8191,"maxOutputTokens":4096,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:swe-bench-team","name":"SWE-bench team","url":"https://github.com","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gpt41106","providerId":"openai","slug":"openai-gpt-4-1106","name":"GPT-4 (1106)","provider":"OpenAI","description":"GPT-4 (1106) is a model from OpenAI. It has 2 scored results in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":11.63043023967007,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"coding":2},"benchmarks":[{"benchmarkId":"swe-bench:swe-bench-verified:overall","benchmarkName":"SWE-bench Verified","category":"coding","domain":null,"eligible":true,"ranked":true,"score":12.6,"unit":"percent","family":"swe-bench-verified","index":10.521153009546168,"aboveFit":false,"run":"SWE-agent","source":"SWE-bench team","sourceUrl":"https://www.swebench.com/","measuredAt":"2026-02-26","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"swe-bench:swe-bench-lite:overall","benchmarkName":"SWE-bench Lite","category":"coding","domain":null,"eligible":true,"ranked":true,"score":10.335,"unit":"percent","family":"swe-bench-lite","index":12.739707469793956,"aboveFit":false,"run":"SWE-agent","source":"SWE-bench team","sourceUrl":"https://www.swebench.com/","measuredAt":"2025-09-11","stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":20,"benchmarkCount":2,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:swe-bench-team","name":"SWE-bench team","url":"https://www.swebench.com","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gpt4jun2023","providerId":"openai","slug":"openai-gpt-4-jun-2023","name":"GPT-4 (Jun 2023)","provider":"OpenAI","description":"GPT-4 (Jun 2023) is a model from OpenAI. It has 4 scored results in 3 categories. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":47.973976962532255,"multimodal":null,"knowledge":5.320674833402286,"multilingual":null,"instruction":null,"math":9.860660773042717},"capabilityCounts":{"knowledge":1,"math":2,"reasoning":1},"benchmarks":[{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":30.650252525252526,"unit":"percent","family":"gpqa-diamond","index":5.320674833402286,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.3373229482706233,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-math-level-5:overall","benchmarkName":"MATH level 5","category":"math","domain":null,"eligible":true,"ranked":true,"score":22.970166163141993,"unit":"percent","family":"epoch-ai:epoch-math-level-5:overall","index":10.155792840928385,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0.8465534027077483,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-mystery-game-puzzles:overall","benchmarkName":"Mystery Game Puzzles","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":12,"unit":"percent","family":"epoch-ai:epoch-mystery-game-puzzles:overall","index":47.973976962532255,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":3.2659863237109046,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":1.1111111111111112,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":9.565528705157057,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0.5362818328685681,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":40,"benchmarkCount":4,"categoryCount":3,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gpt4mar2023","providerId":"openai","slug":"openai-gpt-4-mar-2023","name":"GPT-4 (Mar 2023)","provider":"OpenAI","description":"GPT-4 (Mar 2023) is a model from OpenAI. It has 2 scored results in 2 categories. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":9.98181121793781,"multilingual":null,"instruction":null,"math":9.242309790860894},"capabilityCounts":{"knowledge":1,"math":1},"benchmarks":[{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":35.73232323232323,"unit":"percent","family":"gpqa-diamond","index":9.98181121793781,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.363099375599328,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":0.5555555555555556,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":9.242309790860894,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0.3883473863441843,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":25,"benchmarkCount":2,"categoryCount":2,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gpt-4-turbo","providerId":"openai","slug":"gpt-4-turbo","name":"GPT-4 Turbo","provider":"OpenAI","description":"GPT-4 Turbo is a standard model from OpenAI. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":"2023-11-06","contextWindowTokens":128000,"modalities":["text","image"],"openWeights":false,"details":{"apiId":"openai/gpt-4-turbo","maxOutputTokens":4096,"knowledgeCutoff":"2023-12-31","retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":null,"weightsUrl":null},"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":27.49685182290584,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"knowledge":1},"benchmarks":[{"benchmarkId":"vals-ai:medqa:overall","benchmarkName":"MedQA","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":81.986,"unit":"percent","family":"vals-ai:medqa:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medqa","measuredAt":"2026-04-16","stderr":0.686,"costPerTest":0.012152,"latency":8.413},{"benchmarkId":"vals-ai:legal_bench:overall","benchmarkName":"LegalBench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":80.462,"unit":"percent","family":"vals-ai:legal_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","measuredAt":"2026-10-13","stderr":0.482,"costPerTest":null,"latency":1.405},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":3.1,"unit":"percent","family":"hle","index":27.49685182290584,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":10,"outputPerMillion":30,"batch":{"inputPerMillion":5,"outputPerMillion":15},"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":15,"timeToFirstToken":1.594,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":{"elo":1271.699249411648,"eloUpper":1275.593631540456,"eloLower":1267.80486728284,"votes":98114,"rank":284,"rankUpper":null,"rankLower":null,"arenaName":"GPT-4 Turbo-2024-04-09","organization":"openai","updatedAt":"2026-11-09"},"quality":{"coveragePercent":15,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gpt4turboapr2024","providerId":"openai","slug":"openai-gpt-4-turbo-apr-2024","name":"GPT-4 Turbo (Apr 2024)","provider":"OpenAI","description":"GPT-4 Turbo (Apr 2024) is a model from OpenAI. It has 4 scored results in 2 categories. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":19.941009455703146,"multilingual":null,"instruction":null,"math":21.920229110404076},"capabilityCounts":{"math":3,"knowledge":1},"benchmarks":[{"benchmarkId":"epoch-ai:epoch-math-level-5:overall","benchmarkName":"MATH level 5","category":"math","domain":null,"eligible":true,"ranked":true,"score":46.733383685800604,"unit":"percent","family":"epoch-ai:epoch-math-level-5:overall","index":21.698903555806847,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.142825893039598,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":46.590909090909086,"unit":"percent","family":"gpqa-diamond","index":19.941009455703146,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.657688333258055,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":6.666666666666667,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":12.797717848118644,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.4034381938205005,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tiers-1-3-v2-private:overall","benchmarkName":"FrontierMath-Tiers-1-3-v2-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":0.7017543859649122,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tiers-1-3-v2-private:overall","index":31.26406592728675,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0.49534089619636035,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":25,"benchmarkCount":4,"categoryCount":2,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gpt4turbonov2023","providerId":"openai","slug":"openai-gpt-4-turbo-nov-2023","name":"GPT-4 Turbo (Nov 2023)","provider":"OpenAI","description":"GPT-4 Turbo (Nov 2023) is a model from OpenAI. It has 2 scored results in 2 categories. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":16.018127591699653,"multilingual":null,"instruction":null,"math":17.319217130415936},"capabilityCounts":{"knowledge":1,"math":1},"benchmarks":[{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":42.31376262626263,"unit":"percent","family":"gpqa-diamond","index":16.018127591699653,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.4076142618301675,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-math-level-5:overall","benchmarkName":"MATH level 5","category":"math","domain":null,"eligible":true,"ranked":true,"score":37.71714501510574,"unit":"percent","family":"epoch-ai:epoch-math-level-5:overall","index":17.319217130415936,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.0538794632397386,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":25,"benchmarkCount":2,"categoryCount":2,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gpt-4o-audio","providerId":"openai","slug":"gpt-4o-audio","name":"GPT-4o Audio","provider":"OpenAI","description":"GPT-4o Audio is a standard model from OpenAI. It has no scored result.","logoUrl":null,"releaseDate":"2024-10-17","contextWindowTokens":128000,"modalities":["text"],"openWeights":false,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":2.5,"outputPerMillion":10,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gpt-4o-mini-audio","providerId":"openai","slug":"gpt-4o-mini-audio","name":"GPT-4o mini Audio","provider":"OpenAI","description":"GPT-4o mini Audio is a standard model from OpenAI. It has no scored result.","logoUrl":null,"releaseDate":"2024-12-17","contextWindowTokens":128000,"modalities":["text"],"openWeights":false,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":0.15,"outputPerMillion":0.6,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gpt5codex","providerId":"openai","slug":"openai-gpt-5-codex","name":"GPT-5 Codex","provider":"OpenAI","description":"GPT-5 Codex is a model from OpenAI. It has 3 scored results in 2 categories. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":60.73428545974524,"coding":52.31758028973095,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"coding":2,"agentic":1},"benchmarks":[{"benchmarkId":"vals-ai:lcb:overall","benchmarkName":"LiveCodeBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":84.725,"unit":"percent","family":"livecodebench","index":59.64359213075585,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","measuredAt":"2026-09-01","stderr":1.032,"costPerTest":0.097451,"latency":134.963},{"benchmarkId":"vals-ai:terminal-bench:overall","benchmarkName":"Terminal-Bench 1.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":58.75,"unit":"percent","family":"vals-ai:terminal-bench:overall","index":60.73428545974524,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench","measuredAt":"2026-01-12","stderr":5.504,"costPerTest":1.380093,"latency":663.824},{"benchmarkId":"vals-ai:ioi-v1:overall","benchmarkName":"IOI v1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":9.75,"unit":"percent","family":"vals-ai:ioi-v1:overall","index":44.99156844870605,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/ioi-v1","measuredAt":"2026-08-09","stderr":7.717,"costPerTest":2.669955,"latency":1542.894}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":40,"benchmarkCount":3,"categoryCount":2,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gpt5image","providerId":"openai","slug":"openai-gpt-5-image","name":"GPT-5 Image","provider":"OpenAI","description":"GPT-5 Image is a model from OpenAI. It has no scored result.","logoUrl":null,"releaseDate":"2025-10-14T13:19:46.000Z","contextWindowTokens":400000,"modalities":["image","text","file"],"openWeights":null,"details":{"apiId":"openai/gpt-5-image","maxOutputTokens":128000,"knowledgeCutoff":null,"retiresAt":"2026-12-11","tools":false,"structuredOutput":true,"cacheReadPerMillion":1.25,"weightsUrl":null},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":10,"outputPerMillion":10,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":10,"outputPerMillion":10,"cacheReadPerMillion":1.25,"contextTokens":400000,"maxOutputTokens":128000,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gpt5imagemini","providerId":"openai","slug":"openai-gpt-5-image-mini","name":"GPT-5 Image Mini","provider":"OpenAI","description":"GPT-5 Image Mini is a model from OpenAI. It has no scored result.","logoUrl":null,"releaseDate":"2025-10-16T14:23:03.000Z","contextWindowTokens":400000,"modalities":["file","image","text"],"openWeights":null,"details":{"apiId":"openai/gpt-5-image-mini","maxOutputTokens":128000,"knowledgeCutoff":null,"retiresAt":"2026-12-01","tools":false,"structuredOutput":true,"cacheReadPerMillion":0.25,"weightsUrl":null},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":2.5,"outputPerMillion":2,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":2.5,"outputPerMillion":2,"cacheReadPerMillion":0.25,"contextTokens":400000,"maxOutputTokens":128000,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gpt5pro","providerId":"openai","slug":"openai-gpt-5-pro","name":"GPT-5 Pro","provider":"OpenAI","description":"GPT-5 Pro is a model from OpenAI. It has 6 scored results in 2 categories. It has no index.","logoUrl":null,"releaseDate":"2025-10-06T18:51:03.000Z","contextWindowTokens":400000,"modalities":["image","text","file"],"openWeights":null,"details":{"apiId":"openai/gpt-5-pro","maxOutputTokens":128000,"knowledgeCutoff":"2024-09-30","retiresAt":"2026-12-11","tools":true,"structuredOutput":true,"cacheReadPerMillion":null,"weightsUrl":null},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":59.08963358979441,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":58.14819240585005},"capabilityCounts":{"reasoning":3,"math":3},"benchmarks":[{"benchmarkId":"arc-prize:v1-public-eval:overall","benchmarkName":"ARC-AGI-1 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":77,"unit":"percent","family":"arc-prize:v1-public-eval:overall","index":null,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":4.0316,"latency":null},{"benchmarkId":"arc-prize:v1-semi-private:overall","benchmarkName":"ARC-AGI-1 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":70.17,"unit":"percent","family":"arc-prize:v1-semi-private:overall","index":59.247041205904196,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":4.7785,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-2025-02-28-public:overall","benchmarkName":"FrontierMath-2025-02-28-Public","category":"math","domain":null,"eligible":false,"ranked":true,"score":60,"unit":"percent","family":"epoch-ai:epoch-frontiermath-2025-02-28-public:overall","index":null,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":16.329931618554518,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tiers-1-3-v2-private:overall","benchmarkName":"FrontierMath-Tiers-1-3-v2-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":55.78947368421052,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tiers-1-3-v2-private:overall","index":60.259581251461846,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.946997752822398,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-chess-puzzles:overall","benchmarkName":"Chess Puzzles","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":48,"unit":"percent","family":"epoch-ai:epoch-chess-puzzles:overall","index":70.41740509234356,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":5.021167315686783,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tier-4-v2-private:overall","benchmarkName":"FrontierMath-Tier-4-v2-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":19.51219512195122,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tier-4-v2-private:overall","index":56.153671107835635,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":6.265967111543964,"costPerTest":null,"latency":null},{"benchmarkId":"arc-prize:v2-semi-private:overall","benchmarkName":"ARC-AGI-2 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":18.33,"unit":"percent","family":"arc-prize:v2-semi-private:overall","index":47.60445447113548,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":7.1432,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-private:overall","benchmarkName":"FrontierMath-Tier-4-2025-07-01-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":14.6,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-private:overall","index":58.03132485825266,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":5.1,"costPerTest":null,"latency":null},{"benchmarkId":"arc-prize:v2-public-eval:overall","benchmarkName":"ARC-AGI-2 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":13.33,"unit":"percent","family":"arc-prize:v2-public-eval:overall","index":null,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":8.0107,"latency":null}],"price":{"inputPerMillion":15,"outputPerMillion":120,"batch":{"inputPerMillion":7.5,"outputPerMillion":60},"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":8,"timeToFirstToken":14.242,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":25,"benchmarkCount":6,"categoryCount":2,"confidence":"unranked","state":"partial"},"hosts":[{"host":"azure","name":"Microsoft Azure","kind":"cloud","url":"https://learn.microsoft.com/en-us/azure/ai-services/openai/concepts/models","inputPerMillion":15,"outputPerMillion":120,"cacheReadPerMillion":null,"contextTokens":400000,"maxOutputTokens":128000,"deprecated":false},{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":15,"outputPerMillion":120,"cacheReadPerMillion":null,"contextTokens":400000,"maxOutputTokens":128000,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:arc-prize-foundation","name":"ARC Prize Foundation","url":"https://arcprize.org","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gpt51thinkinghigh","providerId":"openai","slug":"openai-gpt-5-1-thinking-high","name":"GPT-5.1 (Thinking, High)","provider":"OpenAI","description":"GPT-5.1 (Thinking, High) is a model from OpenAI. It has 2 scored results in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":53.87879538363901,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"reasoning":2},"benchmarks":[{"benchmarkId":"arc-prize:v1-public-eval:overall","benchmarkName":"ARC-AGI-1 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":77.12,"unit":"percent","family":"arc-prize:v1-public-eval:overall","index":null,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.546,"latency":null},{"benchmarkId":"arc-prize:v1-semi-private:overall","benchmarkName":"ARC-AGI-1 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":72.83,"unit":"percent","family":"arc-prize:v1-semi-private:overall","index":60.469939239816085,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.674,"latency":null},{"benchmarkId":"arc-prize:v2-public-eval:overall","benchmarkName":"ARC-AGI-2 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":18.33,"unit":"percent","family":"arc-prize:v2-public-eval:overall","index":null,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":1.2166,"latency":null},{"benchmarkId":"arc-prize:v2-semi-private:overall","benchmarkName":"ARC-AGI-2 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":17.64,"unit":"percent","family":"arc-prize:v2-semi-private:overall","index":47.28765152746194,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":1.1704,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":15,"benchmarkCount":2,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:arc-prize-foundation","name":"ARC Prize Foundation","url":"https://arcprize.org","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gpt51thinkinglow","providerId":"openai","slug":"openai-gpt-5-1-thinking-low","name":"GPT-5.1 (Thinking, Low)","provider":"OpenAI","description":"GPT-5.1 (Thinking, Low) is a model from OpenAI. It has 2 scored results in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":41.15802105899391,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"reasoning":2},"benchmarks":[{"benchmarkId":"arc-prize:v1-public-eval:overall","benchmarkName":"ARC-AGI-1 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":44,"unit":"percent","family":"arc-prize:v1-public-eval:overall","index":null,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.0929,"latency":null},{"benchmarkId":"arc-prize:v1-semi-private:overall","benchmarkName":"ARC-AGI-1 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":33.17,"unit":"percent","family":"arc-prize:v1-semi-private:overall","index":42.23680539585153,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.1,"latency":null},{"benchmarkId":"arc-prize:v2-public-eval:overall","benchmarkName":"ARC-AGI-2 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":2.22,"unit":"percent","family":"arc-prize:v2-public-eval:overall","index":null,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.1348,"latency":null},{"benchmarkId":"arc-prize:v2-semi-private:overall","benchmarkName":"ARC-AGI-2 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":1.94,"unit":"percent","family":"arc-prize:v2-semi-private:overall","index":40.07923672213629,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.129,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":15,"benchmarkCount":2,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:arc-prize-foundation","name":"ARC Prize Foundation","url":"https://arcprize.org","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gpt51thinkingmedium","providerId":"openai","slug":"openai-gpt-5-1-thinking-medium","name":"GPT-5.1 (Thinking, Medium)","provider":"OpenAI","description":"GPT-5.1 (Thinking, Medium) is a model from OpenAI. It has 2 scored results in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":47.843502459170544,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"reasoning":2},"benchmarks":[{"benchmarkId":"arc-prize:v1-public-eval:overall","benchmarkName":"ARC-AGI-1 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":68.86,"unit":"percent","family":"arc-prize:v1-public-eval:overall","index":null,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.2563,"latency":null},{"benchmarkId":"arc-prize:v1-semi-private:overall","benchmarkName":"ARC-AGI-1 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":57.67,"unit":"percent","family":"arc-prize:v1-semi-private:overall","index":53.500339918724244,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.3149,"latency":null},{"benchmarkId":"arc-prize:v2-public-eval:overall","benchmarkName":"ARC-AGI-2 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":8.4,"unit":"percent","family":"arc-prize:v2-public-eval:overall","index":null,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.4599,"latency":null},{"benchmarkId":"arc-prize:v2-semi-private:overall","benchmarkName":"ARC-AGI-2 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":6.529999999999999,"unit":"percent","family":"arc-prize:v2-semi-private:overall","index":42.186664999616845,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.4206,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":15,"benchmarkCount":2,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:arc-prize-foundation","name":"ARC Prize Foundation","url":"https://arcprize.org","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gpt51codexmini","providerId":"openai","slug":"openai-gpt-5-1-codex-mini","name":"GPT-5.1 Codex Mini","provider":"OpenAI","description":"GPT-5.1 Codex Mini is a model from OpenAI. It has no scored result.","logoUrl":null,"releaseDate":"2025-11-13T18:17:00.000Z","contextWindowTokens":400000,"modalities":["image","text"],"openWeights":null,"details":{"apiId":"openai/gpt-5.1-codex-mini","maxOutputTokens":128000,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.03,"weightsUrl":null},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":0.25,"outputPerMillion":2,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":78,"timeToFirstToken":4.011,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":0.25,"outputPerMillion":2,"cacheReadPerMillion":0.03,"contextTokens":400000,"maxOutputTokens":128000,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gpt51thinking","providerId":"openai","slug":"openai-gpt-5-1-thinking","name":"GPT-5.1 Thinking","provider":"OpenAI","description":"GPT-5.1 Thinking is a model from OpenAI. It has 2 scored results in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":34.52448615859204,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"reasoning":2},"benchmarks":[{"benchmarkId":"arc-prize:v1-public-eval:overall","benchmarkName":"ARC-AGI-1 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":12.379999999999999,"unit":"percent","family":"arc-prize:v1-public-eval:overall","index":null,"aboveFit":false,"run":"none effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.0275,"latency":null},{"benchmarkId":"arc-prize:v1-semi-private:overall","benchmarkName":"ARC-AGI-1 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":5.83,"unit":"percent","family":"arc-prize:v1-semi-private:overall","index":29.667620340531546,"aboveFit":false,"run":"none effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.0338,"latency":null},{"benchmarkId":"arc-prize:v2-semi-private:overall","benchmarkName":"ARC-AGI-2 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":0.42,"unit":"percent","family":"arc-prize:v2-semi-private:overall","index":39.38135197665253,"aboveFit":false,"run":"none effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.0581,"latency":null},{"benchmarkId":"arc-prize:v2-public-eval:overall","benchmarkName":"ARC-AGI-2 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":0,"unit":"percent","family":"arc-prize:v2-public-eval:overall","index":null,"aboveFit":false,"run":"none effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.0591,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":15,"benchmarkCount":2,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:arc-prize-foundation","name":"ARC Prize Foundation","url":"https://arcprize.org","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gpt52refinement","providerId":"openai","slug":"openai-gpt-5-2-refinement","name":"GPT-5.2 (Refinement)","provider":"OpenAI","description":"GPT-5.2 (Refinement) is a model from OpenAI. It has 2 scored results in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":71.54592784733778,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"reasoning":2},"benchmarks":[{"benchmarkId":"arc-prize:v1-semi-private:overall","benchmarkName":"ARC-AGI-1 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":94.5,"unit":"percent","family":"arc-prize:v1-semi-private:overall","index":70.43242059127125,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":11.4,"latency":null},{"benchmarkId":"arc-prize:v2-semi-private:overall","benchmarkName":"ARC-AGI-2 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":72.89999999999999,"unit":"percent","family":"arc-prize:v2-semi-private:overall","index":72.65943510340432,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":38.99,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":15,"benchmarkCount":2,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:arc-prize-foundation","name":"ARC Prize Foundation","url":"https://arcprize.org","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gpt52chat","providerId":"openai","slug":"openai-gpt-5-2-chat","name":"GPT-5.2 Chat","provider":"OpenAI","description":"GPT-5.2 Chat is a model from OpenAI. It has no scored result.","logoUrl":null,"releaseDate":"2025-12-10T18:03:03.000Z","contextWindowTokens":128000,"modalities":["file","image","text"],"openWeights":null,"details":{"apiId":"openai/gpt-5.2-chat","maxOutputTokens":32000,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.175,"weightsUrl":null},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":1.75,"outputPerMillion":14,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":{"elo":1438.5402747357148,"eloUpper":1442.546168723385,"eloLower":1434.5343807480444,"votes":35708,"rank":81,"rankUpper":null,"rankLower":null,"arenaName":"GPT-5.2 Chat-20260210","organization":"openai","updatedAt":"2026-11-09"},"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gpt-5-2-instant","providerId":"openai","slug":"gpt-5-2-instant","name":"GPT-5.2 Instant","provider":"OpenAI","description":"GPT-5.2 Instant is a reasoning model from OpenAI in the GPT-5.2 family. It has no scored result.","logoUrl":null,"releaseDate":"2025-12-11","contextWindowTokens":128000,"modalities":["text"],"openWeights":false,"details":null,"reasoning":true,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":1.5,"outputPerMillion":6,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gpt-5-2-pro","providerId":"openai","slug":"gpt-5-2-pro","name":"GPT-5.2 Pro","provider":"OpenAI","description":"GPT-5.2 Pro is a reasoning model from OpenAI in the GPT-5.2 family. It has 6 scored results in 2 categories. It has no index.","logoUrl":null,"releaseDate":"2025-12-11","contextWindowTokens":400000,"modalities":["image","text","file"],"openWeights":false,"details":{"apiId":"openai/gpt-5.2-pro","maxOutputTokens":128000,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":null,"weightsUrl":null},"reasoning":true,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":70.72477147354319,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":70.46578494697219},"capabilityCounts":{"reasoning":3,"math":3},"benchmarks":[{"benchmarkId":"arc-prize:v1-public-eval:overall","benchmarkName":"ARC-AGI-1 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":97.61,"unit":"percent","family":"arc-prize:v1-public-eval:overall","index":null,"aboveFit":false,"run":"xhigh effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":7.7201,"latency":null},{"benchmarkId":"arc-prize:v1-semi-private:overall","benchmarkName":"ARC-AGI-1 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":90.5,"unit":"percent","family":"arc-prize:v1-semi-private:overall","index":68.59347617937367,"aboveFit":false,"run":"xhigh effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":11.6542,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tiers-1-3-v2-private:overall","benchmarkName":"FrontierMath-Tiers-1-3-v2-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":74,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tiers-1-3-v2-private:overall","index":69.844722941428,"aboveFit":false,"run":"xhigh effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.6349537704451795,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-chess-puzzles:overall","benchmarkName":"Chess Puzzles","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":60,"unit":"percent","family":"epoch-ai:epoch-chess-puzzles:overall","index":79.52558743414471,"aboveFit":false,"run":"xhigh effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":4.92365963917331,"costPerTest":null,"latency":null},{"benchmarkId":"arc-prize:v2-semi-private:overall","benchmarkName":"ARC-AGI-2 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":54.16,"unit":"percent","family":"arc-prize:v2-semi-private:overall","index":64.05525080711115,"aboveFit":false,"run":"high effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":15.721,"latency":null},{"benchmarkId":"arc-prize:v2-public-eval:overall","benchmarkName":"ARC-AGI-2 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":51.68000000000001,"unit":"percent","family":"arc-prize:v2-public-eval:overall","index":null,"aboveFit":false,"run":"high effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":16.662,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tier-4-v2-private:overall","benchmarkName":"FrontierMath-Tier-4-v2-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":46,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tier-4-v2-private:overall","index":66.55940628170484,"aboveFit":false,"run":"xhigh effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":7.789619230571372,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-private:overall","benchmarkName":"FrontierMath-Tier-4-2025-07-01-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":31.3,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-private:overall","index":74.99322561778372,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":6.7,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":21,"outputPerMillion":168,"batch":{"inputPerMillion":10.5,"outputPerMillion":84},"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":9,"timeToFirstToken":4.831,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":25,"benchmarkCount":6,"categoryCount":2,"confidence":"unranked","state":"partial"},"hosts":[{"host":"digitalocean","name":"DigitalOcean","kind":"cloud","url":"https://docs.digitalocean.com/products/gradient-ai-platform/details/models/","inputPerMillion":21,"outputPerMillion":168,"cacheReadPerMillion":null,"contextTokens":400000,"maxOutputTokens":128000,"deprecated":false},{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":21,"outputPerMillion":168,"cacheReadPerMillion":null,"contextTokens":400000,"maxOutputTokens":128000,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:arc-prize-foundation","name":"ARC Prize Foundation","url":"https://arcprize.org","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gpt-5-3-instant","providerId":"openai","slug":"gpt-5-3-instant","name":"GPT-5.3 Instant","provider":"OpenAI","description":"GPT-5.3 Instant is a reasoning model from OpenAI in the GPT-5.3 family. It has no scored result.","logoUrl":null,"releaseDate":"2026-03-03","contextWindowTokens":400000,"modalities":["text"],"openWeights":false,"details":null,"reasoning":true,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":1.75,"outputPerMillion":14,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gpt-5-3-codex-spark","providerId":"openai","slug":"gpt-5-3-codex-spark","name":"GPT-5.3-Codex-Spark","provider":"OpenAI","description":"GPT-5.3-Codex-Spark is a reasoning model from OpenAI in the GPT-5.3 Codex family. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":"2026-02-12","contextWindowTokens":256000,"modalities":["text"],"openWeights":false,"details":null,"reasoning":true,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":61.97866734834783,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"agentic":1},"benchmarks":[{"benchmarkId":"mazeBench","benchmarkName":"MazeBench: visual spatial reasoning in a 3D open world","category":"agentic","score":0,"unit":"percent","family":"mazeBench","index":61.97866734834783,"aboveFit":false,"run":null,"source":"Jonathan Pappas and David Pappas","sourceUrl":"https://mazebench.com/blog?post=maze-bench-results","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":20,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gpt54image2","providerId":"openai","slug":"openai-gpt-5-4-image-2","name":"GPT-5.4 Image 2","provider":"OpenAI","description":"GPT-5.4 Image 2 is a model from OpenAI. It has no scored result.","logoUrl":null,"releaseDate":"2026-04-21T18:52:08.000Z","contextWindowTokens":272000,"modalities":["image","text","file"],"openWeights":null,"details":{"apiId":"openai/gpt-5.4-image-2","maxOutputTokens":128000,"knowledgeCutoff":null,"retiresAt":null,"tools":false,"structuredOutput":true,"cacheReadPerMillion":2,"weightsUrl":null},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":8,"outputPerMillion":15,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":8,"outputPerMillion":15,"cacheReadPerMillion":2,"contextTokens":272000,"maxOutputTokens":128000,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gpt54mini20260317none","providerId":"openai","slug":"openai-gpt-5-4-mini-none","name":"GPT-5.4 Mini None","provider":"OpenAI","description":"GPT-5.4 Mini None is a model from OpenAI. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":47.57589560951651,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-french:overall","benchmarkName":"EuroEval French","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":61.43375,"unit":"percent","family":"euroeval","index":55.571698705349945,"aboveFit":false,"run":"none effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/french","measuredAt":null,"stderr":0.37385241071845454,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-portuguese:overall","benchmarkName":"EuroEval Portuguese","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":57.781249999999986,"unit":"percent","family":"euroeval","index":50.61609662165674,"aboveFit":false,"run":"none effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/portuguese","measuredAt":null,"stderr":0.44550708467991845,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-italian:overall","benchmarkName":"EuroEval Italian","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":56.43625,"unit":"percent","family":"euroeval","index":48.79124109733989,"aboveFit":false,"run":"none effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/italian","measuredAt":null,"stderr":0.4831957419100462,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":55.77875,"unit":"percent","family":"euroeval","index":47.8991648837798,"aboveFit":false,"run":"none effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":0.6237888264468995,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-dutch:overall","benchmarkName":"EuroEval Dutch","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":55.30222222222222,"unit":"percent","family":"euroeval","index":47.25262633525323,"aboveFit":false,"run":"none effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/dutch","measuredAt":null,"stderr":0.4881244022774988,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-spanish:overall","benchmarkName":"EuroEval Spanish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":54.03750000000001,"unit":"percent","family":"euroeval","index":45.53668928399041,"aboveFit":false,"run":"none effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/spanish","measuredAt":null,"stderr":0.46086196414978753,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":53.381428571428565,"unit":"percent","family":"euroeval","index":44.64655131315394,"aboveFit":false,"run":"none effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.483813507670137,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":44.129999999999995,"unit":"percent","family":"euroeval","index":32.094491434793895,"aboveFit":false,"run":"none effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.5386252755859122,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gpt54nano20260317none","providerId":"openai","slug":"openai-gpt-5-4-nano-none","name":"GPT-5.4 Nano None","provider":"OpenAI","description":"GPT-5.4 Nano None is a model from OpenAI. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":28.99116293394588,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-french:overall","benchmarkName":"EuroEval French","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":45.54,"unit":"percent","family":"euroeval","index":34.00753700303685,"aboveFit":false,"run":"none effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/french","measuredAt":null,"stderr":0.46007302409508866,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-portuguese:overall","benchmarkName":"EuroEval Portuguese","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":45.106249999999996,"unit":"percent","family":"euroeval","index":33.41903805606849,"aboveFit":false,"run":"none effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/portuguese","measuredAt":null,"stderr":0.4690032649140089,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-italian:overall","benchmarkName":"EuroEval Italian","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":44.23875,"unit":"percent","family":"euroeval","index":32.24204016213179,"aboveFit":false,"run":"none effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/italian","measuredAt":null,"stderr":0.5743896216854897,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":43.17875000000001,"unit":"percent","family":"euroeval","index":30.80386406118319,"aboveFit":false,"run":"none effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":0.679471301822233,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-dutch:overall","benchmarkName":"EuroEval Dutch","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":40.50666666666667,"unit":"percent","family":"euroeval","index":27.17846180670858,"aboveFit":false,"run":"none effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/dutch","measuredAt":null,"stderr":0.5152070182023322,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-spanish:overall","benchmarkName":"EuroEval Spanish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":40.225,"unit":"percent","family":"euroeval","index":26.796304949695507,"aboveFit":false,"run":"none effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/spanish","measuredAt":null,"stderr":0.4940204955262484,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":37.84428571428572,"unit":"percent","family":"euroeval","index":23.5662234507321,"aboveFit":false,"run":"none effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.5438318317499352,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":33.8775,"unit":"percent","family":"euroeval","index":18.184207967835817,"aboveFit":false,"run":"none effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.7177830452163105,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gpt5420260305none","providerId":"openai","slug":"openai-gpt-5-4-none","name":"GPT-5.4 None","provider":"OpenAI","description":"GPT-5.4 None is a model from OpenAI. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":55.80819123767312,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-french:overall","benchmarkName":"EuroEval French","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":67.11125000000001,"unit":"percent","family":"euroeval","index":63.2747598498176,"aboveFit":false,"run":"none effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/french","measuredAt":null,"stderr":0.4652519478734076,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-portuguese:overall","benchmarkName":"EuroEval Portuguese","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":63.49875,"unit":"percent","family":"euroeval","index":58.37342856238661,"aboveFit":false,"run":"none effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/portuguese","measuredAt":null,"stderr":0.5517953651490741,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-italian:overall","benchmarkName":"EuroEval Italian","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":62.7025,"unit":"percent","family":"euroeval","index":57.29310052429196,"aboveFit":false,"run":"none effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/italian","measuredAt":null,"stderr":0.5453453607577495,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":62.375,"unit":"percent","family":"euroeval","index":56.8487583798951,"aboveFit":false,"run":"none effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":0.6826682484926335,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-dutch:overall","benchmarkName":"EuroEval Dutch","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":60.84111111111112,"unit":"percent","family":"euroeval","index":54.76762409545114,"aboveFit":false,"run":"none effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/dutch","measuredAt":null,"stderr":0.4146082488325576,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":59.51428571428571,"unit":"percent","family":"euroeval","index":52.96742732578515,"aboveFit":false,"run":"none effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.47213583310098295,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-spanish:overall","benchmarkName":"EuroEval Spanish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":56.35375,"unit":"percent","family":"euroeval","index":48.67930758004908,"aboveFit":false,"run":"none effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/spanish","measuredAt":null,"stderr":0.43327243161779866,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":53.21,"unit":"percent","family":"euroeval","index":44.4139621863159,"aboveFit":false,"run":"none effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.5618913373598137,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gpt55codex","providerId":"openai","slug":"openai-gpt-5-5-codex","name":"GPT-5.5 Codex","provider":"OpenAI","description":"GPT-5.5 Codex is a model from OpenAI. It has 4 scored results in 2 categories. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":54.65508815775005,"coding":64.17573375148157,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"coding":3,"agentic":1},"benchmarks":[{"benchmarkId":"vals-ai:swebench:overall","benchmarkName":"SWE-bench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":76.4,"unit":"percent","family":"swe-bench-verified","index":57.577326472145714,"aboveFit":false,"run":"Codex","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/swebench","measuredAt":"2026-09-01","stderr":1.901,"costPerTest":0.652332,"latency":108.544},{"benchmarkId":"vals-ai:skillsbench:overall","benchmarkName":"SkillsBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":62.555,"unit":"percent","family":"vals-ai:skillsbench:overall","index":71.70304269502577,"aboveFit":false,"run":"Codex","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/skillsbench","measuredAt":"2026-10-29","stderr":4.412,"costPerTest":1.572689,"latency":279.51},{"benchmarkId":"vals-ai:vibe-code:overall","benchmarkName":"Vibe Code Bench v1.1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":58.194,"unit":"percent","family":"vals-ai:vibe-code:overall","index":63.24683208727321,"aboveFit":false,"run":"Codex","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","measuredAt":"2026-11-09","stderr":4.933,"costPerTest":3.6264,"latency":708.51},{"benchmarkId":"vals-ai:terminal-bench-2-1:overall","benchmarkName":"Terminal-Bench 2.1","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":57.303,"unit":"percent","family":"terminal-bench-2-1","index":54.65508815775005,"aboveFit":false,"run":"Codex","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2-1","measuredAt":"2026-10-01","stderr":5.273,"costPerTest":0.645217,"latency":202.469}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":40,"benchmarkCount":4,"categoryCount":2,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gpt55factory","providerId":"openai","slug":"openai-gpt-5-5-factory","name":"GPT-5.5 Factory","provider":"OpenAI","description":"GPT-5.5 Factory is a model from OpenAI. It has 3 scored results in 2 categories. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":54.65508815775005,"coding":61.99753456978464,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"coding":2,"agentic":1},"benchmarks":[{"benchmarkId":"vals-ai:swebench:overall","benchmarkName":"SWE-bench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":76.2,"unit":"percent","family":"swe-bench-verified","index":57.42981495658897,"aboveFit":false,"run":"Factory","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/swebench","measuredAt":"2026-09-01","stderr":1.906,"costPerTest":1.129284,"latency":410.148},{"benchmarkId":"vals-ai:vibe-code:overall","benchmarkName":"Vibe Code Bench v1.1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":67.391,"unit":"percent","family":"vals-ai:vibe-code:overall","index":66.56525418298031,"aboveFit":false,"run":"Factory","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","measuredAt":"2026-11-09","stderr":4.75,"costPerTest":5.9268,"latency":1177.43},{"benchmarkId":"vals-ai:terminal-bench-2-1:overall","benchmarkName":"Terminal-Bench 2.1","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":57.303,"unit":"percent","family":"terminal-bench-2-1","index":54.65508815775005,"aboveFit":false,"run":"Factory","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2-1","measuredAt":"2026-10-01","stderr":5.273,"costPerTest":0.751531,"latency":389.265}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":40,"benchmarkCount":3,"categoryCount":2,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gpt55instant","providerId":"openai","slug":"openai-gpt-5-5-instant","name":"GPT-5.5 Instant","provider":"OpenAI","description":"GPT-5.5 Instant is a model from OpenAI. It has 4 scored results in 2 categories. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":52.88742688179897,"multilingual":null,"instruction":null,"math":47.56864677679706},"capabilityCounts":{"knowledge":1,"math":3},"benchmarks":[{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":82.51262626262627,"unit":"percent","family":"gpqa-diamond","index":52.88742688179897,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.2532782662252124,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":68.05555555555556,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":48.51340787784421,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":6.212711436196764,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tiers-1-3-v2-private:overall","benchmarkName":"FrontierMath-Tiers-1-3-v2-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":26.31578947368421,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tiers-1-3-v2-private:overall","index":44.74605712897326,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.6129809832490087,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tier-4-v2-private:overall","benchmarkName":"FrontierMath-Tier-4-v2-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":2.4390243902439024,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tier-4-v2-private:overall","index":49.446475323573715,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.4390243902439024,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":{"elo":1419.3698209983645,"eloUpper":1424.3094647513972,"eloLower":1414.4301772453318,"votes":26865,"rank":119,"rankUpper":null,"rankLower":null,"arenaName":"GPT-5.5 Instant","organization":"openai","updatedAt":"2026-11-09"},"quality":{"coveragePercent":25,"benchmarkCount":4,"categoryCount":2,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gpt-5-6-cyber","providerId":"openai","slug":"gpt-5-6-cyber","name":"GPT-5.6 Cyber","provider":"OpenAI","description":"GPT-5.6 Cyber is a reasoning model from OpenAI in the GPT-5.6 family. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":"2026-08-10","contextWindowTokens":null,"modalities":["text"],"openWeights":false,"details":null,"reasoning":true,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":78.46458506679647,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"agentic":1},"benchmarks":[{"benchmarkId":"exploitGym","benchmarkName":"ExploitGym","category":"agentic","score":32.1,"unit":"percent","family":"exploitGym","index":78.46458506679647,"aboveFit":false,"run":null,"source":"Zhun Wang, Nico Schiller, Hongwei Li, Srijiith Sesha Narayana, Milad Nasr, Nicholas Carlini, Xiangyu Qi, Eric Wallace, Elie Bursztein, Luca Invernizzi, Kurt Thomas, Yan Shoshitaishvili, Wenbo Guo, Jingxuan He, Thorsten Holz, Dawn Song","sourceUrl":"https://arxiv.org/abs/2605.11086","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":20,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gpt-5-6-luna-low","providerId":"openai","slug":"gpt-5-6-luna-low","name":"GPT-5.6 Luna (low)","provider":"OpenAI","description":"GPT-5.6 Luna (low) is a reasoning model from OpenAI. It has 2 scored results in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":1050000,"modalities":["file","image","text"],"openWeights":false,"details":{"apiId":"openai/gpt-5.6-luna","maxOutputTokens":128000,"knowledgeCutoff":"2026-02-16","retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.02,"weightsUrl":null},"reasoning":true,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":62.51251829043488,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"reasoning":2},"benchmarks":[{"benchmarkId":"jevBench15","benchmarkName":"jevBench15","category":"reasoning","score":22.371016433424646,"unit":"percent","family":"jevBench15","index":65.02116512871856,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"jevBench14","benchmarkName":"jevBench14","category":"reasoning","score":18.506333238992966,"unit":"percent","family":"jevBench14","index":60.003871452151216,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":0.19999999999999998,"outputPerMillion":1.2,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":51,"timeToFirstToken":1.747,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":15,"benchmarkCount":2,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gpt56luna20260730","providerId":"openai","slug":"openai-gpt-5-6-luna-2026-07-30","name":"GPT-5.6 Luna 2026-07-30","provider":"OpenAI","description":"GPT-5.6 Luna 2026-07-30 is a model from OpenAI. It has 2 scored results in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":67.60769509930395,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"reasoning":2},"benchmarks":[{"benchmarkId":"arc-prize:v1-public-eval:overall","benchmarkName":"ARC-AGI-1 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":91.25,"unit":"percent","family":"arc-prize:v1-public-eval:overall","index":null,"aboveFit":false,"run":"max effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.041845892999999974,"latency":null},{"benchmarkId":"arc-prize:v1-semi-private:overall","benchmarkName":"ARC-AGI-1 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":90.66666666666666,"unit":"percent","family":"arc-prize:v1-semi-private:overall","index":68.67009886320272,"aboveFit":false,"run":"max effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.06538362,"latency":null},{"benchmarkId":"arc-prize:v2-public-eval:overall","benchmarkName":"ARC-AGI-2 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":64.16666666666667,"unit":"percent","family":"arc-prize:v2-public-eval:overall","index":null,"aboveFit":false,"run":"max effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.1898519750000001,"latency":null},{"benchmarkId":"arc-prize:v2-semi-private:overall","benchmarkName":"ARC-AGI-2 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":59.583333333333336,"unit":"percent","family":"arc-prize:v2-semi-private:overall","index":66.54529133540518,"aboveFit":false,"run":"max effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.17671712499999995,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":15,"benchmarkCount":2,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:arc-prize-foundation","name":"ARC Prize Foundation","url":"https://arcprize.org","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gpt56lunapro","providerId":"openai","slug":"openai-gpt-5-6-luna-pro","name":"GPT-5.6 Luna Pro","provider":"OpenAI","description":"GPT-5.6 Luna Pro is a model from OpenAI. It has no scored result.","logoUrl":null,"releaseDate":"2026-07-09T09:54:27.000Z","contextWindowTokens":1050000,"modalities":["file","image","text"],"openWeights":null,"details":{"apiId":"openai/gpt-5.6-luna-pro","maxOutputTokens":128000,"knowledgeCutoff":"2026-02-16","retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.02,"weightsUrl":null},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":0.19999999999999998,"outputPerMillion":1.2,"batch":{"inputPerMillion":0.09999999999999999,"outputPerMillion":0.6},"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":96,"timeToFirstToken":7.227,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gpt56solexa","providerId":"openai","slug":"openai-gpt-5-6-sol-exa","name":"GPT-5.6 Sol Exa","provider":"OpenAI","description":"GPT-5.6 Sol Exa is a model from OpenAI. It has no scored result.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[{"benchmarkId":"vals-ai:web_search:overall","benchmarkName":"Vals Web Search Index","category":null,"domain":"Other","eligible":false,"ranked":true,"score":45.2405,"unit":"percent","family":"vals-ai:web_search:overall","index":null,"aboveFit":false,"run":"Exa","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/web_search","measuredAt":"2026-11-04","stderr":2.0060384467900905,"costPerTest":18.968851,"latency":530.7515}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gpt56solkeenable","providerId":"openai","slug":"openai-gpt-5-6-sol-keenable","name":"GPT-5.6 Sol Keenable","provider":"OpenAI","description":"GPT-5.6 Sol Keenable is a model from OpenAI. It has no scored result.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[{"benchmarkId":"vals-ai:web_search:overall","benchmarkName":"Vals Web Search Index","category":null,"domain":"Other","eligible":false,"ranked":true,"score":49.076695,"unit":"percent","family":"vals-ai:web_search:overall","index":null,"aboveFit":false,"run":"Keenable","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/web_search","measuredAt":"2026-11-04","stderr":2.020759,"costPerTest":8.842075,"latency":395.278729}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gpt56solpro","providerId":"openai","slug":"openai-gpt-5-6-sol-pro","name":"GPT-5.6 Sol Pro","provider":"OpenAI","description":"GPT-5.6 Sol Pro is a model from OpenAI. It has no scored result.","logoUrl":null,"releaseDate":"2026-07-09T09:54:14.000Z","contextWindowTokens":1050000,"modalities":["file","image","text"],"openWeights":null,"details":{"apiId":"openai/gpt-5.6-sol-pro","maxOutputTokens":128000,"knowledgeCutoff":"2026-02-16","retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.39999999999999997,"weightsUrl":null},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":4,"outputPerMillion":20,"batch":{"inputPerMillion":2,"outputPerMillion":10},"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":98,"timeToFirstToken":9.4995,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gpt56terrapro","providerId":"openai","slug":"openai-gpt-5-6-terra-pro","name":"GPT-5.6 Terra Pro","provider":"OpenAI","description":"GPT-5.6 Terra Pro is a model from OpenAI. It has no scored result.","logoUrl":null,"releaseDate":"2026-07-09T09:54:21.000Z","contextWindowTokens":1050000,"modalities":["file","image","text"],"openWeights":null,"details":{"apiId":"openai/gpt-5.6-terra-pro","maxOutputTokens":128000,"knowledgeCutoff":"2026-02-16","retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.19999999999999998,"weightsUrl":null},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":2,"outputPerMillion":12,"batch":{"inputPerMillion":1,"outputPerMillion":6},"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":120,"timeToFirstToken":13.264,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gpt6astrapro","providerId":"openai","slug":"openai-gpt-6-astra-pro","name":"GPT-6 Astra Pro","provider":"OpenAI","description":"GPT-6 Astra Pro is a model from OpenAI. It has no scored result.","logoUrl":null,"releaseDate":"2026-09-04T20:13:55.000Z","contextWindowTokens":1050000,"modalities":["file","image","text"],"openWeights":null,"details":{"apiId":"openai/gpt-6-astra-pro","maxOutputTokens":128000,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":1,"weightsUrl":null},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":10,"outputPerMillion":50,"batch":{"inputPerMillion":5,"outputPerMillion":25},"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":48,"timeToFirstToken":39.25,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gpt-6-cyber","providerId":"openai","slug":"gpt-6-cyber","name":"GPT-6 Cyber","provider":"OpenAI","description":"GPT-6 Cyber is a reasoning model from OpenAI in the GPT-6 family. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":"2026-11-05","contextWindowTokens":1050000,"modalities":["text"],"openWeights":false,"details":null,"reasoning":true,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":88.41638529287471,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"agentic":1},"benchmarks":[{"benchmarkId":"exploitGym","benchmarkName":"ExploitGym","category":"agentic","score":48.5,"unit":"percent","family":"exploitGym","index":88.41638529287471,"aboveFit":false,"run":null,"source":"Zhun Wang, Nico Schiller, Hongwei Li, Srijiith Sesha Narayana, Milad Nasr, Nicholas Carlini, Xiangyu Qi, Eric Wallace, Elie Bursztein, Luca Invernizzi, Kurt Thomas, Yan Shoshitaishvili, Wenbo Guo, Jingxuan He, Thorsten Holz, Dawn Song","sourceUrl":"https://arxiv.org/abs/2605.11086","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":20,"outputPerMillion":100,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":20,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gpt-6-luna-decisions","providerId":"openai","slug":"gpt-6-luna-decisions","name":"GPT-6 Luna (Decisions)","provider":"OpenAI","description":"GPT-6 Luna (Decisions) is a model from OpenAI. It has no scored result.","logoUrl":null,"releaseDate":"2026-10-06","contextWindowTokens":1050000,"modalities":["text"],"openWeights":false,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":0.1,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gpt-6-luna-low","providerId":"openai","slug":"gpt-6-luna-low","name":"GPT-6 Luna (low)","provider":"OpenAI","description":"GPT-6 Luna (low) is a reasoning model from OpenAI. It has 2 scored results in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":1050000,"modalities":["file","image","text"],"openWeights":null,"details":{"apiId":"openai/gpt-6-luna","maxOutputTokens":128000,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.01,"weightsUrl":null},"reasoning":true,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":75.64854203607625,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"reasoning":2},"benchmarks":[{"benchmarkId":"jevBench15","benchmarkName":"jevBench15","category":"reasoning","score":40.481958334046816,"unit":"percent","family":"jevBench15","index":75.98862523667387,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"jevBench14","benchmarkName":"jevBench14","category":"reasoning","score":35.11224166817657,"unit":"percent","family":"jevBench14","index":75.30845883547862,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":0.09999999999999999,"outputPerMillion":0.5,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":66,"timeToFirstToken":2.766,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":15,"benchmarkCount":2,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gpt-6-luna-medium","providerId":"openai","slug":"gpt-6-luna-medium","name":"GPT-6 Luna (medium)","provider":"OpenAI","description":"GPT-6 Luna (medium) is a reasoning model from OpenAI. It has 2 scored results in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":1050000,"modalities":["file","image","text"],"openWeights":null,"details":{"apiId":"openai/gpt-6-luna","maxOutputTokens":128000,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.01,"weightsUrl":null},"reasoning":true,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":74.27559769679368,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"reasoning":2},"benchmarks":[{"benchmarkId":"jevBench15","benchmarkName":"jevBench15","category":"reasoning","score":38.751615238726444,"unit":"percent","family":"jevBench15","index":74.94077974221341,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"jevBench14","benchmarkName":"jevBench14","category":"reasoning","score":33.26981698087274,"unit":"percent","family":"jevBench14","index":73.61041565137397,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":0.09999999999999999,"outputPerMillion":0.5,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":66,"timeToFirstToken":2.766,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":15,"benchmarkCount":2,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gpt6lunapro","providerId":"openai","slug":"openai-gpt-6-luna-pro","name":"GPT-6 Luna Pro","provider":"OpenAI","description":"GPT-6 Luna Pro is a model from OpenAI. It has no scored result.","logoUrl":null,"releaseDate":"2026-09-22T18:13:11.000Z","contextWindowTokens":1050000,"modalities":["file","image","text"],"openWeights":null,"details":{"apiId":"openai/gpt-6-luna-pro","maxOutputTokens":128000,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.01,"weightsUrl":null},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":0.09999999999999999,"outputPerMillion":0.5,"batch":{"inputPerMillion":0.049999999999999996,"outputPerMillion":0.25},"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":76,"timeToFirstToken":8.173,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gpt6solpro","providerId":"openai","slug":"openai-gpt-6-sol-pro","name":"GPT-6 Sol Pro","provider":"OpenAI","description":"GPT-6 Sol Pro is a model from OpenAI. It has no scored result.","logoUrl":null,"releaseDate":"2026-09-22T18:13:01.000Z","contextWindowTokens":1050000,"modalities":["file","image","text"],"openWeights":null,"details":{"apiId":"openai/gpt-6-sol-pro","maxOutputTokens":128000,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.19999999999999998,"weightsUrl":null},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":2,"outputPerMillion":10,"batch":{"inputPerMillion":1,"outputPerMillion":5},"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":79,"timeToFirstToken":6.523,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gpt61solpro","providerId":"openai","slug":"openai-gpt-6-1-sol-pro","name":"GPT-6.1 Sol Pro","provider":"OpenAI","description":"GPT-6.1 Sol Pro is a model from OpenAI. It has no scored result.","logoUrl":null,"releaseDate":"2026-09-29T17:28:06.000Z","contextWindowTokens":1050000,"modalities":["file","image","text"],"openWeights":null,"details":{"apiId":"openai/gpt-6.1-sol-pro","maxOutputTokens":128000,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.09999999999999999,"weightsUrl":null},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":2,"outputPerMillion":10,"batch":{"inputPerMillion":1,"outputPerMillion":5},"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":49,"timeToFirstToken":13.317,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gpt61terrapro","providerId":"openai","slug":"openai-gpt-6-1-terra-pro","name":"GPT-6.1 Terra Pro","provider":"OpenAI","description":"GPT-6.1 Terra Pro is a model from OpenAI. It has no scored result.","logoUrl":null,"releaseDate":"2026-10-26T18:15:56.000Z","contextWindowTokens":1050000,"modalities":["file","image","text"],"openWeights":null,"details":{"apiId":"openai/gpt-6.1-terra-pro","maxOutputTokens":128000,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.049999999999999996,"weightsUrl":null},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":1,"outputPerMillion":5,"batch":{"inputPerMillion":0.5,"outputPerMillion":2.5},"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":71,"timeToFirstToken":11.129,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gpt-live-1","providerId":"openai","slug":"gpt-live-1","name":"GPT-Live-1","provider":"OpenAI","description":"GPT-Live-1 is a standard model from OpenAI. It has no scored result.","logoUrl":null,"releaseDate":"2026-09-10","contextWindowTokens":null,"modalities":["text"],"openWeights":false,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"granite40micro","providerId":"ibm","slug":"ibm-granite-4-0-micro","name":"Granite 4.0 Micro","provider":"IBM","description":"Granite 4.0 Micro is a model from IBM. It has 3 scored results in 3 categories. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":5,"multilingual":19.24297305562986,"instruction":null,"math":10.535185448045532},"capabilityCounts":{"multilingual":1,"knowledge":1,"math":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":37.31714285714286,"unit":"percent","family":"euroeval","index":22.851011885705102,"aboveFit":false,"run":"granite-4.0-micro","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.27958423943201,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":31.998571428571427,"unit":"percent","family":"euroeval","index":15.634934225554623,"aboveFit":false,"run":"granite-4.0-h-micro","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.29937008698196926,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":28.28282828282828,"unit":"percent","family":"gpqa-diamond","index":5,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.8237830822136196,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":2.7777777777777777,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":10.535185448045532,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.4262647723679736,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":30,"benchmarkCount":3,"categoryCount":3,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"granite-4-0-1b","providerId":"ibm","slug":"granite-4-0-1b","name":"Granite-4.0-1B","provider":"IBM","description":"Granite-4.0-1B is a standard model from IBM in the Granite 4.0 1B family. It has 7 scored results in 5 categories. It has no index.","logoUrl":null,"releaseDate":"2025-10-28","contextWindowTokens":128000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":15.224164501727344,"reasoning":null,"multimodal":null,"knowledge":5,"multilingual":5,"instruction":19.01937444027874,"math":9.403919248008975},"capabilityCounts":{"instruction":1,"coding":1,"knowledge":3,"multilingual":1,"math":1},"benchmarks":[{"benchmarkId":"ifeval","benchmarkName":"Instruction-Following Eval","category":"instruction","score":77.38,"unit":"percent","family":"ifeval","index":19.01937444027874,"aboveFit":false,"run":null,"source":"Jeffrey Zhou, Tianjian Lu, Swaroop Mishra, Siddhartha Brahma, Sujoy Basu, Yi Luan, Denny Zhou, Le Hou","sourceUrl":"https://arxiv.org/abs/2311.07911","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"humaneval","benchmarkName":"Evaluating Large Language Models Trained on Code","category":"coding","score":74,"unit":"percent","family":"humaneval","index":15.224164501727344,"aboveFit":false,"run":null,"source":"Mark Chen, Jerry Tworek, Heewoo Jun, Qiming Yuan, Henrique Ponde de Oliveira Pinto, Jared Kaplan, Harri Edwards, Yuri Burda, Nicholas Joseph, Greg Brockman, Alex Ray, Raul Puri, Gretchen Krueger, Michael Petrov, Heidy Khlaaf, Girish Sastry, Pamela Mishkin, Brooke Chan, Scott Gray, Nick Ryder, Mikhail Pavlov, Alethea Power, Lukasz Kaiser, Mohammad Bavarian, Clemens Winter, Philippe Tillet, Felipe Petroski Such, Dave Cummings, Matthias Plappert, Fotios Chantzis, Elizabeth Barnes, Ariel Herbert-Voss, William Hebgen Guss, Alex Nichol, Alex Paino, Nikolas Tezak, Jie Tang, Igor Babuschkin, Suchir Balaji, Shantanu Jain, William Saunders, Christopher Hesse, Andrew N. Carr, Jan Leike, Josh Achiam, Vedant Misra, Evan Morikawa, Alec Radford, Matthew Knight, Miles Brundage, Mira Murati, Katie Mayer, Peter Welinder, Bob McGrew, Dario Amodei, Sam McCandlish, Ilya Sutskever, Wojciech Zaremba","sourceUrl":"https://arxiv.org/abs/2107.03374","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"bbh","benchmarkName":"BIG-Bench Hard","category":"reasoning","score":60.37,"unit":"percent","family":"bbh","index":null,"aboveFit":false,"run":null,"source":"Mirac Suzgun, Nathan Scales, Nathanael Schärli, Sebastian Gehrmann, Yi Tay, Hyung Won Chung, Aakanksha Chowdhery, Quoc V. Le, Ed H. Chi, Denny Zhou, Jason Wei","sourceUrl":"https://arxiv.org/abs/2210.09261","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmlu","benchmarkName":"Massive Multitask Language Understanding","category":"knowledge","score":59.39,"unit":"percent","family":"mmlu","index":5,"aboveFit":false,"run":null,"source":"Dan Hendrycks, Collin Burns, Steven Basart, Andy Zou, Mantas Mazeika, Dawn Song, Jacob Steinhardt","sourceUrl":"https://arxiv.org/abs/2009.03300","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mgsm","benchmarkName":"Multilingual Grade School Math","category":"multilingual","score":37.84,"unit":"percent","family":"mgsm","index":5,"aboveFit":false,"run":null,"source":"Freda Shi, Mirac Suzgun, Markus Freitag, Xuezhi Wang, Suraj Srivats, Soroush Vosoughi, Hyung Won Chung, Yi Tay, Sebastian Ruder, Denny Zhou, Dipanjan Das, Jason Wei","sourceUrl":"https://arxiv.org/abs/2210.03057","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmluPro","benchmarkName":"Massive Multitask Language Understanding Professional","category":"knowledge","score":34.02,"unit":"percent","family":"mmlu-pro","index":5,"aboveFit":false,"run":null,"source":"Yubo Wang, Xueguang Ma, Ge Zhang, Yuansheng Ni, Abhranil Chandra, Shiguang Guo, Weiming Ren, Aaran Arulraj, Xuan He, Ziyan Jiang, Tianle Li, Max Ku, Kai Wang, Alex Zhuang, Rongqi Fan, Xiang Yue, Wenhu Chen","sourceUrl":"https://arxiv.org/abs/2406.01574","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gpqa","benchmarkName":"Graduate-Level Google-Proof Q&A","category":"knowledge","score":29.91,"unit":"percent","family":"gpqa-diamond","index":5,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":23.98989898989899,"unit":"percent","family":"gpqa-diamond","index":5,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.6194510312980719,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":0.8333333333333334,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":9.403919248008975,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0.4700633956814717,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":55.00000000000001,"benchmarkCount":7,"categoryCount":5,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"grok-2-mini","providerId":"xai","slug":"grok-2-mini","name":"Grok 2 mini","provider":"xAI","description":"Grok 2 mini is a standard model from xAI in the Grok 2 family. It has 5 scored results in 3 categories. It has no index.","logoUrl":null,"releaseDate":"2024-08-13","contextWindowTokens":null,"modalities":["text"],"openWeights":false,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":26.36071141430358,"reasoning":null,"multimodal":30.38205983385258,"knowledge":31.210483132622937,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"knowledge":3,"coding":1,"multimodal":1},"benchmarks":[{"benchmarkId":"mmlu","benchmarkName":"Massive Multitask Language Understanding","category":"knowledge","score":86.2,"unit":"percent","family":"mmlu","index":34.45443199062757,"aboveFit":false,"run":null,"source":"Dan Hendrycks, Collin Burns, Steven Basart, Andy Zou, Mantas Mazeika, Dawn Song, Jacob Steinhardt","sourceUrl":"https://arxiv.org/abs/2009.03300","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"humaneval","benchmarkName":"Evaluating Large Language Models Trained on Code","category":"coding","score":85.7,"unit":"percent","family":"humaneval","index":26.36071141430358,"aboveFit":false,"run":null,"source":"Mark Chen, Jerry Tworek, Heewoo Jun, Qiming Yuan, Henrique Ponde de Oliveira Pinto, Jared Kaplan, Harri Edwards, Yuri Burda, Nicholas Joseph, Greg Brockman, Alex Ray, Raul Puri, Gretchen Krueger, Michael Petrov, Heidy Khlaaf, Girish Sastry, Pamela Mishkin, Brooke Chan, Scott Gray, Nick Ryder, Mikhail Pavlov, Alethea Power, Lukasz Kaiser, Mohammad Bavarian, Clemens Winter, Philippe Tillet, Felipe Petroski Such, Dave Cummings, Matthias Plappert, Fotios Chantzis, Elizabeth Barnes, Ariel Herbert-Voss, William Hebgen Guss, Alex Nichol, Alex Paino, Nikolas Tezak, Jie Tang, Igor Babuschkin, Suchir Balaji, Shantanu Jain, William Saunders, Christopher Hesse, Andrew N. Carr, Jan Leike, Josh Achiam, Vedant Misra, Evan Morikawa, Alec Radford, Matthew Knight, Miles Brundage, Mira Murati, Katie Mayer, Peter Welinder, Bob McGrew, Dario Amodei, Sam McCandlish, Ilya Sutskever, Wojciech Zaremba","sourceUrl":"https://arxiv.org/abs/2107.03374","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmluPro","benchmarkName":"Massive Multitask Language Understanding Professional","category":"knowledge","score":72,"unit":"percent","family":"mmlu-pro","index":35.19211024848268,"aboveFit":false,"run":null,"source":"Yubo Wang, Xueguang Ma, Ge Zhang, Yuansheng Ni, Abhranil Chandra, Shiguang Guo, Weiming Ren, Aaran Arulraj, Xuan He, Ziyan Jiang, Tianle Li, Max Ku, Kai Wang, Alex Zhuang, Rongqi Fan, Xiang Yue, Wenhu Chen","sourceUrl":"https://arxiv.org/abs/2406.01574","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmmu","benchmarkName":"Massive Multi-discipline Multimodal Understanding","category":"multimodal","score":63.2,"unit":"percent","family":"mmmu","index":30.38205983385258,"aboveFit":false,"run":null,"source":"MMMU authors","sourceUrl":"https://arxiv.org/abs/2401.05508","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gpqa","benchmarkName":"Graduate-Level Google-Proof Q&A","category":"knowledge","score":51,"unit":"percent","family":"gpqa-diamond","index":23.984907158758567,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":{"elo":1280.6998613026158,"eloUpper":1284.4045215732694,"eloLower":1276.995201031962,"votes":52567,"rank":277,"rankUpper":null,"rankLower":null,"arenaName":"Grok 2 mini-2024-08-13","organization":"xai","updatedAt":"2026-11-09"},"quality":{"coveragePercent":45,"benchmarkCount":5,"categoryCount":3,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"grok2vision1212","providerId":"xai","slug":"xai-grok-2-vision","name":"Grok 2 Vision","provider":"xAI","description":"Grok 2 Vision is a model from xAI. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":29.65866720590789,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"multimodal":1},"benchmarks":[{"benchmarkId":"vals-ai:mmmu:overall","benchmarkName":"MMMU Pro","category":"multimodal","domain":null,"eligible":true,"ranked":true,"score":57.25,"unit":"percent","family":"mmmu-pro","index":29.65866720590789,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmmu","measuredAt":"2026-09-01","stderr":1.189,"costPerTest":0.005402,"latency":18.355},{"benchmarkId":"vals-ai:mortgage_tax:overall","benchmarkName":"MortgageTax","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":3.458,"unit":"percent","family":"vals-ai:mortgage_tax:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mortgage_tax","measuredAt":"2026-09-01","stderr":0.719,"costPerTest":0.003772,"latency":0}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":10,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"grok3minifasthighreasoning","providerId":"xai","slug":"xai-grok-3-mini-fast-high-reasoning","name":"Grok 3 Mini Fast High Reasoning","provider":"xAI","description":"Grok 3 Mini Fast High Reasoning is a model from xAI. It has 5 scored results in 3 categories. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":52.58664297808755,"reasoning":null,"multimodal":null,"knowledge":48.468350929894655,"multilingual":null,"instruction":null,"math":48.162431247210364},"capabilityCounts":{"math":2,"knowledge":2,"coding":1},"benchmarks":[{"benchmarkId":"vals-ai:math500:overall","benchmarkName":"MATH 500","category":"math","domain":null,"eligible":true,"ranked":true,"score":94.2,"unit":"percent","family":"math-500","index":43.673406867271986,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/math500","measuredAt":"2026-01-09","stderr":1.045,"costPerTest":0.001879,"latency":22.766},{"benchmarkId":"vals-ai:mgsm:overall","benchmarkName":"MGSM","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":90.436,"unit":"percent","family":"vals-ai:mgsm:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mgsm","measuredAt":"2026-01-09","stderr":1.84,"costPerTest":0.000798,"latency":10.068},{"benchmarkId":"vals-ai:medqa:overall","benchmarkName":"MedQA","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":90.1,"unit":"percent","family":"vals-ai:medqa:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medqa","measuredAt":"2026-04-16","stderr":0.273,"costPerTest":0.000765,"latency":7.065},{"benchmarkId":"vals-ai:aime:overall","benchmarkName":"AIME","category":"math","domain":null,"eligible":true,"ranked":true,"score":85,"unit":"percent","family":"vals-ai:aime:overall","index":52.65145562714874,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/aime","measuredAt":"2026-04-16","stderr":0.534,"costPerTest":0.008281,"latency":102.246},{"benchmarkId":"vals-ai:legal_bench:overall","benchmarkName":"LegalBench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":83.143,"unit":"percent","family":"vals-ai:legal_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","measuredAt":"2026-10-13","stderr":0.42,"costPerTest":null,"latency":5.095},{"benchmarkId":"vals-ai:mmlu_pro:overall","benchmarkName":"MMLU Pro","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":81.374,"unit":"percent","family":"mmlu-pro","index":47.00222820952526,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","measuredAt":"2026-09-01","stderr":0.382,"costPerTest":0.001758,"latency":16.888},{"benchmarkId":"vals-ai:gpqa:overall","benchmarkName":"GPQA Diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":79.293,"unit":"percent","family":"gpqa-diamond","index":49.93447365026405,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","measuredAt":"2026-09-01","stderr":2.044,"costPerTest":0.00388,"latency":40.618},{"benchmarkId":"vals-ai:lcb:overall","benchmarkName":"LiveCodeBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":76.22,"unit":"percent","family":"livecodebench","index":52.58664297808755,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","measuredAt":"2026-09-01","stderr":1.084,"costPerTest":0.008057,"latency":213.662},{"benchmarkId":"vals-ai:tax_eval_v2:overall","benchmarkName":"TaxEval v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":71.136,"unit":"percent","family":"vals-ai:tax_eval_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","measuredAt":"2026-09-01","stderr":0.889,"costPerTest":0.002142,"latency":25.447},{"benchmarkId":"vals-ai:corp_fin_v2:overall","benchmarkName":"CorpFin v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":61.111,"unit":"percent","family":"vals-ai:corp_fin_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","measuredAt":"2026-08-12","stderr":0.957,"costPerTest":0.017292,"latency":14.386}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":45,"benchmarkCount":5,"categoryCount":3,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"grok3minifastlowreasoning","providerId":"xai","slug":"xai-grok-3-mini-fast-low-reasoning","name":"Grok 3 Mini Fast Low Reasoning","provider":"xAI","description":"Grok 3 Mini Fast Low Reasoning is a model from xAI. It has 5 scored results in 3 categories. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":44.32656904336529,"reasoning":null,"multimodal":null,"knowledge":44.71468695496347,"multilingual":null,"instruction":null,"math":32.34006629492711},"capabilityCounts":{"knowledge":2,"math":2,"coding":1},"benchmarks":[{"benchmarkId":"vals-ai:mgsm:overall","benchmarkName":"MGSM","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":90.364,"unit":"percent","family":"vals-ai:mgsm:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mgsm","measuredAt":"2026-01-09","stderr":1.836,"costPerTest":0.000509,"latency":6.56},{"benchmarkId":"vals-ai:medqa:overall","benchmarkName":"MedQA","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":88.65,"unit":"percent","family":"vals-ai:medqa:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medqa","measuredAt":"2026-04-16","stderr":0.29,"costPerTest":0.000558,"latency":4.883},{"benchmarkId":"vals-ai:legal_bench:overall","benchmarkName":"LegalBench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":81.925,"unit":"percent","family":"vals-ai:legal_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","measuredAt":"2026-10-13","stderr":0.434,"costPerTest":null,"latency":3.44},{"benchmarkId":"vals-ai:mmlu_pro:overall","benchmarkName":"MMLU Pro","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":80.011,"unit":"percent","family":"mmlu-pro","index":45.28501135643149,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","measuredAt":"2026-09-01","stderr":0.39,"costPerTest":0.001026,"latency":7.426},{"benchmarkId":"vals-ai:gpqa:overall","benchmarkName":"GPQA Diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":72.98,"unit":"percent","family":"gpqa-diamond","index":44.14436255349545,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","measuredAt":"2026-09-01","stderr":2.233,"costPerTest":0.001599,"latency":14.234},{"benchmarkId":"vals-ai:tax_eval_v2:overall","benchmarkName":"TaxEval v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":72.976,"unit":"percent","family":"vals-ai:tax_eval_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","measuredAt":"2026-09-01","stderr":0.87,"costPerTest":0.00112,"latency":11.728},{"benchmarkId":"vals-ai:aime:overall","benchmarkName":"AIME","category":"math","domain":null,"eligible":true,"ranked":true,"score":70.625,"unit":"percent","family":"vals-ai:aime:overall","index":46.36892282089937,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/aime","measuredAt":"2026-04-16","stderr":0.562,"costPerTest":0.00293,"latency":31.397},{"benchmarkId":"vals-ai:math500:overall","benchmarkName":"MATH 500","category":"math","domain":null,"eligible":true,"ranked":true,"score":70.2,"unit":"percent","family":"math-500","index":18.311209768954853,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/math500","measuredAt":"2026-01-09","stderr":2.046,"costPerTest":0.000893,"latency":9.885},{"benchmarkId":"vals-ai:lcb:overall","benchmarkName":"LiveCodeBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":66.265,"unit":"percent","family":"livecodebench","index":44.32656904336529,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","measuredAt":"2026-09-01","stderr":1.141,"costPerTest":0.002154,"latency":30.707},{"benchmarkId":"vals-ai:corp_fin_v2:overall","benchmarkName":"CorpFin v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":59.479,"unit":"percent","family":"vals-ai:corp_fin_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","measuredAt":"2026-08-12","stderr":0.96,"costPerTest":0.01706,"latency":10.516}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":45,"benchmarkCount":5,"categoryCount":3,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"grok4refinement","providerId":"xai","slug":"xai-grok-4-refinement","name":"Grok 4 (Refinement)","provider":"xAI","description":"Grok 4 (Refinement) is a model from xAI. It has 2 scored results in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":58.143896827966664,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"reasoning":2},"benchmarks":[{"benchmarkId":"arc-prize:v1-semi-private:overall","benchmarkName":"ARC-AGI-1 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":79.60000000000001,"unit":"percent","family":"arc-prize:v1-semi-private:overall","index":63.58235265695275,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":8.42,"latency":null},{"benchmarkId":"arc-prize:v2-semi-private:overall","benchmarkName":"ARC-AGI-2 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":29.439999999999998,"unit":"percent","family":"arc-prize:v2-semi-private:overall","index":52.70544099898058,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":30.4,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":15,"benchmarkCount":2,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:arc-prize-foundation","name":"ARC Prize Foundation","url":"https://arcprize.org","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"grok4thinking","providerId":"xai","slug":"xai-grok-4-thinking","name":"Grok 4 (Thinking)","provider":"xAI","description":"Grok 4 (Thinking) is a model from xAI. It has 2 scored results in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":52.08057854791522,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"reasoning":2},"benchmarks":[{"benchmarkId":"arc-prize:v1-public-eval:overall","benchmarkName":"ARC-AGI-1 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":73.74000000000001,"unit":"percent","family":"arc-prize:v1-public-eval:overall","index":null,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.8179,"latency":null},{"benchmarkId":"arc-prize:v1-semi-private:overall","benchmarkName":"ARC-AGI-1 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":66.67,"unit":"percent","family":"arc-prize:v1-semi-private:overall","index":57.637964845493805,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":1.0136,"latency":null},{"benchmarkId":"arc-prize:v2-public-eval:overall","benchmarkName":"ARC-AGI-2 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":21.15,"unit":"percent","family":"arc-prize:v2-public-eval:overall","index":null,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":2.0759,"latency":null},{"benchmarkId":"arc-prize:v2-semi-private:overall","benchmarkName":"ARC-AGI-2 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":15.975,"unit":"percent","family":"arc-prize:v2-semi-private:overall","index":46.523192250336635,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":2.1659,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":15,"benchmarkCount":2,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:arc-prize-foundation","name":"ARC Prize Foundation","url":"https://arcprize.org","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"grok4fast","providerId":"xai","slug":"xai-grok-4-fast","name":"Grok 4 fast","provider":"xAI","description":"Grok 4 fast is a model from xAI. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":13.220809464746516,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"agentic":1},"benchmarks":[{"benchmarkId":"tau2-bench:banking-knowledge:overall","benchmarkName":"τ²-bench Banking","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":15.72,"unit":"percent","family":"tau2-bench","index":13.220809464746516,"aboveFit":false,"run":"high effort · Sierra","source":"Sierra Research","sourceUrl":"https://github.com/sierra-research/tau2-bench","measuredAt":"2026-10-20","stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":20,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:sierra-research","name":"Sierra Research","url":"https://github.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"grok4fastnonreasoning","providerId":"xai","slug":"xai-grok-4-fast-non-reasoning","name":"Grok 4 Fast Non Reasoning","provider":"xAI","description":"Grok 4 Fast Non Reasoning is a model from xAI. It has 7 scored results in 5 categories. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":31.786955049242604,"coding":34.819354506180886,"reasoning":null,"multimodal":37.675366759041935,"knowledge":33.64446517759882,"multilingual":null,"instruction":null,"math":30.070612322767087},"capabilityCounts":{"knowledge":2,"multimodal":1,"coding":2,"math":1,"agentic":1},"benchmarks":[{"benchmarkId":"vals-ai:mgsm:overall","benchmarkName":"MGSM","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":88,"unit":"percent","family":"vals-ai:mgsm:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mgsm","measuredAt":"2026-01-09","stderr":1.999,"costPerTest":0.000338,"latency":1.811},{"benchmarkId":"vals-ai:medscribe:overall","benchmarkName":"MedScribe","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":79.722,"unit":"percent","family":"vals-ai:medscribe:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medscribe","measuredAt":"2026-10-29","stderr":1.871,"costPerTest":0.002056,"latency":8.482},{"benchmarkId":"vals-ai:legal_bench:overall","benchmarkName":"LegalBench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":78.396,"unit":"percent","family":"vals-ai:legal_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","measuredAt":"2026-10-13","stderr":0.472,"costPerTest":null,"latency":0.933},{"benchmarkId":"vals-ai:medqa:overall","benchmarkName":"MedQA","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":75.358,"unit":"percent","family":"vals-ai:medqa:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medqa","measuredAt":"2026-04-16","stderr":0.393,"costPerTest":0.000481,"latency":2.416},{"benchmarkId":"vals-ai:tax_eval_v2:overall","benchmarkName":"TaxEval v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":71.581,"unit":"percent","family":"vals-ai:tax_eval_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","measuredAt":"2026-09-01","stderr":0.783,"costPerTest":0.000571,"latency":4.25},{"benchmarkId":"vals-ai:mmlu_pro:overall","benchmarkName":"MMLU Pro","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":70.342,"unit":"percent","family":"mmlu-pro","index":33.10322870598122,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","measuredAt":"2026-09-01","stderr":0.431,"costPerTest":0.001084,"latency":2.135},{"benchmarkId":"vals-ai:mmmu:overall","benchmarkName":"MMMU Pro","category":"multimodal","domain":null,"eligible":true,"ranked":true,"score":63.41,"unit":"percent","family":"mmmu-pro","index":37.675366759041935,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmmu","measuredAt":"2026-09-01","stderr":1.158,"costPerTest":0.001555,"latency":7.209},{"benchmarkId":"vals-ai:gpqa:overall","benchmarkName":"GPQA Diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":62.122,"unit":"percent","family":"gpqa-diamond","index":34.18570164921641,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","measuredAt":"2026-09-01","stderr":2.429,"costPerTest":0.001098,"latency":3.668},{"benchmarkId":"vals-ai:corp_fin_v2:overall","benchmarkName":"CorpFin v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":58.392,"unit":"percent","family":"vals-ai:corp_fin_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","measuredAt":"2026-08-12","stderr":0.967,"costPerTest":0.039974,"latency":9.37},{"benchmarkId":"vals-ai:lcb:overall","benchmarkName":"LiveCodeBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":46.095,"unit":"percent","family":"livecodebench","index":27.590688454380032,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","measuredAt":"2026-09-01","stderr":1.146,"costPerTest":0.001012,"latency":8.958},{"benchmarkId":"vals-ai:aime:overall","benchmarkName":"AIME","category":"math","domain":null,"eligible":true,"ranked":true,"score":33.333,"unit":"percent","family":"vals-ai:aime:overall","index":30.070612322767087,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/aime","measuredAt":"2026-04-16","stderr":1.068,"costPerTest":0.000805,"latency":5.879},{"benchmarkId":"vals-ai:medcode:overall","benchmarkName":"MedCode","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":30.036,"unit":"percent","family":"vals-ai:medcode:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medcode","measuredAt":"2026-10-29","stderr":1.974,"costPerTest":0.002149,"latency":18.211},{"benchmarkId":"vals-ai:mortgage_tax:overall","benchmarkName":"MortgageTax","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":25.318,"unit":"percent","family":"vals-ai:mortgage_tax:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mortgage_tax","measuredAt":"2026-09-01","stderr":0.867,"costPerTest":0.003483,"latency":4.508},{"benchmarkId":"vals-ai:terminal-bench:overall","benchmarkName":"Terminal-Bench 1.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":18.75,"unit":"percent","family":"vals-ai:terminal-bench:overall","index":31.786955049242604,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench","measuredAt":"2026-01-12","stderr":4.364,"costPerTest":0.311069,"latency":65.358},{"benchmarkId":"vals-ai:ioi-v1:overall","benchmarkName":"IOI v1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":3.834,"unit":"percent","family":"vals-ai:ioi-v1:overall","index":42.048020557981744,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/ioi-v1","measuredAt":"2026-08-09","stderr":2.848,"costPerTest":0.12675,"latency":577.632}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":{"elo":1406.9616677427762,"eloUpper":1414.7225267940644,"eloLower":1399.2008086914884,"votes":6330,"rank":147,"rankUpper":null,"rankLower":null,"arenaName":"Grok 4 Fast-non-reasoning","organization":"xai","updatedAt":"2026-11-09"},"quality":{"coveragePercent":75,"benchmarkCount":7,"categoryCount":5,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"grok4heavy","providerId":"xai","slug":"xai-grok-4-heavy","name":"Grok 4 Heavy","provider":"xAI","description":"Grok 4 Heavy is a model from xAI. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":45.314977702005415},"capabilityCounts":{"math":1},"benchmarks":[{"benchmarkId":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-private:overall","benchmarkName":"FrontierMath-Tier-4-2025-07-01-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":2.08,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-private:overall","index":45.314977702005415,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.1,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":10,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"grok-4-1","providerId":"xai","slug":"grok-4-1","name":"Grok 4.1","provider":"xAI","description":"Grok 4.1 is a standard model from xAI. It has no scored result.","logoUrl":null,"releaseDate":"2025-11-17","contextWindowTokens":1000000,"modalities":["text"],"openWeights":false,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[{"benchmarkId":"researchClawBench","benchmarkName":"ResearchClawBench","category":"agentic","score":13.5,"unit":"percent","family":"researchClawBench","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":{"elo":1436.1979805611008,"eloUpper":1439.332751923956,"eloLower":1433.0632091982452,"votes":68194,"rank":87,"rankUpper":null,"rankLower":null,"arenaName":"Grok 4.1","organization":"xai","updatedAt":"2026-11-09"},"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"grok42","providerId":"xai","slug":"xai-grok-4-2","name":"Grok 4.2","provider":"xAI","description":"Grok 4.2 is a model from xAI. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":14.76029908440465,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"agentic":1},"benchmarks":[{"benchmarkId":"tau2-bench:banking-knowledge:overall","benchmarkName":"τ²-bench Banking","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":18.04,"unit":"percent","family":"tau2-bench","index":14.76029908440465,"aboveFit":false,"run":"high effort · Sierra","source":"Sierra Research","sourceUrl":"https://github.com/sierra-research/tau2-bench","measuredAt":"2026-10-20","stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":20,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:sierra-research","name":"Sierra Research","url":"https://github.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"grok420betareasoning","providerId":"xai","slug":"xai-grok-4-20-beta-reasoning","name":"Grok 4.20 (Beta Reasoning)","provider":"xAI","description":"Grok 4.20 (Beta Reasoning) is a model from xAI. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":65.34382502626265,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"reasoning":1},"benchmarks":[{"benchmarkId":"arc-prize:v3-semi-private:overall","benchmarkName":"ARC-AGI-3 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":0.09,"unit":"percent","family":"arc-prize:v3-semi-private:overall","index":65.34382502626265,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":{"elo":1451.2779689603399,"eloUpper":1454.944498163924,"eloLower":1447.6114397567553,"votes":65713,"rank":51,"rankUpper":null,"rankLower":null,"arenaName":"Grok 4.20 Beta-reasoning-0309","organization":"xai","updatedAt":"2026-11-09"},"quality":{"coveragePercent":15,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:arc-prize-foundation","name":"ARC Prize Foundation","url":"https://arcprize.org","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"grok-4-20-multi-agent-beta","providerId":"xai","slug":"grok-4-20-multi-agent-beta","name":"Grok 4.20 Multi-agent","provider":"xAI","description":"Grok 4.20 Multi-agent is a reasoning model from xAI in the Grok 4.20 family. It has no scored result.","logoUrl":null,"releaseDate":"2026-03-10","contextWindowTokens":2000000,"modalities":["text","image","file"],"openWeights":false,"details":{"apiId":"x-ai/grok-4.20-multi-agent","maxOutputTokens":1800000,"knowledgeCutoff":"2025-09-01","retiresAt":null,"tools":false,"structuredOutput":true,"cacheReadPerMillion":0.19999999999999998,"weightsUrl":null},"reasoning":true,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":1.25,"outputPerMillion":2.5,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":311.5,"timeToFirstToken":32.4955,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":{"elo":1450.995456316853,"eloUpper":1454.685370381177,"eloLower":1447.3055422525294,"votes":64252,"rank":53,"rankUpper":null,"rankLower":null,"arenaName":"Grok 4.20 Multi-Agent Beta-0309","organization":"xai","updatedAt":"2026-11-09"},"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"grok43beta","providerId":"xai","slug":"xai-grok-4-3-beta","name":"Grok 4.3 Beta","provider":"xAI","description":"Grok 4.3 Beta is a model from xAI. It has 6 scored results in 3 categories. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":47.64694923784066,"multimodal":null,"knowledge":55.12762078141861,"multilingual":null,"instruction":null,"math":56.96114705954547},"capabilityCounts":{"math":3,"knowledge":2,"reasoning":1},"benchmarks":[{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":93.33333333333333,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":63.219868478319434,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":3.041173683586219,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":88.82575757575758,"unit":"percent","family":"gpqa-diamond","index":58.67765841538347,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.9638404200482171,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tiers-1-3-v2-private:overall","benchmarkName":"FrontierMath-Tiers-1-3-v2-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":42.80701754385965,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tiers-1-3-v2-private:overall","index":53.426243245127594,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.9360921879029935,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-simpleqa-verified:overall","benchmarkName":"SimpleQA Verified","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":33.2,"unit":"percent","family":"epoch-ai:epoch-simpleqa-verified:overall","index":51.57758314745375,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.4899597242811566,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-chess-puzzles:overall","benchmarkName":"Chess Puzzles","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":18,"unit":"percent","family":"epoch-ai:epoch-chess-puzzles:overall","index":47.64694923784066,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":3.8612291966536914,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tier-4-v2-private:overall","benchmarkName":"FrontierMath-Tier-4-v2-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":14.634146341463413,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tier-4-v2-private:overall","index":54.23732945518937,"aboveFit":false,"run":"high effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":5.588506945068097,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":40,"benchmarkCount":6,"categoryCount":3,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"grok45exa","providerId":"xai","slug":"xai-grok-4-5-exa","name":"Grok 4.5 Exa","provider":"xAI","description":"Grok 4.5 Exa is a model from xAI. It has no scored result.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[{"benchmarkId":"vals-ai:web_search:overall","benchmarkName":"Vals Web Search Index","category":null,"domain":"Other","eligible":false,"ranked":true,"score":38.747,"unit":"percent","family":"vals-ai:web_search:overall","index":null,"aboveFit":false,"run":"Exa","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/web_search","measuredAt":"2026-11-04","stderr":1.945497365713971,"costPerTest":0.576384,"latency":100.9255}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"groklatest","providerId":"xai","slug":"xai-grok-latest","name":"Grok Latest","provider":"xAI","description":"Grok Latest is a model from xAI. It has no scored result.","logoUrl":null,"releaseDate":"2026-07-08T14:02:40.000Z","contextWindowTokens":500000,"modalities":["text","image","file"],"openWeights":null,"details":{"apiId":"~x-ai/grok-latest","maxOutputTokens":450000,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.5,"weightsUrl":null},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":2,"outputPerMillion":6,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":51,"timeToFirstToken":5.098,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"grok-realtime","providerId":"xai","slug":"grok-realtime","name":"Grok Realtime","provider":"xAI","description":"Grok Realtime is a standard model from xAI. It has no scored result.","logoUrl":null,"releaseDate":"2025-12-17","contextWindowTokens":null,"modalities":["text"],"openWeights":false,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"hark-handoff","providerId":"hark","slug":"hark-handoff","name":"Hark Handoff","provider":"Hark","description":"Hark Handoff is a reasoning model from Hark. It has no scored result.","logoUrl":null,"releaseDate":"2026-08-05","contextWindowTokens":null,"modalities":["text"],"openWeights":false,"details":null,"reasoning":true,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"hermes2thetallama370b","providerId":"nous-research","slug":"nous-research-hermes-2-theta-llama-3-70b","name":"Hermes 2 Theta Llama-3 70B","provider":"Nous Research","description":"Hermes 2 Theta Llama-3 70B is a model from Nous Research. It has 3 scored results in 2 categories. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":11.574124889673541,"multilingual":null,"instruction":null,"math":10.195893529650235},"capabilityCounts":{"knowledge":1,"math":2},"benchmarks":[{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":37.46843434343434,"unit":"percent","family":"gpqa-diamond","index":11.574124889673541,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.4803009993948306,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-math-level-5:overall","benchmarkName":"MATH level 5","category":"math","domain":null,"eligible":true,"ranked":true,"score":22.686933534743204,"unit":"percent","family":"epoch-ai:epoch-math-level-5:overall","index":10.01821106840302,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0.8127708089237323,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":2.5,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":10.37357599089745,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.2309149097933272,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":25,"benchmarkCount":3,"categoryCount":2,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"hermes3llama31405b","providerId":"nous-research","slug":"nous-research-hermes-3-405b-instruct","name":"Hermes 3 405B Instruct","provider":"Nous Research","description":"Hermes 3 405B Instruct is a model from Nous Research. It has no scored result.","logoUrl":null,"releaseDate":"2024-08-16T00:00:00.000Z","contextWindowTokens":131072,"modalities":["text"],"openWeights":true,"details":{"apiId":"nousresearch/hermes-3-llama-3.1-405b","maxOutputTokens":16384,"knowledgeCutoff":"2023-12-31","retiresAt":"2026-11-02","tools":false,"structuredOutput":true,"cacheReadPerMillion":null,"weightsUrl":"https://huggingface.co/NousResearch/Hermes-3-Llama-3.1-405B"},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":1,"outputPerMillion":1,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":27,"timeToFirstToken":0.348,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"hermes3llama3170b","providerId":"nous-research","slug":"nous-research-hermes-3-70b-instruct","name":"Hermes 3 70B Instruct","provider":"Nous Research","description":"Hermes 3 70B Instruct is a model from Nous Research. It has no scored result.","logoUrl":null,"releaseDate":"2024-08-18T00:00:00.000Z","contextWindowTokens":131072,"modalities":["text"],"openWeights":true,"details":{"apiId":"nousresearch/hermes-3-llama-3.1-70b","maxOutputTokens":16384,"knowledgeCutoff":"2023-12-31","retiresAt":"2026-11-02","tools":false,"structuredOutput":true,"cacheReadPerMillion":null,"weightsUrl":"https://huggingface.co/NousResearch/Hermes-3-Llama-3.1-70B"},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":0.7,"outputPerMillion":0.7,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":30,"timeToFirstToken":0.68,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"hermes4405b","providerId":"nous-research","slug":"nous-research-hermes-4-405b","name":"Hermes 4 405B","provider":"Nous Research","description":"Hermes 4 405B is a model from Nous Research. It has no scored result.","logoUrl":null,"releaseDate":"2025-08-26T19:11:03.000Z","contextWindowTokens":131072,"modalities":["text"],"openWeights":true,"details":{"apiId":"nousresearch/hermes-4-405b","maxOutputTokens":117964,"knowledgeCutoff":"2024-08-31","retiresAt":null,"tools":false,"structuredOutput":true,"cacheReadPerMillion":0.19999999999999998,"weightsUrl":"https://huggingface.co/NousResearch/Hermes-4-405B"},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":1,"outputPerMillion":3,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":44,"timeToFirstToken":0.26,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[{"host":"nebius","name":"Nebius","kind":"host","url":"https://docs.tokenfactory.nebius.com/","inputPerMillion":1,"outputPerMillion":3,"cacheReadPerMillion":1,"contextTokens":131072,"maxOutputTokens":8192,"deprecated":false},{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":1,"outputPerMillion":3,"cacheReadPerMillion":0.2,"contextTokens":131072,"maxOutputTokens":117964,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"holo2-235b-a22b","providerId":"h-company","slug":"holo2-235b-a22b","name":"Holo2-235B-A22B","provider":"H Company","description":"Holo2-235B-A22B is a standard model from H Company in the Holo2 family. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":"2026-02-03","contextWindowTokens":262000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":52.29381885495052,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"multimodal":1},"benchmarks":[{"benchmarkId":"screenSpotPro","benchmarkName":"ScreenSpot Pro","category":"multimodal","score":70.6,"unit":"percent","family":"screenSpotPro","index":52.29381885495052,"aboveFit":false,"run":null,"source":"Kaixin Li, Ziyang Meng, Hongzhan Lin, Ziyang Luo, Yuchen Tian, Jing Ma, Zhiyong Huang, Tat-Seng Chua","sourceUrl":"https://arxiv.org/abs/2504.07981","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":10,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"holo2-30b-a3b","providerId":"h-company","slug":"holo2-30b-a3b","name":"Holo2-30B-A3B","provider":"H Company","description":"Holo2-30B-A3B is a standard model from H Company in the Holo2 family. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":"2025-11-13","contextWindowTokens":262000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":48.366590840664806,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"multimodal":1},"benchmarks":[{"benchmarkId":"screenSpotPro","benchmarkName":"ScreenSpot Pro","category":"multimodal","score":66.1,"unit":"percent","family":"screenSpotPro","index":48.366590840664806,"aboveFit":false,"run":null,"source":"Kaixin Li, Ziyang Meng, Hongzhan Lin, Ziyang Luo, Yuchen Tian, Jing Ma, Zhiyong Huang, Tat-Seng Chua","sourceUrl":"https://arxiv.org/abs/2504.07981","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":10,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"holo2-4b","providerId":"h-company","slug":"holo2-4b","name":"Holo2-4B","provider":"H Company","description":"Holo2-4B is a standard model from H Company in the Holo2 family. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":"2025-11-13","contextWindowTokens":262000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":40.59940654574417,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"multimodal":1},"benchmarks":[{"benchmarkId":"screenSpotPro","benchmarkName":"ScreenSpot Pro","category":"multimodal","score":57.2,"unit":"percent","family":"screenSpotPro","index":40.59940654574417,"aboveFit":false,"run":null,"source":"Kaixin Li, Ziyang Meng, Hongzhan Lin, Ziyang Luo, Yuchen Tian, Jing Ma, Zhiyong Huang, Tat-Seng Chua","sourceUrl":"https://arxiv.org/abs/2504.07981","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":10,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"holo2-8b","providerId":"h-company","slug":"holo2-8b","name":"Holo2-8B","provider":"H Company","description":"Holo2-8B is a standard model from H Company in the Holo2 family. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":"2025-11-13","contextWindowTokens":262000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":42.08302601780766,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"multimodal":1},"benchmarks":[{"benchmarkId":"screenSpotPro","benchmarkName":"ScreenSpot Pro","category":"multimodal","score":58.9,"unit":"percent","family":"screenSpotPro","index":42.08302601780766,"aboveFit":false,"run":null,"source":"Kaixin Li, Ziyang Meng, Hongzhan Lin, Ziyang Luo, Yuchen Tian, Jing Ma, Zhiyong Huang, Tat-Seng Chua","sourceUrl":"https://arxiv.org/abs/2504.07981","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":10,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"holo3-122b-a10b","providerId":"h-company","slug":"holo3-122b-a10b","name":"Holo3-122B-A10B","provider":"H Company","description":"Holo3-122B-A10B is a standard model from H Company in the Holo3 family. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":"2026-03-31","contextWindowTokens":64000,"modalities":["text"],"openWeights":false,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":64.13322986476393,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"agentic":1},"benchmarks":[{"benchmarkId":"osWorldVerified","benchmarkName":"OSWorld-Verified","category":"agentic","score":78.85,"unit":"percent","family":"osWorldVerified","index":64.13322986476393,"aboveFit":false,"run":null,"source":"Tianbao Xie, Danyang Zhang, Jixuan Chen, Xiaochuan Li, Siheng Zhao, Ruisheng Cao, Toh Jing Hua, Zhoujun Cheng, Dongchan Shin, Fangyu Lei, Yitao Liu, Yiheng Xu, Shuyan Zhou, Silvio Savarese, Caiming Xiong, Victor Zhong, Tao Yu","sourceUrl":"https://os-world.github.io/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":0.4,"outputPerMillion":3,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":20,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"holo3-35b-a3b","providerId":"h-company","slug":"holo3-35b-a3b","name":"Holo3-35B-A3B","provider":"H Company","description":"Holo3-35B-A3B is a standard model from H Company in the Holo3 family. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":"2026-03-31","contextWindowTokens":64000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":67.12815355616388,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"agentic":1},"benchmarks":[{"benchmarkId":"osWorldVerified","benchmarkName":"OSWorld-Verified","category":"agentic","score":82.56,"unit":"percent","family":"osWorldVerified","index":67.12815355616388,"aboveFit":false,"run":null,"source":"Tianbao Xie, Danyang Zhang, Jixuan Chen, Xiaochuan Li, Siheng Zhao, Ruisheng Cao, Toh Jing Hua, Zhoujun Cheng, Dongchan Shin, Fangyu Lei, Yitao Liu, Yiheng Xu, Shuyan Zhou, Silvio Savarese, Caiming Xiong, Victor Zhong, Tao Yu","sourceUrl":"https://os-world.github.io/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":20,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"holo3-1-0-8b","providerId":"h-company","slug":"holo3-1-0-8b","name":"Holo3.1-0.8B","provider":"H Company","description":"Holo3.1-0.8B is a standard model from H Company in the Holo3.1 family. It has no scored result.","logoUrl":null,"releaseDate":"2026-06-01","contextWindowTokens":262000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"holo3-1-35b-a3b","providerId":"h-company","slug":"holo3-1-35b-a3b","name":"Holo3.1-35B-A3B","provider":"H Company","description":"Holo3.1-35B-A3B is a standard model from H Company in the Holo3.1 family. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":"2026-06-01","contextWindowTokens":262000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":59.63681059520691,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"agentic":1},"benchmarks":[{"benchmarkId":"androidWorld","benchmarkName":"AndroidWorld","category":"agentic","score":79.3,"unit":"percent","family":"androidWorld","index":59.63681059520691,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":0.25,"outputPerMillion":1.8,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":20,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"holo3-1-35b-a3b-fp8","providerId":"h-company","slug":"holo3-1-35b-a3b-fp8","name":"Holo3.1-35B-A3B-FP8","provider":"H Company","description":"Holo3.1-35B-A3B-FP8 is a standard model from H Company in the Holo3.1 family. It has no scored result.","logoUrl":null,"releaseDate":"2026-06-01","contextWindowTokens":262000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"holo3-1-35b-a3b-gguf","providerId":"h-company","slug":"holo3-1-35b-a3b-gguf","name":"Holo3.1-35B-A3B-GGUF","provider":"H Company","description":"Holo3.1-35B-A3B-GGUF is a standard model from H Company in the Holo3.1 family. It has no scored result.","logoUrl":null,"releaseDate":"2026-06-01","contextWindowTokens":262000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"holo3-1-35b-a3b-nvfp4","providerId":"h-company","slug":"holo3-1-35b-a3b-nvfp4","name":"Holo3.1-35B-A3B-NVFP4","provider":"H Company","description":"Holo3.1-35B-A3B-NVFP4 is a standard model from H Company in the Holo3.1 family. It has no scored result.","logoUrl":null,"releaseDate":"2026-06-01","contextWindowTokens":262000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"holo3-1-4b","providerId":"h-company","slug":"holo3-1-4b","name":"Holo3.1-4B","provider":"H Company","description":"Holo3.1-4B is a standard model from H Company in the Holo3.1 family. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":"2026-06-01","contextWindowTokens":262000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":52.95864817452162,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"agentic":1},"benchmarks":[{"benchmarkId":"androidWorld","benchmarkName":"AndroidWorld","category":"agentic","score":71,"unit":"percent","family":"androidWorld","index":52.95864817452162,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":20,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"holo3-1-9b","providerId":"h-company","slug":"holo3-1-9b","name":"Holo3.1-9B","provider":"H Company","description":"Holo3.1-9B is a standard model from H Company in the Holo3.1 family. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":"2026-06-01","contextWindowTokens":262000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":52.95864817452162,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"agentic":1},"benchmarks":[{"benchmarkId":"androidWorld","benchmarkName":"AndroidWorld","category":"agentic","score":71,"unit":"percent","family":"androidWorld","index":52.95864817452162,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":20,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"holotron-12b","providerId":"h-company","slug":"holotron-12b","name":"Holotron-12B","provider":"H Company","description":"Holotron-12B is a standard model from H Company in the Holotron family. It has no scored result.","logoUrl":null,"releaseDate":"2026-03-16","contextWindowTokens":128000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"hopper","providerId":"hopitai","slug":"hopper","name":"Hopper","provider":"HopitAI","description":"Hopper is a standard model from HopitAI. It has 2 scored results in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":93.66594510010319,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"reasoning":2},"benchmarks":[{"benchmarkId":"jevBench15","benchmarkName":"jevBench15","category":"reasoning","score":67.47015061443733,"unit":"percent","family":"jevBench15","index":92.33189020020636,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"jevBench14","benchmarkName":"jevBench14","category":"reasoning","score":59.433442874809664,"unit":"percent","family":"jevBench14","index":95,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":15,"benchmarkCount":2,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"decision-index-0-3-hopper-g-1-2","providerId":"hopitai","slug":"decision-index-0-3-hopper-g-1-2","name":"Hopper (G) 1.2","provider":"HopitAI","description":"Hopper (G) 1.2 is a model from HopitAI. It has no scored result.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"hunyuana13binstruct","providerId":"tencent","slug":"tencent-hunyuan-a13b-instruct","name":"Hunyuan A13B Instruct","provider":"Tencent","description":"Hunyuan A13B Instruct is a model from Tencent. It has no scored result.","logoUrl":null,"releaseDate":"2025-07-08T15:14:24.000Z","contextWindowTokens":131072,"modalities":["text"],"openWeights":true,"details":{"apiId":"tencent/hunyuan-a13b-instruct","maxOutputTokens":117964,"knowledgeCutoff":"2025-03-31","retiresAt":null,"tools":false,"structuredOutput":true,"cacheReadPerMillion":null,"weightsUrl":"https://huggingface.co/tencent/Hunyuan-A13B-Instruct"},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":0.14,"outputPerMillion":0.5700000000000001,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":26,"timeToFirstToken":0.9945,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"hy-asr-3-0-preview","providerId":"tencent","slug":"hy-asr-3-0-preview","name":"Hy ASR 3.0 preview","provider":"Tencent","description":"Hy ASR 3.0 preview is a standard model from Tencent. It has no scored result.","logoUrl":null,"releaseDate":"2026-08-04","contextWindowTokens":null,"modalities":["text"],"openWeights":false,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"hymt27b","providerId":"tencent","slug":"tencent-hy-mt2-7b","name":"Hy MT2 7B","provider":"Tencent","description":"Hy MT2 7B is a model from Tencent. It has no scored result.","logoUrl":null,"releaseDate":"2026-08-19T14:13:17.000Z","contextWindowTokens":8192,"modalities":["text"],"openWeights":true,"details":{"apiId":"tencent/hy-mt2-7b","maxOutputTokens":4096,"knowledgeCutoff":null,"retiresAt":null,"tools":false,"structuredOutput":true,"cacheReadPerMillion":null,"weightsUrl":"https://huggingface.co/tencent/Hy-MT2-7B"},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":0.074,"outputPerMillion":0.295,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":72,"timeToFirstToken":0.827,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":0.074,"outputPerMillion":0.295,"cacheReadPerMillion":null,"contextTokens":8192,"maxOutputTokens":4096,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"hy-memory-30b-a3b","providerId":"tencent","slug":"hy-memory-30b-a3b","name":"Hy-Memory-30B-A3B","provider":"Tencent","description":"Hy-Memory-30B-A3B is a standard model from Tencent in the Hy-Memory family. It has no scored result.","logoUrl":null,"releaseDate":"2026-10-15","contextWindowTokens":256000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"hy-mt1-5-1-8b-1-25bit","providerId":"tencent","slug":"hy-mt1-5-1-8b-1-25bit","name":"Hy-MT1.5-1.8B-1.25bit","provider":"Tencent","description":"Hy-MT1.5-1.8B-1.25bit is a standard model from Tencent in the Hy-MT1.5-1.8B family. It has no scored result.","logoUrl":null,"releaseDate":"2026-04-29","contextWindowTokens":262000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"hy-mt2-1-8b","providerId":"tencent","slug":"hy-mt2-1-8b","name":"Hy-MT2-1.8B","provider":"Tencent","description":"Hy-MT2-1.8B is a standard model from Tencent in the Hy-MT2 family. It has no scored result.","logoUrl":null,"releaseDate":"2026-05-21","contextWindowTokens":8192,"modalities":["text"],"openWeights":true,"details":{"apiId":"tencent/hy-mt2-1.8b","maxOutputTokens":4096,"knowledgeCutoff":null,"retiresAt":null,"tools":false,"structuredOutput":false,"cacheReadPerMillion":null,"weightsUrl":"https://huggingface.co/tencent/Hy-MT2-1.8B"},"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":0.044,"outputPerMillion":0.17700000000000002,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":100,"timeToFirstToken":0.43,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":0.044,"outputPerMillion":0.177,"cacheReadPerMillion":null,"contextTokens":8192,"maxOutputTokens":4096,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"hy-mt2-30b-a3b","providerId":"tencent","slug":"hy-mt2-30b-a3b","name":"Hy-MT2-30B-A3B","provider":"Tencent","description":"Hy-MT2-30B-A3B is a standard model from Tencent in the Hy-MT2 family. It has no scored result.","logoUrl":null,"releaseDate":"2026-05-21","contextWindowTokens":8192,"modalities":["text"],"openWeights":true,"details":{"apiId":"tencent/hy-mt2-30b-a3b","maxOutputTokens":4096,"knowledgeCutoff":null,"retiresAt":null,"tools":false,"structuredOutput":true,"cacheReadPerMillion":null,"weightsUrl":"https://huggingface.co/tencent/Hy-MT2-30B-A3B"},"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":0.074,"outputPerMillion":0.295,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":64,"timeToFirstToken":0.758,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":0.074,"outputPerMillion":0.295,"cacheReadPerMillion":null,"contextTokens":8192,"maxOutputTokens":4096,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"hy3","providerId":"tencent","slug":"hy3","name":"Hy3","provider":"Tencent","description":"Hy3 is a reasoning model from Tencent. It has 7 scored results in 4 categories. It has no index.","logoUrl":null,"releaseDate":"2026-07-06","contextWindowTokens":262144,"modalities":["text"],"openWeights":true,"details":{"apiId":"tencent/hy3","maxOutputTokens":128000,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.032999999999999995,"weightsUrl":"https://huggingface.co/tencent/Hy3"},"reasoning":true,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":57.40130894070968,"coding":59.1991881582673,"reasoning":53.91667762251765,"multimodal":null,"knowledge":58.73822047498292,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"knowledge":2,"reasoning":2,"coding":1,"agentic":2},"benchmarks":[{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":89.7,"unit":"percent","family":"gpqa-diamond","index":59.47948967815425,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":79,"unit":"percent","family":"lcr","index":61.49841098043786,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaSciCode","benchmarkName":"Artificial Analysis SciCode","category":"coding","score":48.6,"unit":"percent","family":"scicode-family","index":59.1991881582673,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/scicode","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":33.5,"unit":"percent","family":"hle","index":57.99695127181159,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gdpvalAaNormalized","benchmarkName":"GDPval-AA normalized","category":"agentic","score":28.2,"unit":"percent","family":"gdpvalAaNormalized","index":54.19611959690245,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":4.9,"unit":"percent","family":"critpt","index":46.33494426459745,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lmarena:agent-steerability:overall","benchmarkName":"Agent Arena steerability","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":-3.087198524057788,"unit":"index","family":"agent-arena","index":63.99264013964999,"aboveFit":false,"run":null,"source":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset","measuredAt":"2026-11-11","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lmarena:agent-bash-recovery:overall","benchmarkName":"Agent Arena command recovery","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":-7.205807009961007,"unit":"index","family":"agent-arena","index":60.60649828451692,"aboveFit":false,"run":null,"source":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset","measuredAt":"2026-10-08","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lmarena:agent-task-outcome:overall","benchmarkName":"Agent Arena task outcome","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":-11.762121004157677,"unit":"index","family":"agent-arena","index":56.86049388189295,"aboveFit":false,"run":null,"source":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset","measuredAt":"2026-11-11","stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":0.13199999999999998,"outputPerMillion":0.5279999999999999,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":49,"timeToFirstToken":3.042,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":{"elo":1438.964448790014,"eloUpper":1444.6312045145555,"eloLower":1433.2976930654722,"votes":15068,"rank":79,"rankUpper":null,"rankLower":null,"arenaName":"Hy3","organization":"tencent","updatedAt":"2026-11-09"},"quality":{"coveragePercent":70,"benchmarkCount":7,"categoryCount":4,"confidence":"unranked","state":"partial"},"hosts":[{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":0.132,"outputPerMillion":0.528,"cacheReadPerMillion":0.033,"contextTokens":262144,"maxOutputTokens":128000,"deprecated":false},{"host":"siliconflow","name":"SiliconFlow","kind":"host","url":"https://cloud.siliconflow.com/models","inputPerMillion":0.132,"outputPerMillion":0.528,"cacheReadPerMillion":0.033,"contextTokens":262144,"maxOutputTokens":262144,"deprecated":false},{"host":"deepinfra","name":"Deep Infra","kind":"host","url":"https://deepinfra.com/models","inputPerMillion":0.13,"outputPerMillion":0.53,"cacheReadPerMillion":0.033,"contextTokens":262144,"maxOutputTokens":128000,"deprecated":true},{"host":"novita-ai","name":"Novita AI","kind":"host","url":"https://novita.ai/docs/guides/introduction","inputPerMillion":0.14,"outputPerMillion":0.58,"cacheReadPerMillion":0.035,"contextTokens":262144,"maxOutputTokens":262144,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:lmarena","name":"LMArena","url":"https://huggingface.co","license":"CC BY 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"hyperclova-x-dash","providerId":"naver-cloud","slug":"hyperclova-x-dash","name":"HyperClova X Dash","provider":"Naver Cloud","description":"HyperClova X Dash is a standard model from Naver Cloud in the HyperClova X family. It has no scored result.","logoUrl":null,"releaseDate":"2024-04-25","contextWindowTokens":128000,"modalities":["text"],"openWeights":false,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"hyperclova-x-think","providerId":"naver-cloud","slug":"hyperclova-x-think","name":"HyperClova X Think 32B","provider":"Naver Cloud","description":"HyperClova X Think 32B is a reasoning model from Naver Cloud in the HyperClova X family. It has no scored result.","logoUrl":null,"releaseDate":"2025-12-28","contextWindowTokens":128000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":true,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"ichigo","providerId":"jan","slug":"ichigo","name":"Ichigo","provider":"Jan","description":"Ichigo is a standard model from Jan. It has no scored result.","logoUrl":null,"releaseDate":"2024-10-20","contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"imajev-4b","providerId":"mohit67890","slug":"imajev-4b","name":"Imajev-4B","provider":"mohit67890","description":"Imajev-4B is a standard model from mohit67890. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":95,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"reasoning":1},"benchmarks":[{"benchmarkId":"jevBench14","benchmarkName":"jevBench14","category":"reasoning","score":67.36821557095253,"unit":"percent","family":"jevBench14","index":95,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":15,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"imajev-4b-rtx5090-a2","providerId":"mohit67890","slug":"imajev-4b-rtx5090-a2","name":"Imajev-4B (RTX 5090)","provider":"mohit67890","description":"Imajev-4B (RTX 5090) is a standard model from mohit67890. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":94.09800242535866,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"reasoning":1},"benchmarks":[{"benchmarkId":"jevBench15","benchmarkName":"jevBench15","category":"reasoning","score":70.38659212275981,"unit":"percent","family":"jevBench15","index":94.09800242535866,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":15,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"ink2","providerId":"unknown","slug":"unknown-ink-2","name":"Ink 2","provider":"Unknown","description":"Ink 2 is a model. It has no scored result.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"instinct-dual-4b","providerId":"zoowork","slug":"instinct-dual-4b","name":"Instinct Dual 4B","provider":"ZooWork","description":"Instinct Dual 4B is a standard model from ZooWork. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":false,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":79.91637445367398,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"reasoning":1},"benchmarks":[{"benchmarkId":"jevBench15","benchmarkName":"jevBench15","category":"reasoning","score":46.96798492261902,"unit":"percent","family":"jevBench15","index":79.91637445367398,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":15,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"instinct","providerId":"zoowork","slug":"instinct","name":"Instinct Qwen3.8-27B","provider":"ZooWork","description":"Instinct Qwen3.8-27B is a standard model from ZooWork. It has 2 scored results in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":false,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":58.035302553323405,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"reasoning":2},"benchmarks":[{"benchmarkId":"jevBench15","benchmarkName":"jevBench15","category":"reasoning","score":18.324676123308297,"unit":"percent","family":"jevBench15","index":62.57081902425183,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"jevBench14","benchmarkName":"jevBench14","category":"reasoning","score":11.449217397802405,"unit":"percent","family":"jevBench14","index":53.49978608239497,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":15,"benchmarkCount":2,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"interfaze-beta","providerId":"interfaze","slug":"interfaze-beta","name":"Interfaze Beta","provider":"Interfaze","description":"Interfaze Beta is a reasoning model from Interfaze in the Interfaze family. It has 4 scored results in 2 categories. It has no index.","logoUrl":null,"releaseDate":"2026-05-11","contextWindowTokens":1000000,"modalities":["text"],"openWeights":false,"details":null,"reasoning":true,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":38.756058344853294,"knowledge":54.6809861557858,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"knowledge":2,"multimodal":2},"benchmarks":[{"benchmarkId":"mmmlu","benchmarkName":"MMMLU","category":"knowledge","score":90.9,"unit":"percent","family":"mmmlu","index":49.6990480984334,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gpqa","benchmarkName":"Graduate-Level Google-Proof Q&A","category":"knowledge","score":89.9,"unit":"percent","family":"gpqa-diamond","index":59.66292421313821,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gpqaDiamond","benchmarkName":"GPQA Diamond","category":"knowledge","score":89.9,"unit":"percent","family":"gpqa-diamond","index":59.66292421313821,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"olmOcr","benchmarkName":"olmOCR-Bench","category":"multimodal","score":85.7,"unit":"percent","family":"olmOcr","index":null,"aboveFit":false,"run":null,"source":"Allen Institute for AI","sourceUrl":"https://github.com/allenai/olmocr/tree/main/olmocr/bench","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"refcocoAvg","benchmarkName":"RefCOCO average","category":"multimodal","score":82.1,"unit":"percent","family":"refcocoAvg","index":29.828889611898273,"aboveFit":false,"run":null,"source":"RefCOCO dataset authors","sourceUrl":"https://github.com/lichengunc/refer","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"sobValueAcc","benchmarkName":"Structured Output Benchmark Value Accuracy","category":"instruction","score":79.5,"unit":"percent","family":"sobValueAcc","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmmuPro","benchmarkName":"Massive Multi-discipline Multimodal Understanding Pro","category":"multimodal","score":71.1,"unit":"percent","family":"mmmu-pro","index":47.68322707780831,"aboveFit":false,"run":null,"source":"MMMU-Pro authors","sourceUrl":"https://arxiv.org/abs/2409.02813","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"ocrBenchV2","benchmarkName":"OCRBench V2","category":"multimodal","score":70.7,"unit":"percent","family":"ocrBenchV2","index":null,"aboveFit":false,"run":null,"source":"OCRBench authors","sourceUrl":"https://arxiv.org/abs/2501.00321","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"spider2Lite","benchmarkName":"Spider 2.0-Lite","category":"coding","score":52.9,"unit":"percent","family":"spider2Lite","index":null,"aboveFit":false,"run":null,"source":"Spider 2.0 authors","sourceUrl":"https://github.com/xlang-ai/Spider2","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":1.5,"outputPerMillion":3.5,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":25,"benchmarkCount":4,"categoryCount":2,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"decision-index-0-3-intern-decision-0-8b","providerId":"internlm","slug":"decision-index-0-3-intern-decision-0-8b","name":"Intern-Decision-0.8B","provider":"internlm","description":"Intern-Decision-0.8B is a model from internlm. It has no scored result.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"decision-index-0-3-intern-decision-2b","providerId":"internlm","slug":"decision-index-0-3-intern-decision-2b","name":"Intern-Decision-2B","provider":"internlm","description":"Intern-Decision-2B is a model from internlm. It has no scored result.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"decision-index-0-3-intern-decision-4b","providerId":"internlm","slug":"decision-index-0-3-intern-decision-4b","name":"Intern-Decision-4B","provider":"internlm","description":"Intern-Decision-4B is a model from internlm. It has no scored result.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"jais-3-0-9b","providerId":"ifm","slug":"jais-3-0-9b","name":"Jais-3-0.9B","provider":"IFM","description":"Jais-3-0.9B is a reasoning model from IFM. It has no scored result.","logoUrl":null,"releaseDate":"2026-10-15","contextWindowTokens":128000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":true,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"jais-3-0-9b-uno","providerId":"ifm","slug":"jais-3-0-9b-uno","name":"Jais-3-0.9B-Uno","provider":"IFM","description":"Jais-3-0.9B-Uno is a reasoning model from IFM in the Jais-3-0.9B family. It has no scored result.","logoUrl":null,"releaseDate":"2026-10-15","contextWindowTokens":128000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":true,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"jais-3-3-7b","providerId":"ifm","slug":"jais-3-3-7b","name":"Jais-3-3.7B","provider":"IFM","description":"Jais-3-3.7B is a reasoning model from IFM. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":"2026-10-15","contextWindowTokens":512000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":true,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":45.237162223641704,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"agentic":1},"benchmarks":[{"benchmarkId":"terminalBench21","benchmarkName":"Terminal-Bench 2.1","category":"agentic","score":41.2,"unit":"percent","family":"terminal-bench-2-1","index":45.237162223641704,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":20,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"jais-3-3-7b-uno","providerId":"ifm","slug":"jais-3-3-7b-uno","name":"Jais-3-3.7B-Uno","provider":"IFM","description":"Jais-3-3.7B-Uno is a reasoning model from IFM in the Jais-3-3.7B family. It has no scored result.","logoUrl":null,"releaseDate":"2026-10-15","contextWindowTokens":512000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":true,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"jais-3-32b","providerId":"ifm","slug":"jais-3-32b","name":"Jais-3-32B","provider":"IFM","description":"Jais-3-32B is a reasoning model from IFM. It has 2 scored results in 2 categories. It has no index.","logoUrl":null,"releaseDate":"2026-10-15","contextWindowTokens":512000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":true,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":58.39640792428369,"coding":54.62709616101095,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"coding":1,"agentic":1},"benchmarks":[{"benchmarkId":"sweVerified","benchmarkName":"Software Engineering Benchmark Verified","category":"coding","score":72.4,"unit":"percent","family":"swe-bench-verified","index":54.62709616101095,"aboveFit":false,"run":null,"source":"Carlos E. Jimenez, John Yang, Alexander Wettig, Shunyu Yao, Kexin Pei, Ofir Press, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2310.06770","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"terminalBench21","benchmarkName":"Terminal-Bench 2.1","category":"agentic","score":63.7,"unit":"percent","family":"terminal-bench-2-1","index":58.39640792428369,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":40,"benchmarkCount":2,"categoryCount":2,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"jais-3-32b-uno","providerId":"ifm","slug":"jais-3-32b-uno","name":"Jais-3-32B-Uno","provider":"IFM","description":"Jais-3-32B-Uno is a reasoning model from IFM in the Jais-3-32B family. It has no scored result.","logoUrl":null,"releaseDate":"2026-10-15","contextWindowTokens":512000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":true,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"jais-3-375b-a23b","providerId":"ifm","slug":"jais-3-375b-a23b","name":"Jais-3-375B-A23B","provider":"IFM","description":"Jais-3-375B-A23B is a reasoning model from IFM. It has 2 scored results in 2 categories. It has no index.","logoUrl":null,"releaseDate":"2026-10-15","contextWindowTokens":512000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":true,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":64.30344710546076,"coding":53.00446948988682,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"agentic":1,"coding":1},"benchmarks":[{"benchmarkId":"terminalBench21","benchmarkName":"Terminal-Bench 2.1","category":"agentic","score":73.8,"unit":"percent","family":"terminal-bench-2-1","index":64.30344710546076,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"sweVerified","benchmarkName":"Software Engineering Benchmark Verified","category":"coding","score":70.2,"unit":"percent","family":"swe-bench-verified","index":53.00446948988682,"aboveFit":false,"run":null,"source":"Carlos E. Jimenez, John Yang, Alexander Wettig, Shunyu Yao, Kexin Pei, Ofir Press, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2310.06770","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":40,"benchmarkCount":2,"categoryCount":2,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"jais-3-7b","providerId":"ifm","slug":"jais-3-7b","name":"Jais-3-7B","provider":"IFM","description":"Jais-3-7B is a reasoning model from IFM. It has 2 scored results in 2 categories. It has no index.","logoUrl":null,"releaseDate":"2026-10-15","contextWindowTokens":512000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":true,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":48.278410118901185,"coding":53.00446948988682,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"coding":1,"agentic":1},"benchmarks":[{"benchmarkId":"sweVerified","benchmarkName":"Software Engineering Benchmark Verified","category":"coding","score":70.2,"unit":"percent","family":"swe-bench-verified","index":53.00446948988682,"aboveFit":false,"run":null,"source":"Carlos E. Jimenez, John Yang, Alexander Wettig, Shunyu Yao, Kexin Pei, Ofir Press, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2310.06770","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"terminalBench21","benchmarkName":"Terminal-Bench 2.1","category":"agentic","score":46.4,"unit":"percent","family":"terminal-bench-2-1","index":48.278410118901185,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":40,"benchmarkCount":2,"categoryCount":2,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"jais-3-7b-uno","providerId":"ifm","slug":"jais-3-7b-uno","name":"Jais-3-7B-Uno","provider":"IFM","description":"Jais-3-7B-Uno is a reasoning model from IFM in the Jais-3-7B family. It has no scored result.","logoUrl":null,"releaseDate":"2026-10-15","contextWindowTokens":512000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":true,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"jais-3-mova-36b-a4b","providerId":"ifm","slug":"jais-3-mova-36b-a4b","name":"Jais-3-MoVA-36B-A4B","provider":"IFM","description":"Jais-3-MoVA-36B-A4B is a reasoning model from IFM. It has 2 scored results in 2 categories. It has no index.","logoUrl":null,"releaseDate":"2026-10-15","contextWindowTokens":512000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":true,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":60.50188723638641,"coding":54.92211919212442,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"coding":1,"agentic":1},"benchmarks":[{"benchmarkId":"sweVerified","benchmarkName":"Software Engineering Benchmark Verified","category":"coding","score":72.8,"unit":"percent","family":"swe-bench-verified","index":54.92211919212442,"aboveFit":false,"run":null,"source":"Carlos E. Jimenez, John Yang, Alexander Wettig, Shunyu Yao, Kexin Pei, Ofir Press, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2310.06770","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"terminalBench21","benchmarkName":"Terminal-Bench 2.1","category":"agentic","score":67.3,"unit":"percent","family":"terminal-bench-2-1","index":60.50188723638641,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":40,"benchmarkCount":2,"categoryCount":2,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"jais-asr","providerId":"ifm","slug":"jais-asr","name":"Jais-ASR","provider":"IFM","description":"Jais-ASR is a standard model from IFM. It has no scored result.","logoUrl":null,"releaseDate":"2026-10-01","contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"jamba15large","providerId":"ai21-labs","slug":"ai21-labs-jamba-1-5-large","name":"Jamba 1.5 Large","provider":"AI21 Labs","description":"Jamba 1.5 Large is a model from AI21 Labs. It has no scored result.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[{"benchmarkId":"vals-ai:legal_bench:overall","benchmarkName":"LegalBench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":74.16,"unit":"percent","family":"vals-ai:legal_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","measuredAt":"2026-10-13","stderr":0.639,"costPerTest":null,"latency":0.808},{"benchmarkId":"vals-ai:medqa:overall","benchmarkName":"MedQA","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":68.108,"unit":"percent","family":"vals-ai:medqa:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medqa","measuredAt":"2026-04-16","stderr":0.833,"costPerTest":0.002198,"latency":5.999},{"benchmarkId":"vals-ai:tax_eval_v2:overall","benchmarkName":"TaxEval v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":58.176,"unit":"percent","family":"vals-ai:tax_eval_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","measuredAt":"2026-09-01","stderr":0.971,"costPerTest":0.005384,"latency":20.028},{"benchmarkId":"vals-ai:corp_fin_v2:overall","benchmarkName":"CorpFin v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":39.433,"unit":"percent","family":"vals-ai:corp_fin_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","measuredAt":"2026-08-12","stderr":0.959,"costPerTest":0.149067,"latency":10.765}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":{"elo":1237.031519974828,"eloUpper":1244.3116441104926,"eloLower":1229.7513958391637,"votes":8662,"rank":303,"rankUpper":null,"rankLower":null,"arenaName":"Jamba 1.5 Large","organization":"","updatedAt":"2026-11-09"},"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"jamba15mini","providerId":"ai21-labs","slug":"ai21-labs-jamba-1-5-mini","name":"Jamba 1.5 Mini","provider":"AI21 Labs","description":"Jamba 1.5 Mini is a model from AI21 Labs. It has no scored result.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[{"benchmarkId":"vals-ai:legal_bench:overall","benchmarkName":"LegalBench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":66.616,"unit":"percent","family":"vals-ai:legal_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","measuredAt":"2026-10-13","stderr":0.673,"costPerTest":null,"latency":0.275},{"benchmarkId":"vals-ai:medqa:overall","benchmarkName":"MedQA","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":55.183,"unit":"percent","family":"vals-ai:medqa:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medqa","measuredAt":"2026-04-16","stderr":0.889,"costPerTest":0.000147,"latency":1.143},{"benchmarkId":"vals-ai:tax_eval_v2:overall","benchmarkName":"TaxEval v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":41.864,"unit":"percent","family":"vals-ai:tax_eval_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","measuredAt":"2026-09-01","stderr":0.967,"costPerTest":0.00028,"latency":4.78},{"benchmarkId":"vals-ai:corp_fin_v2:overall","benchmarkName":"CorpFin v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":33.877,"unit":"percent","family":"vals-ai:corp_fin_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","measuredAt":"2026-08-12","stderr":0.929,"costPerTest":0.014865,"latency":2.315}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":{"elo":1186.6831183761858,"eloUpper":1193.8432391442925,"eloLower":1179.522997608079,"votes":8858,"rank":331,"rankUpper":null,"rankLower":null,"arenaName":"Jamba 1.5 Mini","organization":"","updatedAt":"2026-11-09"},"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"jambalarge16","providerId":"ai21-labs","slug":"ai21-labs-jamba-large-1-6","name":"Jamba Large 1.6","provider":"AI21 Labs","description":"Jamba Large 1.6 is a model from AI21 Labs. It has 5 scored results in 3 categories. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":7.86773944098492,"reasoning":null,"multimodal":null,"knowledge":8.762086677693155,"multilingual":null,"instruction":null,"math":10.342407036148614},"capabilityCounts":{"math":2,"knowledge":2,"coding":1},"benchmarks":[{"benchmarkId":"vals-ai:legal_bench:overall","benchmarkName":"LegalBench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":71.962,"unit":"percent","family":"vals-ai:legal_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","measuredAt":"2026-10-13","stderr":0.424,"costPerTest":null,"latency":0.814},{"benchmarkId":"vals-ai:mgsm:overall","benchmarkName":"MGSM","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":71.236,"unit":"percent","family":"vals-ai:mgsm:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mgsm","measuredAt":"2026-01-09","stderr":2.6,"costPerTest":0.003816,"latency":9.862},{"benchmarkId":"vals-ai:tax_eval_v2:overall","benchmarkName":"TaxEval v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":60.875,"unit":"percent","family":"vals-ai:tax_eval_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","measuredAt":"2026-09-01","stderr":0.949,"costPerTest":0.005162,"latency":16.439},{"benchmarkId":"vals-ai:math500:overall","benchmarkName":"MATH 500","category":"math","domain":null,"eligible":true,"ranked":true,"score":54.8,"unit":"percent","family":"math-500","index":5,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/math500","measuredAt":"2026-01-09","stderr":2.226,"costPerTest":0.00395,"latency":13.007},{"benchmarkId":"vals-ai:medqa:overall","benchmarkName":"MedQA","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":50.7,"unit":"percent","family":"vals-ai:medqa:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medqa","measuredAt":"2026-04-16","stderr":0.456,"costPerTest":0.003147,"latency":7.31},{"benchmarkId":"vals-ai:mmlu_pro:overall","benchmarkName":"MMLU Pro","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":49.778,"unit":"percent","family":"mmlu-pro","index":7.195050153508419,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","measuredAt":"2026-09-01","stderr":0.475,"costPerTest":0.004628,"latency":9.48},{"benchmarkId":"vals-ai:corp_fin_v2:overall","benchmarkName":"CorpFin v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":41.531,"unit":"percent","family":"vals-ai:corp_fin_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","measuredAt":"2026-08-12","stderr":0.963,"costPerTest":0.150029,"latency":27.38},{"benchmarkId":"vals-ai:gpqa:overall","benchmarkName":"GPQA Diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":36.111,"unit":"percent","family":"gpqa-diamond","index":10.329123201877884,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","measuredAt":"2026-09-01","stderr":2.099,"costPerTest":0.00669,"latency":15.471},{"benchmarkId":"vals-ai:lcb:overall","benchmarkName":"LiveCodeBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":22.325,"unit":"percent","family":"livecodebench","index":7.86773944098492,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","measuredAt":"2026-09-01","stderr":1.051,"costPerTest":0.003236,"latency":4.84},{"benchmarkId":"vals-ai:aime:overall","benchmarkName":"AIME","category":"math","domain":null,"eligible":true,"ranked":true,"score":0.417,"unit":"percent","family":"vals-ai:aime:overall","index":15.684814072297222,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/aime","measuredAt":"2026-04-16","stderr":0.417,"costPerTest":0.007658,"latency":18.858}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":45,"benchmarkCount":5,"categoryCount":3,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"jambalarge17","providerId":"ai21-labs","slug":"ai21-labs-jamba-large-1-7","name":"Jamba Large 1.7","provider":"AI21 Labs","description":"Jamba Large 1.7 is a model from AI21 Labs. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":22.74091429596053,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"agentic":1},"benchmarks":[{"benchmarkId":"vals-ai:terminal-bench:overall","benchmarkName":"Terminal-Bench 1.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":6.25,"unit":"percent","family":"vals-ai:terminal-bench:overall","index":22.74091429596053,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench","measuredAt":"2026-01-12","stderr":2.706,"costPerTest":4.016933,"latency":578.51},{"benchmarkId":"vals-ai:finance_agent:overall","benchmarkName":"Finance Agent v1.1","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":0.37,"unit":"percent","family":"vals-ai:finance_agent:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/finance_agent","measuredAt":"2026-06-04","stderr":0.364,"costPerTest":5.040786,"latency":2330.85}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":20,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"jambamini16","providerId":"ai21-labs","slug":"ai21-labs-jamba-mini-1-6","name":"Jamba Mini 1.6","provider":"AI21 Labs","description":"Jamba Mini 1.6 is a model from AI21 Labs. It has 5 scored results in 3 categories. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":5,"reasoning":null,"multimodal":null,"knowledge":5.811414211263518,"multilingual":null,"instruction":null,"math":10.342407036148614},"capabilityCounts":{"knowledge":2,"math":2,"coding":1},"benchmarks":[{"benchmarkId":"vals-ai:legal_bench:overall","benchmarkName":"LegalBench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":69.726,"unit":"percent","family":"vals-ai:legal_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","measuredAt":"2026-10-13","stderr":0.515,"costPerTest":null,"latency":0.305},{"benchmarkId":"vals-ai:medqa:overall","benchmarkName":"MedQA","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":52.517,"unit":"percent","family":"vals-ai:medqa:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medqa","measuredAt":"2026-04-16","stderr":0.456,"costPerTest":0.000203,"latency":2.185},{"benchmarkId":"vals-ai:tax_eval_v2:overall","benchmarkName":"TaxEval v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":44.604,"unit":"percent","family":"vals-ai:tax_eval_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","measuredAt":"2026-09-01","stderr":0.963,"costPerTest":0.00026,"latency":4.337},{"benchmarkId":"vals-ai:mgsm:overall","benchmarkName":"MGSM","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":41.709,"unit":"percent","family":"vals-ai:mgsm:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mgsm","measuredAt":"2026-01-09","stderr":2.871,"costPerTest":0.000279,"latency":3.927},{"benchmarkId":"vals-ai:corp_fin_v2:overall","benchmarkName":"CorpFin v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":38.034,"unit":"percent","family":"vals-ai:corp_fin_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","measuredAt":"2026-08-12","stderr":0.951,"costPerTest":0.014897,"latency":4.248},{"benchmarkId":"vals-ai:gpqa:overall","benchmarkName":"GPQA Diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":32.07,"unit":"percent","family":"gpqa-diamond","index":6.622828422527036,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","measuredAt":"2026-09-01","stderr":1.988,"costPerTest":0.000487,"latency":4.953},{"benchmarkId":"vals-ai:mmlu_pro:overall","benchmarkName":"MMLU Pro","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":30.278,"unit":"percent","family":"mmlu-pro","index":5,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","measuredAt":"2026-09-01","stderr":0.448,"costPerTest":0.000391,"latency":2.426},{"benchmarkId":"vals-ai:math500:overall","benchmarkName":"MATH 500","category":"math","domain":null,"eligible":true,"ranked":true,"score":25.4,"unit":"percent","family":"math-500","index":5,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/math500","measuredAt":"2026-01-09","stderr":1.947,"costPerTest":0.000306,"latency":4.859},{"benchmarkId":"vals-ai:lcb:overall","benchmarkName":"LiveCodeBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":9.918,"unit":"percent","family":"livecodebench","index":5,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","measuredAt":"2026-09-01","stderr":0.867,"costPerTest":0.000199,"latency":1.139},{"benchmarkId":"vals-ai:aime:overall","benchmarkName":"AIME","category":"math","domain":null,"eligible":true,"ranked":true,"score":0.417,"unit":"percent","family":"vals-ai:aime:overall","index":15.684814072297222,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/aime","measuredAt":"2026-04-16","stderr":0.273,"costPerTest":0.000456,"latency":6.624}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":45,"benchmarkCount":5,"categoryCount":3,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"decision-index-0-3-jebadiah-27b","providerId":"frontier-infra","slug":"decision-index-0-3-jebadiah-27b","name":"Jebadiah 27B","provider":"frontier-infra","description":"Jebadiah 27B is a model from frontier-infra. It has no scored result.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"jeff","providerId":"logan-markewich","slug":"jeff","name":"jeff","provider":"Logan Markewich","description":"jeff is a standard model from Logan Markewich. It has 2 scored results in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":61.33793719733242,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"reasoning":2},"benchmarks":[{"benchmarkId":"jevBench14","benchmarkName":"jevBench14","category":"reasoning","score":30.579484596127113,"unit":"percent","family":"jevBench14","index":71.13091100710085,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"jevBench15","benchmarkName":"jevBench15","category":"reasoning","score":0.11730368929731415,"unit":"percent","family":"jevBench15","index":51.544963387564,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":15,"benchmarkCount":2,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"decision-index-0-3-jet-v6-2","providerId":"michaljach","slug":"decision-index-0-3-jet-v6-2","name":"Jet v6.2","provider":"michaljach","description":"Jet v6.2 is a model from michaljach. It has no scored result.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"jev-1-13-0","providerId":"typesafe-ai","slug":"jev-1-13-0","name":"Jev 1.13.0","provider":"TypeSafe AI","description":"Jev 1.13.0 is a standard model from TypeSafe AI. It has 2 scored results in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":false,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":95,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"reasoning":2},"benchmarks":[{"benchmarkId":"jevBench15","benchmarkName":"jevBench15","category":"reasoning","score":72.13292897895896,"unit":"percent","family":"jevBench15","index":95,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"jevBench14","benchmarkName":"jevBench14","category":"reasoning","score":63.29205745601932,"unit":"percent","family":"jevBench14","index":95,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":15,"benchmarkCount":2,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"jev-1-14","providerId":"typesafe-ai","slug":"jev-1-14","name":"Jev 1.14","provider":"TypeSafe AI","description":"Jev 1.14 is a model from TypeSafe AI. It has no scored result.","logoUrl":null,"releaseDate":"2026-10-14","contextWindowTokens":64000,"modalities":["text"],"openWeights":false,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":0.042,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"jev-qwen3-5-9b-base-nvfp4","providerId":"wilflin","slug":"jev-qwen3-5-9b-base-nvfp4","name":"JEV Qwen3.5-9B Base NVFP4","provider":"WilfLin","description":"JEV Qwen3.5-9B Base NVFP4 is a standard model from WilfLin. It has 2 scored results in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":70.66008221923511,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"reasoning":2},"benchmarks":[{"benchmarkId":"jevBench14","benchmarkName":"jevBench14","category":"reasoning","score":37.697189917260424,"unit":"percent","family":"jevBench14","index":77.69083776573675,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"jevBench15","benchmarkName":"jevBench15","category":"reasoning","score":20.07262599814848,"unit":"percent","family":"jevBench15","index":63.62932667273348,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":15,"benchmarkCount":2,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"jevrouter","providerId":"typesafe","slug":"typesafe-jev-router","name":"Jev Router","provider":"TypeSafe","description":"Jev Router is a model from TypeSafe. It has no scored result.","logoUrl":null,"releaseDate":"2026-09-25T19:12:40.000Z","contextWindowTokens":1000000,"modalities":["audio","file","image","text","video"],"openWeights":null,"details":{"apiId":"typesafe/jev-router","maxOutputTokens":64000,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":null,"weightsUrl":null},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"decision-index-0-3-jev-27b","providerId":"autotrust","slug":"decision-index-0-3-jev-27b","name":"JEV-27B","provider":"autotrust","description":"JEV-27B is a model from autotrust. It has no scored result.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"decision-index-0-3-jev-9b","providerId":"autotrust","slug":"decision-index-0-3-jev-9b","name":"JEV-9B","provider":"autotrust","description":"JEV-9B is a model from autotrust. It has no scored result.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"jev-local","providerId":"us-github","slug":"jev-local","name":"jev-local Qwen3.5-9B","provider":"us (GitHub)","description":"jev-local Qwen3.5-9B is a standard model from us (GitHub). It has 2 scored results in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":81.96029192246777,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"reasoning":2},"benchmarks":[{"benchmarkId":"jevBench15","benchmarkName":"jevBench15","category":"reasoning","score":65.23868668964282,"unit":"percent","family":"jevBench15","index":90.98058049407315,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"jevBench14","benchmarkName":"jevBench14","category":"reasoning","score":32.542400749826804,"unit":"percent","family":"jevBench14","index":72.94000335086241,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":15,"benchmarkCount":2,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"jev-omni","providerId":"akhilaaa3","slug":"jev-omni","name":"Jev-Omni","provider":"akhilaaa3","description":"Jev-Omni is a standard model from akhilaaa3. It has 2 scored results in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":92.51916442809322,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"reasoning":2},"benchmarks":[{"benchmarkId":"jevBench15","benchmarkName":"jevBench15","category":"reasoning","score":71.5005378093621,"unit":"percent","family":"jevBench15","index":94.7725755617133,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"jevBench14","benchmarkName":"jevBench14","category":"reasoning","score":51.34132749148857,"unit":"percent","family":"jevBench14","index":90.26575329447314,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":15,"benchmarkCount":2,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"jevact","providerId":"einptein","slug":"jevact","name":"JevAct","provider":"einptein","description":"JevAct is a standard model from einptein. It has 2 scored results in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":false,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":55.472324861056656,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"reasoning":2},"benchmarks":[{"benchmarkId":"jevBench14","benchmarkName":"jevBench14","category":"reasoning","score":16.92690664787098,"unit":"percent","family":"jevBench14","index":58.54821653375505,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"jevBench15","benchmarkName":"jevBench15","category":"reasoning","score":1.523364886742046,"unit":"percent","family":"jevBench15","index":52.39643318835825,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":15,"benchmarkCount":2,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"decision-index-0-3-jevk5-9b-v0-3-3","providerId":"alibiserikbay","slug":"decision-index-0-3-jevk5-9b-v0-3-3","name":"JevK5 9B v0.3.3","provider":"alibiserikbay","description":"JevK5 9B v0.3.3 is a model from alibiserikbay. It has no scored result.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"decision-index-0-3-jevk5-lite","providerId":"alibiserikbay","slug":"decision-index-0-3-jevk5-lite","name":"JevK5 Lite","provider":"alibiserikbay","description":"JevK5 Lite is a model from alibiserikbay. It has no scored result.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"jevk5-v0-2-0","providerId":"allebee","slug":"jevk5-v0-2-0","name":"JevK5 v0.2.0","provider":"allebee","description":"JevK5 v0.2.0 is a standard model from allebee. It has 2 scored results in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":90.83319757300114,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"reasoning":2},"benchmarks":[{"benchmarkId":"jevBench14","benchmarkName":"jevBench14","category":"reasoning","score":62.04446598579722,"unit":"percent","family":"jevBench14","index":95,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"jevBench15","benchmarkName":"jevBench15","category":"reasoning","score":58.11452490392985,"unit":"percent","family":"jevBench15","index":86.66639514600229,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":15,"benchmarkCount":2,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"jevk5-v0-3-4b","providerId":"allebee","slug":"jevk5-v0-3-4b","name":"JevK5 v0.3 4B","provider":"allebee","description":"JevK5 v0.3 4B is a standard model from allebee. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":95,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"reasoning":1},"benchmarks":[{"benchmarkId":"jevBench15","benchmarkName":"jevBench15","category":"reasoning","score":71.89794783714967,"unit":"percent","family":"jevBench15","index":95,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":15,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"jevone","providerId":"juspay","slug":"jevone","name":"JevOne","provider":"Juspay","description":"JevOne is a standard model from Juspay. It has 2 scored results in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":70.54846349890849,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"reasoning":2},"benchmarks":[{"benchmarkId":"jevBench15","benchmarkName":"jevBench15","category":"reasoning","score":38.24864204152178,"unit":"percent","family":"jevBench15","index":74.63619379047319,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"jevBench14","benchmarkName":"jevBench14","category":"reasoning","score":25.512210128129595,"unit":"percent","family":"jevBench14","index":66.46073320734378,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":15,"benchmarkCount":2,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"decision-index-0-3-jevstral-8b","providerId":"amirbraham","slug":"decision-index-0-3-jevstral-8b","name":"Jevstral 8B","provider":"AmirBraham","description":"Jevstral 8B is a model from AmirBraham. It has no scored result.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"decision-index-0-3-jiwo-0-8b","providerId":"eljiwo","slug":"decision-index-0-3-jiwo-0-8b","name":"jiwo 0.8B","provider":"eljiwo","description":"jiwo 0.8B is a model from eljiwo. It has no scored result.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"decision-index-0-3-jiwo-4b","providerId":"eljiwo","slug":"decision-index-0-3-jiwo-4b","name":"jiwo 4B","provider":"eljiwo","description":"jiwo 4B is a model from eljiwo. It has no scored result.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"jobe-qwen3-5-4b","providerId":"mantisshrimpdev","slug":"jobe-qwen3-5-4b","name":"Jobe Qwen3.5-4B","provider":"MantisShrimpdev","description":"Jobe Qwen3.5-4B is a standard model from MantisShrimpdev. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":86.20776387539794,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"reasoning":1},"benchmarks":[{"benchmarkId":"jevBench14","benchmarkName":"jevBench14","category":"reasoning","score":46.93829466139942,"unit":"percent","family":"jevBench14","index":86.20776387539794,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":15,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"decision-index-0-3-jpt-0-8b","providerId":"kirp","slug":"decision-index-0-3-jpt-0-8b","name":"JPT-0.8B","provider":"kirp","description":"JPT-0.8B is a model from kirp. It has no scored result.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"decision-index-0-3-jpt-35b-a3b","providerId":"kirp","slug":"decision-index-0-3-jpt-35b-a3b","name":"JPT-35B-A3B","provider":"kirp","description":"JPT-35B-A3B is a model from kirp. It has no scored result.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"decision-index-0-3-jpt-4b","providerId":"kirp","slug":"decision-index-0-3-jpt-4b","name":"JPT-4B","provider":"kirp","description":"JPT-4B is a model from kirp. It has no scored result.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"decision-index-0-3-jpt-9b","providerId":"kirp","slug":"decision-index-0-3-jpt-9b","name":"JPT-9B","provider":"kirp","description":"JPT-9B is a model from kirp. It has no scored result.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"jqv","providerId":"octalab","slug":"jqv","name":"jqv Qwen3-32B","provider":"Octalab","description":"jqv Qwen3-32B is a standard model from Octalab. It has 2 scored results in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":86.05070346515694,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"reasoning":2},"benchmarks":[{"benchmarkId":"jevBench15","benchmarkName":"jevBench15","category":"reasoning","score":60.77445997498618,"unit":"percent","family":"jevBench15","index":88.27717452686704,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"jevBench14","benchmarkName":"jevBench14","category":"reasoning","score":44.35209587211248,"unit":"percent","family":"jevBench14","index":83.82423240344681,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":15,"benchmarkCount":2,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"decision-index-0-3-julia-1","providerId":"supersoniclabs","slug":"decision-index-0-3-julia-1","name":"Julia 1","provider":"SupersonicLabs","description":"Julia 1 is a model from SupersonicLabs. It has no scored result.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"kanana-essence","providerId":"kakao","slug":"kanana-essence","name":"Kanana Essence","provider":"Kakao","description":"Kanana Essence is a standard model from Kakao in the Kanana family. It has no scored result.","logoUrl":null,"releaseDate":"2025-02-26","contextWindowTokens":64000,"modalities":["text"],"openWeights":false,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"kanana-flag","providerId":"kakao","slug":"kanana-flag","name":"Kanana Flag","provider":"Kakao","description":"Kanana Flag is a standard model from Kakao in the Kanana family. It has no scored result.","logoUrl":null,"releaseDate":"2025-02-26","contextWindowTokens":64000,"modalities":["text"],"openWeights":false,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"kanana-nano","providerId":"kakao","slug":"kanana-nano","name":"Kanana Nano","provider":"Kakao","description":"Kanana Nano is a standard model from Kakao in the Kanana family. It has no scored result.","logoUrl":null,"releaseDate":"2025-02-26","contextWindowTokens":64000,"modalities":["text"],"openWeights":false,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"kanana-2-1-3b-instruct","providerId":"kakao","slug":"kanana-2-1-3b-instruct","name":"Kanana-2 1.3B Instruct","provider":"Kakao","description":"Kanana-2 1.3B Instruct is a standard model from Kakao in the Kanana 2 family. It has 3 scored results in 2 categories. It has no index.","logoUrl":null,"releaseDate":"2026-07-24","contextWindowTokens":32000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":20.482325890645612,"math":9.011737499571893},"capabilityCounts":{"instruction":2,"math":1},"benchmarks":[{"benchmarkId":"ifeval","benchmarkName":"Instruction-Following Eval","category":"instruction","score":77.63,"unit":"percent","family":"ifeval","index":19.522849160790877,"aboveFit":false,"run":null,"source":"Jeffrey Zhou, Tianjian Lu, Swaroop Mishra, Siddhartha Brahma, Sujoy Basu, Yi Luan, Denny Zhou, Le Hou","sourceUrl":"https://arxiv.org/abs/2311.07911","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"math500","benchmarkName":"MATH-500 Problem Set","category":"math","score":61.4,"unit":"percent","family":"math-500","index":9.011737499571893,"aboveFit":false,"run":null,"source":"Dan Hendrycks, Collin Burns, Saurav Kadavath, Akul Arora, Steven Basart, Eric Tang, Dawn Song, Jacob Steinhardt","sourceUrl":"https://arxiv.org/abs/2103.03874","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"ifBench","benchmarkName":"Instruction Following Benchmark","category":"instruction","score":34.69,"unit":"percent","family":"ifbench-family","index":21.44180262050035,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":15,"benchmarkCount":3,"categoryCount":2,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"kanana-2-3b-instruct","providerId":"kakao","slug":"kanana-2-3b-instruct","name":"Kanana-2 3B Instruct","provider":"Kakao","description":"Kanana-2 3B Instruct is a standard model from Kakao in the Kanana 2 family. It has 3 scored results in 2 categories. It has no index.","logoUrl":null,"releaseDate":"2026-07-24","contextWindowTokens":32000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":23.200000311172417,"math":8.80038585708592},"capabilityCounts":{"instruction":2,"math":1},"benchmarks":[{"benchmarkId":"ifeval","benchmarkName":"Instruction-Following Eval","category":"instruction","score":80.96,"unit":"percent","family":"ifeval","index":26.229132438012606,"aboveFit":false,"run":null,"source":"Jeffrey Zhou, Tianjian Lu, Swaroop Mishra, Siddhartha Brahma, Sujoy Basu, Yi Luan, Denny Zhou, Le Hou","sourceUrl":"https://arxiv.org/abs/2311.07911","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"math500","benchmarkName":"MATH-500 Problem Set","category":"math","score":61.2,"unit":"percent","family":"math-500","index":8.80038585708592,"aboveFit":false,"run":null,"source":"Dan Hendrycks, Collin Burns, Saurav Kadavath, Akul Arora, Steven Basart, Eric Tang, Dawn Song, Jacob Steinhardt","sourceUrl":"https://arxiv.org/abs/2103.03874","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"ifBench","benchmarkName":"Instruction Following Benchmark","category":"instruction","score":33.33,"unit":"percent","family":"ifbench-family","index":20.170868184332228,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":15,"benchmarkCount":3,"categoryCount":2,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"decision-index-0-3-kas-4b","providerId":"kpiya","slug":"decision-index-0-3-kas-4b","name":"Kas 4B","provider":"kpiya","description":"Kas 4B is a model from kpiya. It has no scored result.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"katcoderprov25","providerId":"kwaipilot","slug":"kwaipilot-kat-coder-pro-v2-5","name":"KAT-Coder-Pro V2.5","provider":"Kwaipilot","description":"KAT-Coder-Pro V2.5 is a model from Kwaipilot. It has no scored result.","logoUrl":null,"releaseDate":"2026-07-10T20:16:29.000Z","contextWindowTokens":262144,"modalities":["text"],"openWeights":null,"details":{"apiId":"kwaipilot/kat-coder-pro-v2.5","maxOutputTokens":235929,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.15,"weightsUrl":null},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":0.74,"outputPerMillion":2.96,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"kev-0-5b","providerId":"jared-palmer","slug":"kev-0-5b","name":"kev 0.5B","provider":"Jared Palmer","description":"kev 0.5B is a standard model from Jared Palmer. It has 2 scored results in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":55.91245482243339,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"reasoning":2},"benchmarks":[{"benchmarkId":"jevBench14","benchmarkName":"jevBench14","category":"reasoning","score":18.88295792074435,"unit":"percent","family":"jevBench14","index":60.35098196267601,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"jevBench15","benchmarkName":"jevBench15","category":"reasoning","score":0,"unit":"percent","family":"jevBench15","index":51.47392768219078,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":15,"benchmarkCount":2,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"kev-0-6b","providerId":"jared-palmer","slug":"kev-0-6b","name":"kev 0.6B","provider":"Jared Palmer","description":"kev 0.6B is a standard model from Jared Palmer. It has 2 scored results in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":58.998298974460475,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"reasoning":2},"benchmarks":[{"benchmarkId":"jevBench14","benchmarkName":"jevBench14","category":"reasoning","score":24.750427849352384,"unit":"percent","family":"jevBench14","index":65.75864795242772,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"jevBench15","benchmarkName":"jevBench15","category":"reasoning","score":1.2616561727973594,"unit":"percent","family":"jevBench15","index":52.23794999649323,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":15,"benchmarkCount":2,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"decision-index-0-3-kev-27b","providerId":"jaredpalmer","slug":"decision-index-0-3-kev-27b","name":"Kev 27B","provider":"jaredpalmer","description":"Kev 27B is a model from jaredpalmer. It has no scored result.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"kev-4b","providerId":"jared-palmer","slug":"kev-4b","name":"kev 4B","provider":"Jared Palmer","description":"kev 4B is a standard model from Jared Palmer. It has 2 scored results in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":75.39118146186706,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"reasoning":2},"benchmarks":[{"benchmarkId":"jevBench15","benchmarkName":"jevBench15","category":"reasoning","score":38.073131188431404,"unit":"percent","family":"jevBench15","index":74.529909518217,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"jevBench14","benchmarkName":"jevBench14","category":"reasoning","score":36.136502368258114,"unit":"percent","family":"jevBench14","index":76.25245340551713,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":15,"benchmarkCount":2,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"decision-index-0-3-kev-4b-r10","providerId":"jaredpalmer","slug":"decision-index-0-3-kev-4b-r10","name":"Kev 4B r10","provider":"jaredpalmer","description":"Kev 4B r10 is a model from jaredpalmer. It has no scored result.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"kev-8b","providerId":"jared-palmer","slug":"kev-8b","name":"kev 8B","provider":"Jared Palmer","description":"kev 8B is a standard model from Jared Palmer. It has 2 scored results in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":69.33198218136768,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"reasoning":2},"benchmarks":[{"benchmarkId":"jevBench15","benchmarkName":"jevBench15","category":"reasoning","score":34.15264116882949,"unit":"percent","family":"jevBench15","index":72.15577469245491,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"jevBench14","benchmarkName":"jevBench14","category":"reasoning","score":25.56370172722739,"unit":"percent","family":"jevBench14","index":66.50818967028046,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":15,"benchmarkCount":2,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"kimik20905","providerId":"moonshot-ai","slug":"moonshot-ai-kimi-k2-0905","name":"Kimi K2 0905","provider":"Moonshot AI","description":"Kimi K2 0905 is a model from Moonshot AI. It has 2 scored results in 2 categories. It has no index.","logoUrl":null,"releaseDate":"2025-09-04T21:25:47.000Z","contextWindowTokens":262144,"modalities":["text"],"openWeights":true,"details":{"apiId":"moonshotai/kimi-k2-0905","maxOutputTokens":98304,"knowledgeCutoff":"2024-12-31","retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":null,"weightsUrl":"https://huggingface.co/moonshotai/Kimi-K2-Instruct-0905"},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":45.356016179165714,"coding":53.74202706767051,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"coding":1,"agentic":1},"benchmarks":[{"benchmarkId":"swebench-experiments:swe-bench-verified:overall","benchmarkName":"SWE-bench Verified","category":"coding","domain":null,"eligible":true,"ranked":true,"score":71.2,"unit":"percent","family":"swe-bench-verified","index":53.74202706767051,"aboveFit":false,"run":"Lingxi v1.5","source":"SWE-bench team","sourceUrl":"https://github.com/SWE-bench/experiments/tree/main/evaluation/verified","measuredAt":"2026-09-01","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:terminal-bench:overall","benchmarkName":"Terminal-Bench 1.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":37.5,"unit":"percent","family":"vals-ai:terminal-bench:overall","index":45.356016179165714,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench","measuredAt":"2026-01-12","stderr":5.413,"costPerTest":0.433024,"latency":149.814}],"price":{"inputPerMillion":0.6,"outputPerMillion":2.5,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":34,"timeToFirstToken":0.853,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":40,"benchmarkCount":2,"categoryCount":2,"confidence":"unranked","state":"partial"},"hosts":[{"host":"io-net","name":"IO.NET","kind":"host","url":"https://io.net/docs/guides/intelligence/io-intelligence","inputPerMillion":0.57,"outputPerMillion":2.3,"cacheReadPerMillion":0.285,"contextTokens":32768,"maxOutputTokens":4096,"deprecated":false},{"host":"novita-ai","name":"Novita AI","kind":"host","url":"https://novita.ai/docs/guides/introduction","inputPerMillion":0.6,"outputPerMillion":2.5,"cacheReadPerMillion":null,"contextTokens":262144,"maxOutputTokens":98304,"deprecated":false},{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":0.6,"outputPerMillion":2.5,"cacheReadPerMillion":null,"contextTokens":262144,"maxOutputTokens":98304,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:swe-bench-team","name":"SWE-bench team","url":"https://github.com","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"kimik26thinking","providerId":"moonshot-ai","slug":"moonshot-ai-kimi-k2-6-thinking","name":"Kimi K2.6 Thinking","provider":"Moonshot AI","description":"Kimi K2.6 Thinking is a model from Moonshot AI. It has 6 scored results in 6 categories. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":57.54979971419629,"coding":65.32153176679138,"reasoning":55.22087291565681,"multimodal":null,"knowledge":59.52928402622579,"multilingual":null,"instruction":56.811515904503764,"math":57.23225986099874},"capabilityCounts":{"math":1,"reasoning":1,"coding":1,"knowledge":1,"instruction":1,"agentic":1},"benchmarks":[{"benchmarkId":"livebench:livebench-math:overall","benchmarkName":"LiveBench Mathematics","category":"math","domain":null,"eligible":true,"ranked":true,"score":84.27625,"unit":"percent","family":"livebench:livebench-math:overall","index":57.23225986099874,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Mathematics","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.09450791556728232,"latency":null},{"benchmarkId":"livebench:livebench-reasoning:overall","benchmarkName":"LiveBench Reasoning","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":79.375,"unit":"percent","family":"livebench","index":64.44479477955232,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Reasoning","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.08921732673267328,"latency":null},{"benchmarkId":"livebench:livebench-coding:overall","benchmarkName":"LiveBench Coding","category":"coding","domain":null,"eligible":true,"ranked":true,"score":78.56700000000001,"unit":"percent","family":"livebench:livebench-coding:overall","index":65.32153176679138,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Coding","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.02793162393162393,"latency":null},{"benchmarkId":"livebench:livebench-language:overall","benchmarkName":"LiveBench Language","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":75.14133333333334,"unit":"percent","family":"livebench:livebench-language:overall","index":59.52928402622579,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Language","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.06837066666666668,"latency":null},{"benchmarkId":"livebench:livebench-data-analysis:overall","benchmarkName":"LiveBench Data Analysis","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":65.13433333333334,"unit":"percent","family":"livebench","index":45.99695105176128,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Data%20Analysis","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.1250966666666667,"latency":null},{"benchmarkId":"livebench:livebench-instruction-following:overall","benchmarkName":"LiveBench Instruction Following","category":"instruction","domain":null,"eligible":true,"ranked":true,"score":64.35825,"unit":"percent","family":"livebench:livebench-instruction-following:overall","index":56.811515904503764,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=IF","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.0670375,"latency":null},{"benchmarkId":"livebench:livebench-agentic-coding:overall","benchmarkName":"LiveBench Agentic Coding","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":46.919333333333334,"unit":"percent","family":"livebench:livebench-agentic-coding:overall","index":57.54979971419629,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Agentic%20Coding","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.7166305555555554,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":85,"benchmarkCount":6,"categoryCount":6,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:livebench","name":"LiveBench","url":"https://livebench.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"kimilatest","providerId":"moonshot-ai","slug":"moonshot-ai-kimi-latest","name":"Kimi Latest","provider":"Moonshot AI","description":"Kimi Latest is a model from Moonshot AI. It has no scored result.","logoUrl":null,"releaseDate":"2026-04-27T19:33:48.000Z","contextWindowTokens":1048576,"modalities":["text","image","video"],"openWeights":null,"details":{"apiId":"~moonshotai/kimi-latest","maxOutputTokens":943718,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.18,"weightsUrl":null},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":0.18239999999999998,"outputPerMillion":11.0055,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":44,"timeToFirstToken":3.265,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"kimi-audio-7b","providerId":"moonshot-ai","slug":"kimi-audio-7b","name":"Kimi-Audio 7B","provider":"Moonshot AI","description":"Kimi-Audio 7B is a standard model from Moonshot AI. It has no scored result.","logoUrl":null,"releaseDate":"2025-04-25","contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"decision-index-0-3-kodiak-v0-2-1b-accuracy","providerId":"cortex-agent-llc","slug":"decision-index-0-3-kodiak-v0-2-1b-accuracy","name":"Kodiak v0.2 1B (accuracy)","provider":"cortex-agent-llc","description":"Kodiak v0.2 1B (accuracy) is a model from cortex-agent-llc. It has no scored result.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"kokoro-82m","providerId":"hexgrad","slug":"kokoro-82m","name":"Kokoro 82M","provider":"hexgrad","description":"Kokoro 82M is a standard model from hexgrad. It has no scored result.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"labsdevstralsmall2512","providerId":"mistral","slug":"mistral-labs-devstral-small","name":"Labs Devstral Small","provider":"Mistral","description":"Labs Devstral Small is a model from Mistral. It has 3 scored results in 2 categories. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":47.165224329822124,"coding":33.27730632815756,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"coding":2,"agentic":1},"benchmarks":[{"benchmarkId":"vals-ai:lcb:overall","benchmarkName":"LiveCodeBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":43.178,"unit":"percent","family":"livecodebench","index":25.17033328937904,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","measuredAt":"2026-09-01","stderr":1.155,"costPerTest":0.000346,"latency":18.821},{"benchmarkId":"vals-ai:terminal-bench:overall","benchmarkName":"Terminal-Bench 1.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":40,"unit":"percent","family":"vals-ai:terminal-bench:overall","index":47.165224329822124,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench","measuredAt":"2026-01-12","stderr":5.477,"costPerTest":0.135297,"latency":285.171},{"benchmarkId":"vals-ai:ioi-v1:overall","benchmarkName":"IOI v1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":2.5,"unit":"percent","family":"vals-ai:ioi-v1:overall","index":41.38427936693607,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/ioi-v1","measuredAt":"2026-08-09","stderr":1.781,"costPerTest":0.50569,"latency":7466.603}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":40,"benchmarkCount":3,"categoryCount":2,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"laguna-m-1","providerId":"poolside","slug":"laguna-m-1","name":"Laguna M.1","provider":"Poolside","description":"Laguna M.1 is a reasoning model from Poolside in the Laguna family. It has 9 scored results in 3 categories. It has no index.","logoUrl":null,"releaseDate":"2026-04-28","contextWindowTokens":256000,"modalities":["text"],"openWeights":false,"details":null,"reasoning":true,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":44.348570222745636,"coding":48.203633594940555,"reasoning":null,"multimodal":null,"knowledge":18.105444245379886,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"coding":4,"knowledge":2,"agentic":3},"benchmarks":[{"benchmarkId":"vals-ai:legal_bench:overall","benchmarkName":"LegalBench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":75.144,"unit":"percent","family":"vals-ai:legal_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","measuredAt":"2026-10-13","stderr":0.591,"costPerTest":null,"latency":42.363},{"benchmarkId":"sweVerified","benchmarkName":"Software Engineering Benchmark Verified","category":"coding","score":74.6,"unit":"percent","family":"swe-bench-verified","index":56.24972283213506,"aboveFit":false,"run":null,"source":"Carlos E. Jimenez, John Yang, Alexander Wettig, Shunyu Yao, Kexin Pei, Ofir Press, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2310.06770","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:mmlu_pro:overall","benchmarkName":"MMLU Pro","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":68.84,"unit":"percent","family":"mmlu-pro","index":31.210888490759764,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","measuredAt":"2026-09-01","stderr":0.452,"costPerTest":0.001794,"latency":99.583},{"benchmarkId":"vals-ai:lcb:overall","benchmarkName":"LiveCodeBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":68.117,"unit":"percent","family":"livecodebench","index":45.86324979947836,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","measuredAt":"2026-09-01","stderr":1.136,"costPerTest":0.005654,"latency":235.323},{"benchmarkId":"vals-ai:medscribe:overall","benchmarkName":"MedScribe","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":65.911,"unit":"percent","family":"vals-ai:medscribe:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medscribe","measuredAt":"2026-10-29","stderr":2.007,"costPerTest":0.002202,"latency":111.816},{"benchmarkId":"vals-ai:corp_fin_v2:overall","benchmarkName":"CorpFin v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":58.159,"unit":"percent","family":"vals-ai:corp_fin_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","measuredAt":"2026-08-12","stderr":0.971,"costPerTest":null,"latency":211.713},{"benchmarkId":"vals-ai:swebench:overall","benchmarkName":"SWE-bench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":57.6,"unit":"percent","family":"swe-bench-verified","index":43.711244009812305,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/swebench","measuredAt":"2026-09-01","stderr":2.212,"costPerTest":0.520034,"latency":885.993},{"benchmarkId":"swePro","benchmarkName":"SWE-bench Pro","category":"coding","score":49.2,"unit":"percent","family":"swePro","index":50.73859474836229,"aboveFit":false,"run":null,"source":"Xiang Deng, Jeff Da, Edwin Pan, Yannis Yiming He, Charles Ide, Kanak Garg, Niklas Lauffer, Andrew Park, Nitin Pasari, Chetan Rane, Karmini Sampath, Maya Krishnan, Srivatsa Kundurthy, Sean Hendryx, Zifan Wang, Chen Bo Calvin Zhang, Noah Jacobson, Bing Liu, Brad Kenstler","sourceUrl":"https://arxiv.org/abs/2509.16941","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"terminalBench2","benchmarkName":"Terminal-Bench 2.0","category":"agentic","score":45.8,"unit":"percent","family":"terminalBench2","index":46.47726220037741,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:public-benefits-bench:overall","benchmarkName":"Public Benefits Bench v1.1","category":null,"domain":"Public benefits","eligible":false,"ranked":true,"score":43.843,"unit":"percent","family":"vals-ai:public-benefits-bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/public-benefits-bench","measuredAt":"2026-10-12","stderr":1.291,"costPerTest":0.201829,"latency":334.832},{"benchmarkId":"vals-ai:terminal-bench-2-1:overall","benchmarkName":"Terminal-Bench 2.1","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":34.082,"unit":"percent","family":"terminal-bench-2-1","index":41.074161739323046,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2-1","measuredAt":"2026-10-01","stderr":1.633,"costPerTest":0.300575,"latency":1209.608},{"benchmarkId":"vals-ai:terminal-bench-2:overall","benchmarkName":"Terminal-Bench 2.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":31.461,"unit":"percent","family":"vals-ai:terminal-bench-2:overall","index":45.49428672853645,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2","measuredAt":"2026-06-04","stderr":4.95,"costPerTest":null,"latency":0},{"benchmarkId":"vals-ai:gpqa:overall","benchmarkName":"GPQA Diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":27.02,"unit":"percent","family":"gpqa-diamond","index":5,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","measuredAt":"2026-09-01","stderr":1.922,"costPerTest":null,"latency":327.957},{"benchmarkId":"vals-ai:fabv2:overall","benchmarkName":"Finance Agent (v2)","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":25.026,"unit":"percent","family":"vals-ai:fabv2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/fabv2","measuredAt":"2026-10-29","stderr":1.277,"costPerTest":0.818192,"latency":1223.035},{"benchmarkId":"vals-ai:medcode:overall","benchmarkName":"MedCode","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":23.106,"unit":"percent","family":"vals-ai:medcode:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medcode","measuredAt":"2026-10-29","stderr":1.693,"costPerTest":0.002973,"latency":70.995},{"benchmarkId":"vals-ai:vibe-code:overall","benchmarkName":"Vibe Code Bench v1.1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":11.038,"unit":"percent","family":"vals-ai:vibe-code:overall","index":46.23220641094789,"aboveFit":false,"run":"OpenHands","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","measuredAt":"2026-11-09","stderr":2.822,"costPerTest":2.059401,"latency":2381.926},{"benchmarkId":"vals-ai:legal_research:overall","benchmarkName":"Legal Research Bench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":2.404,"unit":"percent","family":"vals-ai:legal_research:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_research","measuredAt":"2026-11-09","stderr":1.065,"costPerTest":0.219839,"latency":466.26},{"benchmarkId":"vals-ai:tax_eval_v2:overall","benchmarkName":"TaxEval v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":1.636,"unit":"percent","family":"vals-ai:tax_eval_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","measuredAt":"2026-09-01","stderr":0.256,"costPerTest":null,"latency":212.643},{"benchmarkId":"vals-ai:programbench:overall","benchmarkName":"ProgramBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":0,"unit":"percent","family":"vals-ai:programbench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/programbench","measuredAt":"2026-11-09","stderr":0,"costPerTest":null,"latency":4921.898}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":55.00000000000001,"benchmarkCount":9,"categoryCount":3,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"laguna-s-2-1","providerId":"poolside","slug":"laguna-s-2-1","name":"Laguna S 2.1","provider":"Poolside","description":"Laguna S 2.1 is a reasoning model from Poolside in the Laguna family. It has 4 scored results in 2 categories. It has no index.","logoUrl":null,"releaseDate":"2026-07-21","contextWindowTokens":1048576,"modalities":["text"],"openWeights":true,"details":{"apiId":"poolside/laguna-s-2.1","maxOutputTokens":131072,"knowledgeCutoff":null,"retiresAt":"2026-10-31","tools":true,"structuredOutput":false,"cacheReadPerMillion":0.009,"weightsUrl":"https://huggingface.co/poolside/Laguna-S-2.1"},"reasoning":true,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":54.171035055231876,"coding":59.42501655647971,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"agentic":3,"coding":1},"benchmarks":[{"benchmarkId":"terminalBench21","benchmarkName":"Terminal-Bench 2.1","category":"agentic","score":70.2,"unit":"percent","family":"terminal-bench-2-1","index":62.197967793358046,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"swePro","benchmarkName":"SWE-bench Pro","category":"coding","score":59.4,"unit":"percent","family":"swePro","index":59.42501655647971,"aboveFit":false,"run":null,"source":"Xiang Deng, Jeff Da, Edwin Pan, Yannis Yiming He, Charles Ide, Kanak Garg, Niklas Lauffer, Andrew Park, Nitin Pasari, Chetan Rane, Karmini Sampath, Maya Krishnan, Srivatsa Kundurthy, Sean Hendryx, Zifan Wang, Chen Bo Calvin Zhang, Noah Jacobson, Bing Liu, Brad Kenstler","sourceUrl":"https://arxiv.org/abs/2509.16941","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"toolathlonVerified","benchmarkName":"Toolathlon-Verified","category":"agentic","score":49.7,"unit":"percent","family":"toolathlonVerified","index":49.86874454611567,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"deepSwe","benchmarkName":"DeepSWE","category":"agentic","score":40.4,"unit":"percent","family":"deepSwe","index":50.44639282622191,"aboveFit":false,"run":null,"source":"Datacurve AI","sourceUrl":"https://deepswe.datacurve.ai/blog","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":0.09,"outputPerMillion":0.18,"batch":null,"freeTier":true,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":99,"timeToFirstToken":0.403,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":40,"benchmarkCount":4,"categoryCount":2,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"laguna-xs-2-1","providerId":"poolside","slug":"laguna-xs-2-1","name":"Laguna XS 2.1","provider":"Poolside","description":"Laguna XS 2.1 is a reasoning model from Poolside in the Laguna family. It has 3 scored results in 2 categories. It has no index.","logoUrl":null,"releaseDate":"2026-07-02","contextWindowTokens":262144,"modalities":["text"],"openWeights":true,"details":{"apiId":"poolside/laguna-xs-2.1","maxOutputTokens":32768,"knowledgeCutoff":null,"retiresAt":"2026-10-31","tools":true,"structuredOutput":false,"cacheReadPerMillion":0.03,"weightsUrl":"https://huggingface.co/poolside/Laguna-XS-2.1"},"reasoning":true,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":40.57309454729956,"coding":51.448389286398466,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"coding":2,"agentic":1},"benchmarks":[{"benchmarkId":"sweVerified","benchmarkName":"Software Engineering Benchmark Verified","category":"coding","score":70.9,"unit":"percent","family":"swe-bench-verified","index":53.52075979433541,"aboveFit":false,"run":null,"source":"Carlos E. Jimenez, John Yang, Alexander Wettig, Shunyu Yao, Kexin Pei, Ofir Press, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2310.06770","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"swePro","benchmarkName":"SWE-bench Pro","category":"coding","score":47.6,"unit":"percent","family":"swePro","index":49.376018778461514,"aboveFit":false,"run":null,"source":"Xiang Deng, Jeff Da, Edwin Pan, Yannis Yiming He, Charles Ide, Kanak Garg, Niklas Lauffer, Andrew Park, Nitin Pasari, Chetan Rane, Karmini Sampath, Maya Krishnan, Srivatsa Kundurthy, Sean Hendryx, Zifan Wang, Chen Bo Calvin Zhang, Noah Jacobson, Bing Liu, Brad Kenstler","sourceUrl":"https://arxiv.org/abs/2509.16941","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"terminalBench2","benchmarkName":"Terminal-Bench 2.0","category":"agentic","score":37.5,"unit":"percent","family":"terminalBench2","index":40.57309454729956,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":0.06,"outputPerMillion":0.12,"batch":null,"freeTier":true,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":134,"timeToFirstToken":0.345,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":40,"benchmarkCount":3,"categoryCount":2,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"laguna-xs-2","providerId":"poolside","slug":"laguna-xs-2","name":"Laguna XS.2","provider":"Poolside","description":"Laguna XS.2 is a reasoning model from Poolside in the Laguna family. It has 9 scored results in 3 categories. It has no index.","logoUrl":null,"releaseDate":"2026-04-28","contextWindowTokens":256000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":true,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":39.67092690798009,"coding":46.34076380318104,"reasoning":null,"multimodal":null,"knowledge":29.587459923294794,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"coding":4,"knowledge":2,"agentic":3},"benchmarks":[{"benchmarkId":"vals-ai:legal_bench:overall","benchmarkName":"LegalBench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":71.026,"unit":"percent","family":"vals-ai:legal_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","measuredAt":"2026-10-13","stderr":0.563,"costPerTest":null,"latency":11.924},{"benchmarkId":"sweVerified","benchmarkName":"Software Engineering Benchmark Verified","category":"coding","score":69.9,"unit":"percent","family":"swe-bench-verified","index":52.783202216551715,"aboveFit":false,"run":null,"source":"Carlos E. Jimenez, John Yang, Alexander Wettig, Shunyu Yao, Kexin Pei, Ofir Press, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2310.06770","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:mmlu_pro:overall","benchmarkName":"MMLU Pro","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":69.05,"unit":"percent","family":"mmlu-pro","index":31.475463354405896,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","measuredAt":"2026-09-01","stderr":0.48,"costPerTest":0.000605,"latency":47.986},{"benchmarkId":"vals-ai:lcb:overall","benchmarkName":"LiveCodeBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":67.802,"unit":"percent","family":"livecodebench","index":45.6018813123425,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","measuredAt":"2026-09-01","stderr":1.137,"costPerTest":0.002349,"latency":186.427},{"benchmarkId":"vals-ai:medscribe:overall","benchmarkName":"MedScribe","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":61.432,"unit":"percent","family":"vals-ai:medscribe:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medscribe","measuredAt":"2026-10-29","stderr":2.349,"costPerTest":0.001126,"latency":68.824},{"benchmarkId":"vals-ai:tax_eval_v2:overall","benchmarkName":"TaxEval v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":58.953,"unit":"percent","family":"vals-ai:tax_eval_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","measuredAt":"2026-09-01","stderr":0.964,"costPerTest":0.000305,"latency":27.449},{"benchmarkId":"vals-ai:corp_fin_v2:overall","benchmarkName":"CorpFin v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":56.333,"unit":"percent","family":"vals-ai:corp_fin_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","measuredAt":"2026-08-12","stderr":0.976,"costPerTest":null,"latency":188.244},{"benchmarkId":"vals-ai:swebench:overall","benchmarkName":"SWE-bench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":55.2,"unit":"percent","family":"swe-bench-verified","index":41.941105823131444,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/swebench","measuredAt":"2026-09-01","stderr":2.226,"costPerTest":0.170419,"latency":270.261},{"benchmarkId":"vals-ai:gpqa:overall","benchmarkName":"GPQA Diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":55.05,"unit":"percent","family":"gpqa-diamond","index":27.699456492183696,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","measuredAt":"2026-09-01","stderr":2.499,"costPerTest":0.001373,"latency":69.312},{"benchmarkId":"swePro","benchmarkName":"SWE-bench Pro","category":"coding","score":46.3,"unit":"percent","family":"swePro","index":48.26892580291714,"aboveFit":false,"run":null,"source":"Xiang Deng, Jeff Da, Edwin Pan, Yannis Yiming He, Charles Ide, Kanak Garg, Niklas Lauffer, Andrew Park, Nitin Pasari, Chetan Rane, Karmini Sampath, Maya Krishnan, Srivatsa Kundurthy, Sean Hendryx, Zifan Wang, Chen Bo Calvin Zhang, Noah Jacobson, Bing Liu, Brad Kenstler","sourceUrl":"https://arxiv.org/abs/2509.16941","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:public-benefits-bench:overall","benchmarkName":"Public Benefits Bench v1.1","category":null,"domain":"Public benefits","eligible":false,"ranked":true,"score":41.407,"unit":"percent","family":"vals-ai:public-benefits-bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/public-benefits-bench","measuredAt":"2026-10-12","stderr":1.281,"costPerTest":0.113029,"latency":305.41},{"benchmarkId":"terminalBench2","benchmarkName":"Terminal-Bench 2.0","category":"agentic","score":35.7,"unit":"percent","family":"terminalBench2","index":39.292672646632084,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:terminal-bench-2:overall","benchmarkName":"Terminal-Bench 2.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":28.09,"unit":"percent","family":"vals-ai:terminal-bench-2:overall","index":43.464569685878004,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2","measuredAt":"2026-06-04","stderr":4.791,"costPerTest":null,"latency":0},{"benchmarkId":"vals-ai:terminal-bench-2-1:overall","benchmarkName":"Terminal-Bench 2.1","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":25.843,"unit":"percent","family":"terminal-bench-2-1","index":36.255538391430186,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2-1","measuredAt":"2026-10-01","stderr":2.973,"costPerTest":0.253714,"latency":1218.578},{"benchmarkId":"vals-ai:medcode:overall","benchmarkName":"MedCode","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":21.251,"unit":"percent","family":"vals-ai:medcode:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medcode","measuredAt":"2026-10-29","stderr":1.703,"costPerTest":0.001445,"latency":33.094},{"benchmarkId":"vals-ai:fabv2:overall","benchmarkName":"Finance Agent (v2)","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":15.601,"unit":"percent","family":"vals-ai:fabv2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/fabv2","measuredAt":"2026-10-29","stderr":0.326,"costPerTest":0.19216,"latency":526.624},{"benchmarkId":"vals-ai:vibe-code:overall","benchmarkName":"Vibe Code Bench v1.1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":5.212,"unit":"percent","family":"vals-ai:vibe-code:overall","index":44.130094077622935,"aboveFit":false,"run":"OpenHands","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","measuredAt":"2026-11-09","stderr":2.18,"costPerTest":0.981155,"latency":2199.244},{"benchmarkId":"vals-ai:legal_research:overall","benchmarkName":"Legal Research Bench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":0.962,"unit":"percent","family":"vals-ai:legal_research:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_research","measuredAt":"2026-11-09","stderr":0.678,"costPerTest":0.107892,"latency":352.036},{"benchmarkId":"vals-ai:programbench:overall","benchmarkName":"ProgramBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":0,"unit":"percent","family":"vals-ai:programbench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/programbench","measuredAt":"2026-11-09","stderr":0,"costPerTest":null,"latency":1541.675}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":55.00000000000001,"benchmarkCount":9,"categoryCount":3,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"decision-index-0-3-lavoir","providerId":"moganai","slug":"decision-index-0-3-lavoir","name":"Lavoir","provider":"moganai","description":"Lavoir is a model from moganai. It has no scored result.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"laya","providerId":"convai-innovations","slug":"laya","name":"Laya","provider":"Convai Innovations","description":"Laya is a standard model from Convai Innovations. It has 2 scored results in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":61.15117766004816,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"reasoning":2},"benchmarks":[{"benchmarkId":"jevBench14","benchmarkName":"jevBench14","category":"reasoning","score":30.251281619829957,"unit":"percent","family":"jevBench14","index":70.82842763790555,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"jevBench15","benchmarkName":"jevBench15","category":"reasoning","score":0,"unit":"percent","family":"jevBench15","index":51.47392768219078,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":15,"benchmarkCount":2,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"laya-multilingual","providerId":"convai-innovations","slug":"laya-multilingual","name":"Laya multilingual","provider":"Convai Innovations","description":"Laya multilingual is a standard model from Convai Innovations. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":51.48525703793339,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"reasoning":1},"benchmarks":[{"benchmarkId":"jevBench15","benchmarkName":"jevBench15","category":"reasoning","score":0.018708552536889286,"unit":"percent","family":"jevBench15","index":51.48525703793339,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":15,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"laya-typed-decisions","providerId":"convai-innovations","slug":"laya-typed-decisions","name":"Laya typed-decisions","provider":"Convai Innovations","description":"Laya typed-decisions is a standard model from Convai Innovations. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":51.47392768219078,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"reasoning":1},"benchmarks":[{"benchmarkId":"jevBench15","benchmarkName":"jevBench15","category":"reasoning","score":0,"unit":"percent","family":"jevBench15","index":51.47392768219078,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":15,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"leanstral","providerId":"mistral","slug":"leanstral","name":"Leanstral","provider":"Mistral","description":"Leanstral is a reasoning model from Mistral. It has no scored result.","logoUrl":null,"releaseDate":"2026-03-16","contextWindowTokens":256000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":true,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"leanstral-1-5","providerId":"mistral","slug":"leanstral-1-5","name":"Leanstral 1.5","provider":"Mistral","description":"Leanstral 1.5 is a reasoning model from Mistral in the Leanstral family. It has no scored result.","logoUrl":null,"releaseDate":"2026-07-01","contextWindowTokens":256000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":true,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"decision-index-0-3-lev","providerId":"interfaze-ai","slug":"decision-index-0-3-lev","name":"lev","provider":"interfaze-ai","description":"lev is a model from interfaze-ai. It has no scored result.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"lev-350m","providerId":"franck-verrot-franckverrot","slug":"lev-350m","name":"lev-350m","provider":"Franck Verrot (franckverrot)","description":"lev-350m is a standard model from Franck Verrot (franckverrot). It has 2 scored results in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":60.344677007348025,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"reasoning":2},"benchmarks":[{"benchmarkId":"jevBench14","benchmarkName":"jevBench14","category":"reasoning","score":28.501129766483672,"unit":"percent","family":"jevBench14","index":69.21542633250527,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"jevBench15","benchmarkName":"jevBench15","category":"reasoning","score":0,"unit":"percent","family":"jevBench15","index":51.47392768219078,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":15,"benchmarkCount":2,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"lfg-1","providerId":"glenn2","slug":"lfg-1","name":"LFG-1","provider":"glenn2","description":"LFG-1 is a standard model from glenn2. It has no scored result.","logoUrl":null,"releaseDate":"2026-04-20","contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"lfg-2","providerId":"glenn2","slug":"lfg-2","name":"LFG-2","provider":"glenn2","description":"LFG-2 is a standard model from glenn2. It has no scored result.","logoUrl":null,"releaseDate":"2026-06-30","contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"lfg-3","providerId":"glenn2","slug":"lfg-3","name":"LFG-3","provider":"glenn2","description":"LFG-3 is a standard model from glenn2. It has no scored result.","logoUrl":null,"releaseDate":"2026-08-06","contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"lfm212b","providerId":"liquidai","slug":"liquidai-lfm2-1-2b","name":"LFM2 1.2B","provider":"LiquidAI","description":"LFM2 1.2B is a model from LiquidAI. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":5,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-french:overall","benchmarkName":"EuroEval French","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":24.47125,"unit":"percent","family":"euroeval","index":5.422091034300159,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/french","measuredAt":null,"stderr":0.27934913191202154,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-portuguese:overall","benchmarkName":"EuroEval Portuguese","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":24.44125,"unit":"percent","family":"euroeval","index":5.381387937103497,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/portuguese","measuredAt":null,"stderr":0.4379586881202381,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-spanish:overall","benchmarkName":"EuroEval Spanish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":22.2575,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/spanish","measuredAt":null,"stderr":0.4075766799020768,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":20.43875,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.47363290109535255,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-italian:overall","benchmarkName":"EuroEval Italian","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":19.48875,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/italian","measuredAt":null,"stderr":0.34028480718362963,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-dutch:overall","benchmarkName":"EuroEval Dutch","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":18.15666666666667,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/dutch","measuredAt":null,"stderr":0.4459820624195551,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":12.44625,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":0.560041851114718,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":10.235714285714284,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.6304371661063165,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"lfm2350m","providerId":"liquidai","slug":"liquidai-lfm2-350m","name":"LFM2 350M","provider":"LiquidAI","description":"LFM2 350M is a model from LiquidAI. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":5,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-portuguese:overall","benchmarkName":"EuroEval Portuguese","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":16.89875,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/portuguese","measuredAt":null,"stderr":0.49599804687115456,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-italian:overall","benchmarkName":"EuroEval Italian","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":15.2425,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/italian","measuredAt":null,"stderr":0.4206616811643295,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-french:overall","benchmarkName":"EuroEval French","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":14.084999999999999,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/french","measuredAt":null,"stderr":0.6226819513202546,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-spanish:overall","benchmarkName":"EuroEval Spanish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":13.992499999999998,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/spanish","measuredAt":null,"stderr":0.3128897569432403,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":13.9225,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.40202806183150946,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-dutch:overall","benchmarkName":"EuroEval Dutch","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":10.373333333333333,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/dutch","measuredAt":null,"stderr":0.4804704484688437,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":6.87375,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":0.5408139120806713,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":6.307142857142857,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.371249957053526,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"lfm2700m","providerId":"liquidai","slug":"liquidai-lfm2-700m","name":"LFM2 700M","provider":"LiquidAI","description":"LFM2 700M is a model from LiquidAI. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":5,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-french:overall","benchmarkName":"EuroEval French","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":23.17,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/french","measuredAt":null,"stderr":0.30102169772293824,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-portuguese:overall","benchmarkName":"EuroEval Portuguese","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":20.353749999999998,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/portuguese","measuredAt":null,"stderr":0.3831632321870145,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-spanish:overall","benchmarkName":"EuroEval Spanish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":17.756249999999998,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/spanish","measuredAt":null,"stderr":0.3500133926009118,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-italian:overall","benchmarkName":"EuroEval Italian","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":15.410000000000004,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/italian","measuredAt":null,"stderr":0.339770142890749,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-dutch:overall","benchmarkName":"EuroEval Dutch","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":13.284444444444443,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/dutch","measuredAt":null,"stderr":0.36019542021066947,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":12.578750000000001,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.3499754455672569,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":10.67375,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":0.53984228020784,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":8.471428571428572,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.4019061724191001,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"lfm2-24b-a2b","providerId":"liquidai","slug":"lfm2-24b-a2b","name":"LFM2-24B-A2B","provider":"LiquidAI","description":"LFM2-24B-A2B is a standard model from LiquidAI. It has no scored result.","logoUrl":null,"releaseDate":"2026-02-24","contextWindowTokens":32000,"modalities":["text"],"openWeights":false,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":92,"timeToFirstToken":0.42,"source":"BenchLM","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"lfm258ba1bbase","providerId":"liquidai","slug":"liquidai-lfm2-5-8b-a1b-base","name":"LFM2.5 8B A1B Base","provider":"LiquidAI","description":"LFM2.5 8B A1B Base is a model from LiquidAI. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":17.882326663627275,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-french:overall","benchmarkName":"EuroEval French","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":42.12,"unit":"percent","family":"euroeval","index":29.36738392261777,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/french","measuredAt":null,"stderr":0.5566641716510953,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-portuguese:overall","benchmarkName":"EuroEval Portuguese","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":36.654999999999994,"unit":"percent","family":"euroeval","index":21.952636383293125,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/portuguese","measuredAt":null,"stderr":0.6100448241727816,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-dutch:overall","benchmarkName":"EuroEval Dutch","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":36.32666666666667,"unit":"percent","family":"euroeval","index":21.507163597307482,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/dutch","measuredAt":null,"stderr":0.682811001813482,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-spanish:overall","benchmarkName":"EuroEval Spanish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":34.013749999999995,"unit":"percent","family":"euroeval","index":18.369067867603977,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/spanish","measuredAt":null,"stderr":0.6313032155786948,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-italian:overall","benchmarkName":"EuroEval Italian","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":33.29625,"unit":"percent","family":"euroeval","index":17.39558545965057,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/italian","measuredAt":null,"stderr":0.7078212609833079,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":33.082499999999996,"unit":"percent","family":"euroeval","index":17.105575892124364,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.7341661937191061,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":32.87428571428571,"unit":"percent","family":"euroeval","index":16.82307701515232,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.8136990367752562,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":29.9725,"unit":"percent","family":"euroeval","index":12.886021482737434,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":0.8981534877179957,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"lfm2-5-1-2b-instruct","providerId":"liquidai","slug":"lfm2-5-1-2b-instruct","name":"LFM2.5-1.2B-Instruct","provider":"LiquidAI","description":"LFM2.5-1.2B-Instruct is a standard model from LiquidAI in the LFM2.5 1.2B family. It has no scored result.","logoUrl":null,"releaseDate":"2026-01-05","contextWindowTokens":32000,"modalities":["text"],"openWeights":false,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"lfm2-5-1-2b-jp-202606","providerId":"liquidai","slug":"lfm2-5-1-2b-jp-202606","name":"LFM2.5-1.2B-JP-202606","provider":"LiquidAI","description":"LFM2.5-1.2B-JP-202606 is a standard model from LiquidAI in the LFM2.5-1.2B family. It has no scored result.","logoUrl":null,"releaseDate":"2026-05-26","contextWindowTokens":32000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"lfm2-5-1-2b-thinking","providerId":"liquidai","slug":"lfm2-5-1-2b-thinking","name":"LFM2.5-1.2B-Thinking","provider":"LiquidAI","description":"LFM2.5-1.2B-Thinking is a reasoning model from LiquidAI in the LFM2.5 1.2B family. It has no scored result.","logoUrl":null,"releaseDate":"2026-01-20","contextWindowTokens":32000,"modalities":["text"],"openWeights":false,"details":null,"reasoning":true,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"decision-index-0-3-lfm2-5-2-6b-rlcd","providerId":"monotykamary","slug":"decision-index-0-3-lfm2-5-2-6b-rlcd","name":"LFM2.5-2.6B-RLCD","provider":"monotykamary","description":"LFM2.5-2.6B-RLCD is a model from monotykamary. It has no scored result.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"lfm2-5-230m","providerId":"liquidai","slug":"lfm2-5-230m","name":"LFM2.5-230M","provider":"LiquidAI","description":"LFM2.5-230M is a standard model from LiquidAI. It has 5 scored results in 3 categories. It has no index.","logoUrl":null,"releaseDate":"2026-06-25","contextWindowTokens":32000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":5,"coding":null,"reasoning":null,"multimodal":null,"knowledge":5,"multilingual":null,"instruction":16.254702390290575,"math":null},"capabilityCounts":{"instruction":2,"knowledge":2,"agentic":1},"benchmarks":[{"benchmarkId":"ifeval","benchmarkName":"Instruction-Following Eval","category":"instruction","score":71.71,"unit":"percent","family":"ifeval","index":7.600567779063347,"aboveFit":false,"run":null,"source":"Jeffrey Zhou, Tianjian Lu, Swaroop Mishra, Siddhartha Brahma, Sujoy Basu, Yi Luan, Denny Zhou, Le Hou","sourceUrl":"https://arxiv.org/abs/2311.07911","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"ifBench","benchmarkName":"Instruction Following Benchmark","category":"instruction","score":38.4,"unit":"percent","family":"ifbench-family","index":24.90883700151781,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gpqa","benchmarkName":"Graduate-Level Google-Proof Q&A","category":"knowledge","score":25.41,"unit":"percent","family":"gpqa-diamond","index":5,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gpqaDiamond","benchmarkName":"GPQA Diamond","category":"knowledge","score":25.41,"unit":"percent","family":"gpqa-diamond","index":5,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"bfclV4","benchmarkName":"Berkeley Function Calling Leaderboard v4","category":"agentic","score":21.03,"unit":"percent","family":"bfclV4","index":5,"aboveFit":false,"run":null,"source":"Shishir G. Patil et al.","sourceUrl":"https://gorilla.cs.berkeley.edu/leaderboard.html","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmluPro","benchmarkName":"Massive Multitask Language Understanding Professional","category":"knowledge","score":20.25,"unit":"percent","family":"mmlu-pro","index":5,"aboveFit":false,"run":null,"source":"Yubo Wang, Xueguang Ma, Ge Zhang, Yuansheng Ni, Abhranil Chandra, Shiguang Guo, Weiming Ren, Aaran Arulraj, Xuan He, Ziyan Jiang, Tianle Li, Max Ku, Kai Wang, Alex Zhuang, Rongqi Fan, Xiang Yue, Wenhu Chen","sourceUrl":"https://arxiv.org/abs/2406.01574","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":40,"benchmarkCount":5,"categoryCount":3,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"lfm2-5-350m","providerId":"liquidai","slug":"lfm2-5-350m","name":"LFM2.5-350M","provider":"LiquidAI","description":"LFM2.5-350M is a standard model from LiquidAI. It has 3 scored results in 2 categories. It has no index.","logoUrl":null,"releaseDate":"2026-03-31","contextWindowTokens":32000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":5.155635748695872,"multilingual":null,"instruction":18.17353690981833,"math":null},"capabilityCounts":{"instruction":1,"knowledge":2},"benchmarks":[{"benchmarkId":"ifeval","benchmarkName":"Instruction-Following Eval","category":"instruction","score":76.96,"unit":"percent","family":"ifeval","index":18.17353690981833,"aboveFit":false,"run":null,"source":"Jeffrey Zhou, Tianjian Lu, Swaroop Mishra, Siddhartha Brahma, Sujoy Basu, Yi Luan, Denny Zhou, Le Hou","sourceUrl":"https://arxiv.org/abs/2311.07911","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gpqa","benchmarkName":"Graduate-Level Google-Proof Q&A","category":"knowledge","score":30.64,"unit":"percent","family":"gpqa-diamond","index":5.311271497391743,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmluPro","benchmarkName":"Massive Multitask Language Understanding Professional","category":"knowledge","score":20.01,"unit":"percent","family":"mmlu-pro","index":5,"aboveFit":false,"run":null,"source":"Yubo Wang, Xueguang Ma, Ge Zhang, Yuansheng Ni, Abhranil Chandra, Shiguang Guo, Weiming Ren, Aaran Arulraj, Xuan He, Ziyan Jiang, Tianle Li, Max Ku, Kai Wang, Alex Zhuang, Rongqi Fan, Xiang Yue, Wenhu Chen","sourceUrl":"https://arxiv.org/abs/2406.01574","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":20,"benchmarkCount":3,"categoryCount":2,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"decision-index-0-3-lfm2-5-350m-rlcd","providerId":"notnotsamuel","slug":"decision-index-0-3-lfm2-5-350m-rlcd","name":"LFM2.5-350M-RLCD","provider":"notnotsamuel","description":"LFM2.5-350M-RLCD is a model from notnotsamuel. It has no scored result.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"lfm2-5-colbert-350m","providerId":"liquidai","slug":"lfm2-5-colbert-350m","name":"LFM2.5-ColBERT-350M","provider":"LiquidAI","description":"LFM2.5-ColBERT-350M is a standard model from LiquidAI in the LFM2.5 Retrievers family. It has no scored result.","logoUrl":null,"releaseDate":"2026-06-18","contextWindowTokens":32000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[{"benchmarkId":"mkqa11","benchmarkName":"MKQA-11 multilingual retrieval","category":"multilingual","score":69.4,"unit":"percent","family":"mkqa11","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"nanoBeirMultilingual","benchmarkName":"NanoBEIR Multilingual Extended","category":"multilingual","score":60.5,"unit":"percent","family":"nanoBeirMultilingual","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"lfm2-5-vl-1-6b-extract","providerId":"liquidai","slug":"lfm2-5-vl-1-6b-extract","name":"LFM2.5-VL-1.6B-Extract","provider":"LiquidAI","description":"LFM2.5-VL-1.6B-Extract is a standard model from LiquidAI in the LFM2.5-VL Extract family. It has 7 scored results in 5 categories. It has no index.","logoUrl":null,"releaseDate":"2026-05-26","contextWindowTokens":128000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":8.429811596672522,"coding":null,"reasoning":22.073582735971804,"multimodal":5,"knowledge":17.251718656482716,"multilingual":null,"instruction":19.955930742921446,"math":null},"capabilityCounts":{"instruction":1,"knowledge":2,"multimodal":1,"agentic":1,"reasoning":2},"benchmarks":[{"benchmarkId":"liquidExtractJsonValidity","benchmarkName":"Liquid image-to-JSON extraction JSON validity","category":"multimodal","score":99.6,"unit":"percent","family":"liquidExtractJsonValidity","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"liquidExtractSchemaF1","benchmarkName":"Liquid image-to-JSON extraction schema consistency F1","category":"multimodal","score":99.6,"unit":"percent","family":"liquidExtractSchemaF1","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"liquidExtractVlmJudge","benchmarkName":"Liquid image-to-JSON extraction VLM judge score","category":"multimodal","score":90.6,"unit":"percent","family":"liquidExtractVlmJudge","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaIfBench","benchmarkName":"Artificial Analysis IFBench","category":"instruction","score":33.1,"unit":"percent","family":"ifbench-family","index":19.955930742921446,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":28.9,"unit":"percent","family":"gpqa-diamond","index":5,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaMmmuPro","benchmarkName":"Artificial Analysis MMMU-Pro","category":"multimodal","score":26.5,"unit":"percent","family":"mmmu-pro","index":5,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmmu-pro","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"tau2Bench","benchmarkName":"τ²-Bench Tool-Agent-User Evaluation","category":"agentic","score":8.5,"unit":"percent","family":"tau2-bench","index":8.429811596672522,"aboveFit":false,"run":null,"source":"Victor Barres, Honghua Dong, Soham Ray, Xujie Si, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2506.07982","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":5.1,"unit":"percent","family":"hle","index":29.503437312965428,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":0,"unit":"percent","family":"lcr","index":7.298049459170706,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":0,"unit":"percent","family":"critpt","index":36.8491160127729,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":65,"benchmarkCount":7,"categoryCount":5,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"lfm2-5-vl-3b","providerId":"liquidai","slug":"lfm2-5-vl-3b","name":"LFM2.5-VL-3B","provider":"LiquidAI","description":"LFM2.5-VL-3B is a standard model from LiquidAI. It has 9 scored results in 3 categories. It has no index.","logoUrl":null,"releaseDate":"2026-08-12","contextWindowTokens":32000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":15.313963692411882,"coding":null,"reasoning":null,"multimodal":27.036708561060927,"knowledge":null,"multilingual":null,"instruction":21.030880097311883,"math":null},"capabilityCounts":{"multimodal":6,"instruction":2,"agentic":1},"benchmarks":[{"benchmarkId":"refcocoAvg","benchmarkName":"RefCOCO average","category":"multimodal","score":87.9,"unit":"percent","family":"refcocoAvg","index":42.36304165616576,"aboveFit":false,"run":null,"source":"RefCOCO dataset authors","sourceUrl":"https://github.com/lichengunc/refer","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"countBench","benchmarkName":"CountBench","category":"multimodal","score":87.3,"unit":"percent","family":"countBench","index":34.167763960013424,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"ifeval","benchmarkName":"Instruction-Following Eval","category":"instruction","score":82.3,"unit":"percent","family":"ifeval","index":28.927756939957696,"aboveFit":false,"run":null,"source":"Jeffrey Zhou, Tianjian Lu, Swaroop Mishra, Siddhartha Brahma, Sujoy Basu, Yi Luan, Denny Zhou, Le Hou","sourceUrl":"https://arxiv.org/abs/2311.07911","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"realWorldQa","benchmarkName":"RealWorldQA","category":"multimodal","score":73.1,"unit":"percent","family":"realWorldQa","index":38.48140947237789,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmmu","benchmarkName":"Massive Multi-discipline Multimodal Understanding","category":"multimodal","score":48.4,"unit":"percent","family":"mmmu","index":14.275640678503422,"aboveFit":false,"run":null,"source":"MMMU authors","sourceUrl":"https://arxiv.org/abs/2401.05508","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"ocrBenchV2","benchmarkName":"OCRBench V2","category":"multimodal","score":47.5,"unit":"percent","family":"ocrBenchV2","index":null,"aboveFit":false,"run":null,"source":"OCRBench authors","sourceUrl":"https://arxiv.org/abs/2501.00321","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"simpleVqa","benchmarkName":"SimpleVQA","category":"multimodal","score":35.4,"unit":"percent","family":"simpleVqa","index":27.932395599305043,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"bfclV4","benchmarkName":"Berkeley Function Calling Leaderboard v4","category":"agentic","score":32.5,"unit":"percent","family":"bfclV4","index":15.313963692411882,"aboveFit":false,"run":null,"source":"Shishir G. Patil et al.","sourceUrl":"https://gorilla.cs.berkeley.edu/leaderboard.html","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmmuPro","benchmarkName":"Massive Multi-discipline Multimodal Understanding Pro","category":"multimodal","score":30.5,"unit":"percent","family":"mmmu-pro","index":5,"aboveFit":false,"run":null,"source":"MMMU-Pro authors","sourceUrl":"https://arxiv.org/abs/2409.02813","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"ifBench","benchmarkName":"Instruction Following Benchmark","category":"instruction","score":25.8,"unit":"percent","family":"ifbench-family","index":13.13400325466607,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":35,"benchmarkCount":9,"categoryCount":3,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"lfm2-5-vl-450m","providerId":"liquidai","slug":"lfm2-5-vl-450m","name":"LFM2.5-VL-450M","provider":"LiquidAI","description":"LFM2.5-VL-450M is a standard model from LiquidAI. It has 7 scored results in 4 categories. It has no index.","logoUrl":null,"releaseDate":"2026-04-08","contextWindowTokens":128000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":5,"coding":null,"reasoning":null,"multimodal":8.436343774802559,"knowledge":5,"multilingual":null,"instruction":5,"math":null},"capabilityCounts":{"multimodal":3,"instruction":1,"knowledge":2,"agentic":1},"benchmarks":[{"benchmarkId":"countBench","benchmarkName":"CountBench","category":"multimodal","score":73.31,"unit":"percent","family":"countBench","index":7.295403375171972,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"ifeval","benchmarkName":"Instruction-Following Eval","category":"instruction","score":61.16,"unit":"percent","family":"ifeval","index":5,"aboveFit":false,"run":null,"source":"Jeffrey Zhou, Tianjian Lu, Swaroop Mishra, Siddhartha Brahma, Sujoy Basu, Yi Luan, Denny Zhou, Le Hou","sourceUrl":"https://arxiv.org/abs/2311.07911","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"realWorldQa","benchmarkName":"RealWorldQA","category":"multimodal","score":58.43,"unit":"percent","family":"realWorldQa","index":13.01362794923569,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmmu","benchmarkName":"Massive Multi-discipline Multimodal Understanding","category":"multimodal","score":32.67,"unit":"percent","family":"mmmu","index":5,"aboveFit":false,"run":null,"source":"MMMU authors","sourceUrl":"https://arxiv.org/abs/2401.05508","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gpqa","benchmarkName":"Graduate-Level Google-Proof Q&A","category":"knowledge","score":25.66,"unit":"percent","family":"gpqa-diamond","index":5,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"bfclV4","benchmarkName":"Berkeley Function Calling Leaderboard v4","category":"agentic","score":21.08,"unit":"percent","family":"bfclV4","index":5,"aboveFit":false,"run":null,"source":"Shishir G. Patil et al.","sourceUrl":"https://gorilla.cs.berkeley.edu/leaderboard.html","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmluPro","benchmarkName":"Massive Multitask Language Understanding Professional","category":"knowledge","score":19.32,"unit":"percent","family":"mmlu-pro","index":5,"aboveFit":false,"run":null,"source":"Yubo Wang, Xueguang Ma, Ge Zhang, Yuansheng Ni, Abhranil Chandra, Shiguang Guo, Weiming Ren, Aaran Arulraj, Xuan He, Ziyan Jiang, Tianle Li, Max Ku, Kai Wang, Alex Zhuang, Rongqi Fan, Xiang Yue, Wenhu Chen","sourceUrl":"https://arxiv.org/abs/2406.01574","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":50,"benchmarkCount":7,"categoryCount":4,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"lfm2-5-vl-450m-extract","providerId":"liquidai","slug":"lfm2-5-vl-450m-extract","name":"LFM2.5-VL-450M-Extract","provider":"LiquidAI","description":"LFM2.5-VL-450M-Extract is a standard model from LiquidAI in the LFM2.5-VL Extract family. It has no scored result.","logoUrl":null,"releaseDate":"2026-05-26","contextWindowTokens":128000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[{"benchmarkId":"liquidExtractJsonValidity","benchmarkName":"Liquid image-to-JSON extraction JSON validity","category":"multimodal","score":98.9,"unit":"percent","family":"liquidExtractJsonValidity","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"liquidExtractSchemaF1","benchmarkName":"Liquid image-to-JSON extraction schema consistency F1","category":"multimodal","score":98.8,"unit":"percent","family":"liquidExtractSchemaF1","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"liquidExtractVlmJudge","benchmarkName":"Liquid image-to-JSON extraction VLM judge score","category":"multimodal","score":84.5,"unit":"percent","family":"liquidExtractVlmJudge","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"lfm3-1-7b-base","providerId":"liquidai","slug":"lfm3-1-7b-base","name":"LFM3-1.7B-Base","provider":"LiquidAI","description":"LFM3-1.7B-Base is a standard model from LiquidAI in the LFM3 family. It has no scored result.","logoUrl":null,"releaseDate":"2026-10-27","contextWindowTokens":256000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"lfm3-audio-2b","providerId":"liquidai","slug":"lfm3-audio-2b","name":"LFM3-Audio-2B","provider":"LiquidAI","description":"LFM3-Audio-2B is a reasoning model from LiquidAI in the LFM3 family. It has no scored result.","logoUrl":null,"releaseDate":"2026-10-27","contextWindowTokens":32000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":true,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"lfm3-omni-2b","providerId":"liquidai","slug":"lfm3-omni-2b","name":"LFM3-Omni-2B","provider":"LiquidAI","description":"LFM3-Omni-2B is a standard model from LiquidAI in the LFM3 family. It has no scored result.","logoUrl":null,"releaseDate":"2026-10-27","contextWindowTokens":32000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"lfm3-vl-2b","providerId":"liquidai","slug":"lfm3-vl-2b","name":"LFM3-VL-2B","provider":"LiquidAI","description":"LFM3-VL-2B is a reasoning model from LiquidAI. It has 11 scored results in 1 category. It has no index.","logoUrl":null,"releaseDate":"2026-10-27","contextWindowTokens":128000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":true,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":36.88209618714102,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"multimodal":11},"benchmarks":[{"benchmarkId":"countBench","benchmarkName":"CountBench","category":"multimodal","score":93,"unit":"percent","family":"countBench","index":45.11647413396599,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"refcocoAvg","benchmarkName":"RefCOCO average","category":"multimodal","score":90.5,"unit":"percent","family":"refcocoAvg","index":47.981799469113234,"aboveFit":false,"run":null,"source":"RefCOCO dataset authors","sourceUrl":"https://github.com/lichengunc/refer","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vStar","benchmarkName":"V*","category":"multimodal","score":86.4,"unit":"percent","family":"vStar","index":45.15573823437365,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"realWorldQa","benchmarkName":"RealWorldQA","category":"multimodal","score":79.3,"unit":"percent","family":"realWorldQa","index":49.24488905271066,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"dynaMath","benchmarkName":"DynaMath","category":"multimodal","score":79.2,"unit":"percent","family":"dynaMath","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mlvuAvg","benchmarkName":"MLVU mean average","category":"multimodal","score":71.3,"unit":"percent","family":"mlvuAvg","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"charxivNoTools","benchmarkName":"CharXiv Reasoning without tools","category":"multimodal","score":70.2,"unit":"percent","family":"charxivNoTools","index":46.35336119876823,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"videoMmeWithSub","benchmarkName":"Video-MME with subtitle","category":"multimodal","score":69,"unit":"percent","family":"videoMmeWithSub","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"weMath","benchmarkName":"We-Math","category":"multimodal","score":68.6,"unit":"percent","family":"weMath","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"ocrBenchV2","benchmarkName":"OCRBench V2","category":"multimodal","score":63.5,"unit":"percent","family":"ocrBenchV2","index":null,"aboveFit":false,"run":null,"source":"OCRBench authors","sourceUrl":"https://arxiv.org/abs/2501.00321","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"videoMmeNoSub","benchmarkName":"Video-MME without subtitle","category":"multimodal","score":62.8,"unit":"percent","family":"videoMmeNoSub","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmmu","benchmarkName":"Massive Multi-discipline Multimodal Understanding","category":"multimodal","score":55.7,"unit":"percent","family":"mmmu","index":22.22002309972294,"aboveFit":false,"run":null,"source":"MMMU authors","sourceUrl":"https://arxiv.org/abs/2401.05508","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"screenSpotPro","benchmarkName":"ScreenSpot Pro","category":"multimodal","score":55.5,"unit":"percent","family":"screenSpotPro","index":39.11578707368068,"aboveFit":false,"run":null,"source":"Kaixin Li, Ziyang Meng, Hongzhan Lin, Ziyang Luo, Yuchen Tian, Jing Ma, Zhiyong Huang, Tat-Seng Chua","sourceUrl":"https://arxiv.org/abs/2504.07981","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmvu","benchmarkName":"Multimodal Multi-disciplinary Video Understanding","category":"multimodal","score":52.9,"unit":"percent","family":"mmvu","index":null,"aboveFit":false,"run":null,"source":"MMVU benchmark maintainers","sourceUrl":"https://www.kimi.com/blog/kimi-k2-5.html","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mathVision","benchmarkName":"MathVision","category":"multimodal","score":51.6,"unit":"percent","family":"mathVision","index":24.306970093689195,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"videoMmmu","benchmarkName":"VideoMMMU","category":"multimodal","score":49.6,"unit":"percent","family":"videoMmmu","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"erqa","benchmarkName":"ERQA","category":"multimodal","score":48.2,"unit":"percent","family":"erqa","index":42.80436745489113,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmmuPro","benchmarkName":"Massive Multi-discipline Multimodal Understanding Pro","category":"multimodal","score":45.7,"unit":"percent","family":"mmmu-pro","index":14.627355543781533,"aboveFit":false,"run":null,"source":"MMMU-Pro authors","sourceUrl":"https://arxiv.org/abs/2409.02813","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lvBench","benchmarkName":"LVBench","category":"multimodal","score":41.2,"unit":"percent","family":"lvBench","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"simpleVqa","benchmarkName":"SimpleVQA","category":"multimodal","score":36.3,"unit":"percent","family":"simpleVqa","index":28.776292703854025,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":10,"benchmarkCount":11,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"lightning-v3-1-pro","providerId":"smallest-ai","slug":"lightning-v3-1-pro","name":"Lightning v3.1 Pro","provider":"Smallest AI","description":"Lightning v3.1 Pro is a standard model from Smallest AI in the Smallest AI Lightning family. It has no scored result.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":false,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"ling30flashafrc3","providerId":"unknown","slug":"unknown-ling-3-0-flash-af-rc3","name":"Ling 3.0 Flash Af Rc3","provider":"Unknown","description":"Ling 3.0 Flash Af Rc3 is a model. It has 3 scored results in 2 categories. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":50.493257384160344,"coding":44.81720769225703,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"agentic":1,"coding":2},"benchmarks":[{"benchmarkId":"vals-ai:medscribe:overall","benchmarkName":"MedScribe","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":75.593,"unit":"percent","family":"vals-ai:medscribe:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medscribe","measuredAt":"2026-10-29","stderr":2.026,"costPerTest":0.001618,"latency":44.811},{"benchmarkId":"vals-ai:legal_bench:overall","benchmarkName":"LegalBench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":68.743,"unit":"percent","family":"vals-ai:legal_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","measuredAt":"2026-10-13","stderr":0.525,"costPerTest":0.000201,"latency":6.35},{"benchmarkId":"vals-ai:cyber:overall","benchmarkName":"CyberBench v1.1","category":null,"domain":"Other","eligible":false,"ranked":true,"score":58.334,"unit":"percent","family":"vals-ai:cyber:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/cyber","measuredAt":"2026-11-05","stderr":6.076,"costPerTest":0.104472,"latency":1163.067},{"benchmarkId":"vals-ai:fabv2:overall","benchmarkName":"Finance Agent (v2)","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":54.927,"unit":"percent","family":"vals-ai:fabv2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/fabv2","measuredAt":"2026-10-29","stderr":0.679,"costPerTest":0.044792,"latency":375.066},{"benchmarkId":"vals-ai:terminal-bench-2-1:overall","benchmarkName":"Terminal-Bench 2.1","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":50.187,"unit":"percent","family":"terminal-bench-2-1","index":50.493257384160344,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2-1","measuredAt":"2026-10-01","stderr":1.35,"costPerTest":0.043393,"latency":833.528},{"benchmarkId":"vals-ai:emb:overall","benchmarkName":"EMB","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":36.832,"unit":"percent","family":"vals-ai:emb:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/emb","measuredAt":"2026-10-20","stderr":3.325,"costPerTest":0.179543,"latency":2207.467},{"benchmarkId":"vals-ai:medcode:overall","benchmarkName":"MedCode","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":29.305,"unit":"percent","family":"vals-ai:medcode:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medcode","measuredAt":"2026-10-29","stderr":1.943,"costPerTest":0.001354,"latency":31.167},{"benchmarkId":"vals-ai:tax_agent_bench:overall","benchmarkName":"Tax Agent Bench","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":20.599,"unit":"percent","family":"vals-ai:tax_agent_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench","measuredAt":"2026-11-09","stderr":2.908,"costPerTest":0.025079,"latency":333.833},{"benchmarkId":"vals-ai:legal_research:overall","benchmarkName":"Legal Research Bench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":15.385,"unit":"percent","family":"vals-ai:legal_research:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_research","measuredAt":"2026-11-09","stderr":2.508,"costPerTest":0.028227,"latency":515.464},{"benchmarkId":"vals-ai:vibe-code:overall","benchmarkName":"Vibe Code Bench v1.1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":3.4,"unit":"percent","family":"vals-ai:vibe-code:overall","index":43.4762960089678,"aboveFit":false,"run":"OpenHands","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","measuredAt":"2026-11-09","stderr":1.291,"costPerTest":0.233022,"latency":1451.032},{"benchmarkId":"vals-ai:code-migration:overall","benchmarkName":"Code Migration","category":"coding","domain":null,"eligible":true,"ranked":true,"score":1.824,"unit":"percent","family":"vals-ai:code-migration:overall","index":46.158119375546256,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/code-migration","measuredAt":"2026-10-29","stderr":0.866,"costPerTest":0.103478,"latency":1770.047}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":40,"benchmarkCount":3,"categoryCount":2,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"ling-3-0-flash-fin","providerId":"inclusionai","slug":"ling-3-0-flash-fin","name":"Ling 3.0 Flash Fin","provider":"InclusionAI","description":"Ling 3.0 Flash Fin is a reasoning model from InclusionAI in the Ling 3.0 family. It has 3 scored results in 1 category. It has no index.","logoUrl":null,"releaseDate":"2026-08-28","contextWindowTokens":262144,"modalities":["text"],"openWeights":false,"details":{"apiId":"inclusionai/ling-3.0-flash-fin","maxOutputTokens":32768,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.008400000000000001,"weightsUrl":null},"reasoning":true,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":60.48779919826268,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"agentic":3},"benchmarks":[{"benchmarkId":"aaTau3Banking","benchmarkName":"Artificial Analysis Tau3-Banking","category":"agentic","score":38.6,"unit":"percent","family":"aaTau3Banking","index":62.273509595385654,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/tau3-banking","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"apexAgents","benchmarkName":"APEX-Agents","category":"agentic","score":29.17,"unit":"percent","family":"apexAgents","index":58.28451622925047,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"apexAgentsAa","benchmarkName":"APEX-Agents-AA","category":"agentic","score":27.4,"unit":"percent","family":"apexAgentsAa","index":60.9053717701519,"aboveFit":false,"run":null,"source":"Artificial Analysis / Mercor","sourceUrl":"https://artificialanalysis.ai/evaluations/apex-agents-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"spreadsheetBench2","benchmarkName":"SpreadsheetBench 2","category":"agentic","score":21.81,"unit":"percent","family":"spreadsheetBench2","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":0.041999999999999996,"outputPerMillion":0.12319999999999999,"batch":null,"freeTier":true,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":122.5,"timeToFirstToken":0.835,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":20,"benchmarkCount":3,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"ling-3-0-flash-fp8","providerId":"inclusionai","slug":"ling-3-0-flash-fp8","name":"Ling 3.0 Flash FP8","provider":"InclusionAI","description":"Ling 3.0 Flash FP8 is a reasoning model from InclusionAI in the Ling 3.0 family. It has 3 scored results in 3 categories. It has no index.","logoUrl":null,"releaseDate":"2026-08-04","contextWindowTokens":262000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":true,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":48.46981002327404,"reasoning":null,"multimodal":null,"knowledge":54.25160543111147,"multilingual":null,"instruction":57.61670852055043,"math":null},"capabilityCounts":{"knowledge":1,"instruction":1,"coding":1},"benchmarks":[{"benchmarkId":"gpqa","benchmarkName":"Graduate-Level Google-Proof Q&A","category":"knowledge","score":84,"unit":"percent","family":"gpqa-diamond","index":54.25160543111147,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gpqaDiamond","benchmarkName":"GPQA Diamond","category":"knowledge","score":84,"unit":"percent","family":"gpqa-diamond","index":54.25160543111147,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"ifBench","benchmarkName":"Instruction Following Benchmark","category":"instruction","score":73.4,"unit":"percent","family":"ifbench-family","index":57.61670852055043,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"sciCode","benchmarkName":"Scientific Code Benchmark","category":"coding","score":40.37,"unit":"percent","family":"scicode-family","index":48.46981002327404,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":40,"benchmarkCount":3,"categoryCount":3,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"ling-3-0-flash-sante","providerId":"inclusionai","slug":"ling-3-0-flash-sante","name":"Ling 3.0 Flash Sante","provider":"InclusionAI","description":"Ling 3.0 Flash Sante is a reasoning model from InclusionAI in the Ling 3.0 family. It has 6 scored results in 3 categories. It has no index.","logoUrl":null,"releaseDate":"2026-10-19","contextWindowTokens":262000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":true,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":64.45739930520828,"coding":null,"reasoning":55.66672651295975,"multimodal":null,"knowledge":66.61037012880703,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"agentic":4,"reasoning":1,"knowledge":1},"benchmarks":[{"benchmarkId":"deepSearchQa","benchmarkName":"DeepSearchQA","category":"agentic","score":90.8,"unit":"percent","family":"deepSearchQa","index":66.49872690335096,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"browseComp","benchmarkName":"BrowseComp","category":"agentic","score":80.2,"unit":"percent","family":"browseComp","index":63.26858249174306,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mcpAtlas","benchmarkName":"MCP Atlas","category":"agentic","score":75.6,"unit":"percent","family":"mcpAtlas","index":61.970990374635704,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":70.5,"unit":"percent","family":"lcr","index":55.66672651295975,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hleWithTools","benchmarkName":"Humanity's Last Exam with tools","category":"agentic","score":57.1,"unit":"percent","family":"hleWithTools","index":66.0912974511034,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"healthBenchProfessional","benchmarkName":"HealthBench Professional","category":"knowledge","score":54.3,"unit":"percent","family":"healthBenchProfessional","index":66.61037012880703,"aboveFit":false,"run":null,"source":"Rebecca Soskin Hicks, Mikhail Trofimov, Dominick Lim, Rahul K. Arora, Foivos Tsimpourlas, Preston Bowman, Michael Sharman, Chi Tong, Kavin Karthik, Arnav Dugar, Akshay Jagadeesh, Khaled Saab, Johannes Heidecke, Ashley Alexander, Nate Gross, Karan Singhal","sourceUrl":"https://arxiv.org/abs/2604.27470","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"medXpertQaText","benchmarkName":"MedXpertQA Text","category":"knowledge","score":51,"unit":"percent","family":"medXpertQaText","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":0,"outputPerMillion":0,"batch":null,"freeTier":true,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":50,"benchmarkCount":6,"categoryCount":3,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"ling-3-0-flash-vl","providerId":"inclusionai","slug":"ling-3-0-flash-vl","name":"Ling 3.0 Flash VL","provider":"InclusionAI","description":"Ling 3.0 Flash VL is a reasoning model from InclusionAI in the Ling 3.0 family. It has 7 scored results in 5 categories. It has no index.","logoUrl":null,"releaseDate":"2026-09-04","contextWindowTokens":131072,"modalities":["text","image","video"],"openWeights":true,"details":{"apiId":"inclusionai/ling-3.0-flash-vl","maxOutputTokens":32768,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.012,"weightsUrl":"https://huggingface.co/inclusionAI/Ling-3.0-flash-VL"},"reasoning":true,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":57.628008053023244,"coding":53.46294711404248,"reasoning":50.86951862934929,"multimodal":57.96438397224972,"knowledge":51.364235009951976,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"knowledge":2,"multimodal":1,"reasoning":2,"coding":1,"agentic":1},"benchmarks":[{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":86.2,"unit":"percent","family":"gpqa-diamond","index":56.269385315935004,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaMmmuPro","benchmarkName":"Artificial Analysis MMMU-Pro","category":"multimodal","score":79,"unit":"percent","family":"mmmu-pro","index":57.96438397224972,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmmu-pro","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":78.3,"unit":"percent","family":"lcr","index":61.018154612527894,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaSciCode","benchmarkName":"Artificial Analysis SciCode","category":"coding","score":44.2,"unit":"percent","family":"scicode-family","index":53.46294711404248,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/scicode","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gdpvalAaNormalized","benchmarkName":"GDPval-AA normalized","category":"agentic","score":33,"unit":"percent","family":"gdpvalAaNormalized","index":57.628008053023244,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":22,"unit":"percent","family":"hle","index":46.459084703968955,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":2,"unit":"percent","family":"critpt","index":40.72088264617068,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":0.06,"outputPerMillion":0.18,"batch":null,"freeTier":true,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":105,"timeToFirstToken":0.673,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":80,"benchmarkCount":7,"categoryCount":5,"confidence":"unranked","state":"partial"},"hosts":[{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":0.06,"outputPerMillion":0.18,"cacheReadPerMillion":0.012,"contextTokens":131072,"maxOutputTokens":32768,"deprecated":false},{"host":"novita-ai","name":"Novita AI","kind":"host","url":"https://novita.ai/docs/guides/introduction","inputPerMillion":0.075,"outputPerMillion":0.22,"cacheReadPerMillion":0.015,"contextTokens":262144,"maxOutputTokens":32768,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"ling-3-0-tiny","providerId":"inclusionai","slug":"ling-3-0-tiny","name":"Ling 3.0 Tiny","provider":"InclusionAI","description":"Ling 3.0 Tiny is a reasoning model from InclusionAI in the Ling 3.0 family. It has 7 scored results in 4 categories. It has no index.","logoUrl":null,"releaseDate":"2026-08-10","contextWindowTokens":262144,"modalities":["text"],"openWeights":true,"details":{"apiId":"inclusionai/ling-3.0-tiny","maxOutputTokens":250000,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.003,"weightsUrl":"https://huggingface.co/inclusionAI/Ling-3.0-tiny"},"reasoning":true,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":36.17870520226828,"coding":27.389124185747818,"reasoning":42.75891058237946,"multimodal":null,"knowledge":45.566831393844446,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"knowledge":3,"reasoning":2,"coding":1,"agentic":1},"benchmarks":[{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":73.4,"unit":"percent","family":"gpqa-diamond","index":44.529575076961756,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":60.3,"unit":"percent","family":"lcr","index":48.66870515198601,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"omniscienceAccuracy","benchmarkName":"Artificial Analysis Omniscience Accuracy","category":"knowledge","score":37.9,"unit":"percent","family":"omniscienceAccuracy","index":58.45365226248101,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaSciCode","benchmarkName":"Artificial Analysis SciCode","category":"coding","score":24.2,"unit":"percent","family":"scicode-family","index":27.389124185747818,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/scicode","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":9.3,"unit":"percent","family":"hle","index":33.71726684209057,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gdpvalAaNormalized","benchmarkName":"GDPval-AA normalized","category":"agentic","score":3,"unit":"percent","family":"gdpvalAaNormalized","index":36.17870520226828,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":0,"unit":"percent","family":"critpt","index":36.8491160127729,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":0.015,"outputPerMillion":0.045,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":124,"timeToFirstToken":0.406,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":70,"benchmarkCount":7,"categoryCount":4,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"lingmini20","providerId":"inclusionai","slug":"inclusionai-ling-mini-2-0","name":"Ling Mini 2.0","provider":"InclusionAI","description":"Ling Mini 2.0 is a model from InclusionAI. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":5,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-italian:overall","benchmarkName":"EuroEval Italian","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":12.858749999999999,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/italian","measuredAt":null,"stderr":0.638946887464052,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-portuguese:overall","benchmarkName":"EuroEval Portuguese","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":11.99375,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/portuguese","measuredAt":null,"stderr":0.764847819830847,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-spanish:overall","benchmarkName":"EuroEval Spanish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":11.305,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/spanish","measuredAt":null,"stderr":0.6224585829595412,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-dutch:overall","benchmarkName":"EuroEval Dutch","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":9.942222222222222,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/dutch","measuredAt":null,"stderr":0.6250728352621926,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":8.89625,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":0.7608599164760883,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":8.09375,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":1.1859364706846653,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-french:overall","benchmarkName":"EuroEval French","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":7.6225,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/french","measuredAt":null,"stderr":0.8856953482998541,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":0.7014285714285714,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.7611604995810338,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"lingminibase20","providerId":"inclusionai","slug":"inclusionai-ling-mini-base-2-0","name":"Ling Mini Base 2.0","provider":"InclusionAI","description":"Ling Mini Base 2.0 is a model from InclusionAI. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":5,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-french:overall","benchmarkName":"EuroEval French","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":5.838750000000001,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/french","measuredAt":null,"stderr":0.8491355898794962,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-portuguese:overall","benchmarkName":"EuroEval Portuguese","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":4.9487499999999995,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/portuguese","measuredAt":null,"stderr":0.8646241958215141,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":3.4212499999999997,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.7607110160895529,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"litjev","providerId":"zhengxu-yu","slug":"litjev","name":"LitJev Qwen3.8-27B","provider":"Zhengxu Yu","description":"LitJev Qwen3.8-27B is a standard model from Zhengxu Yu. It has 2 scored results in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":61.13889046361925,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"reasoning":2},"benchmarks":[{"benchmarkId":"jevBench14","benchmarkName":"jevBench14","category":"reasoning","score":19.51031342209139,"unit":"percent","family":"jevBench14","index":60.92917478673755,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"jevBench15","benchmarkName":"jevBench15","category":"reasoning","score":16.306394195686046,"unit":"percent","family":"jevBench15","index":61.34860614050095,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":15,"benchmarkCount":2,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"lizzy7b","providerId":"unknown","slug":"unknown-lizzy-7b","name":"Lizzy 7B","provider":"Unknown","description":"Lizzy 7B is a model. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":5.136509812996749,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-dutch:overall","benchmarkName":"EuroEval Dutch","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":28.817777777777774,"unit":"percent","family":"euroeval","index":11.319329121195665,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/dutch","measuredAt":null,"stderr":0.559451892259814,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":19.703749999999996,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.45625342464468144,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"llada2-2-flash","providerId":"inclusionai","slug":"llada2-2-flash","name":"LLaDA2.2-flash","provider":"InclusionAI","description":"LLaDA2.2-flash is a reasoning model from InclusionAI in the LLaDA2.2 family. It has 7 scored results in 2 categories. It has no index.","logoUrl":null,"releaseDate":"2026-07-16","contextWindowTokens":128000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":true,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":47.80496291769249,"coding":36.0238045351619,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"agentic":5,"coding":2},"benchmarks":[{"benchmarkId":"pinchBench","benchmarkName":"PinchBench","category":"agentic","score":81.66,"unit":"percent","family":"pinchBench","index":41.17515043032531,"aboveFit":false,"run":null,"source":"Kilo Code","sourceUrl":"https://pinchbench.com/about","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"tau2Bench","benchmarkName":"τ²-Bench Tool-Agent-User Evaluation","category":"agentic","score":80.33,"unit":"percent","family":"tau2-bench","index":56.09426822600147,"aboveFit":false,"run":null,"source":"Victor Barres, Honghua Dong, Soham Ray, Xujie Si, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2506.07982","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"clawEval","benchmarkName":"Claw-Eval","category":"agentic","score":64.22,"unit":"percent","family":"clawEval","index":57.98500641609814,"aboveFit":false,"run":null,"source":"Bowen Ye, Rang Li, Qibin Yang, Yuanxin Liu, Linli Yao, Hanglong Lv, Zhihui Xie, Chenxin An, Lei Li, Lingpeng Kong, Qi Liu, Zhifang Sui, Tong Yang","sourceUrl":"https://arxiv.org/abs/2604.06132","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"bfclV4","benchmarkName":"Berkeley Function Calling Leaderboard v4","category":"agentic","score":60.78,"unit":"percent","family":"bfclV4","index":41.5904957099515,"aboveFit":false,"run":null,"source":"Shishir G. Patil et al.","sourceUrl":"https://gorilla.cs.berkeley.edu/leaderboard.html","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"sweVerified","benchmarkName":"Software Engineering Benchmark Verified","category":"coding","score":49.28,"unit":"percent","family":"swe-bench-verified","index":37.57476496265198,"aboveFit":false,"run":null,"source":"Carlos E. Jimenez, John Yang, Alexander Wettig, Shunyu Yao, Kexin Pei, Ofir Press, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2310.06770","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mcpAtlas","benchmarkName":"MCP Atlas","category":"agentic","score":46.21,"unit":"percent","family":"mcpAtlas","index":42.17989380608603,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"swePro","benchmarkName":"SWE-bench Pro","category":"coding","score":30.1,"unit":"percent","family":"swePro","index":34.47284410767182,"aboveFit":false,"run":null,"source":"Xiang Deng, Jeff Da, Edwin Pan, Yannis Yiming He, Charles Ide, Kanak Garg, Niklas Lauffer, Andrew Park, Nitin Pasari, Chetan Rane, Karmini Sampath, Maya Krishnan, Srivatsa Kundurthy, Sean Hendryx, Zifan Wang, Chen Bo Calvin Zhang, Noah Jacobson, Bing Liu, Brad Kenstler","sourceUrl":"https://arxiv.org/abs/2509.16941","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":40,"benchmarkCount":7,"categoryCount":2,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"llama213b","providerId":"meta","slug":"meta-llama-2-13b","name":"Llama 2 13B","provider":"Meta","description":"Llama 2 13B is a model from Meta. It has no scored result.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[{"benchmarkId":"vals-ai:legal_bench:overall","benchmarkName":"LegalBench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":53.696,"unit":"percent","family":"vals-ai:legal_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","measuredAt":"2026-10-13","stderr":0.733,"costPerTest":null,"latency":1.673}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"llama270bhf","providerId":"meta","slug":"meta-llama-2-70b-hf","name":"Llama 2 70B Hf","provider":"Meta","description":"Llama 2 70B Hf is a model from Meta. It has no scored result.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[{"benchmarkId":"vals-ai:legal_bench:overall","benchmarkName":"LegalBench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":62.434,"unit":"percent","family":"vals-ai:legal_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","measuredAt":"2026-10-13","stderr":0.711,"costPerTest":null,"latency":14.204}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"llama27b","providerId":"meta","slug":"meta-llama-2-7b","name":"Llama 2 7B","provider":"Meta","description":"Llama 2 7B is a model from Meta. It has no scored result.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[{"benchmarkId":"vals-ai:legal_bench:overall","benchmarkName":"LegalBench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":51.869,"unit":"percent","family":"vals-ai:legal_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","measuredAt":"2026-10-13","stderr":0.63,"costPerTest":null,"latency":0.984}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"llama270b","providerId":"meta","slug":"meta-llama-2-70b","name":"Llama 2-70B","provider":"Meta","description":"Llama 2-70B is a model from Meta. It has 3 scored results in 2 categories. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":5,"multilingual":null,"instruction":null,"math":6.959545438282369},"capabilityCounts":{"knowledge":1,"math":2},"benchmarks":[{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":26.325757575757574,"unit":"percent","family":"gpqa-diamond","index":5,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.9901808558284102,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-math-level-5:overall","benchmarkName":"MATH level 5","category":"math","domain":null,"eligible":true,"ranked":true,"score":3.2854984894259815,"unit":"percent","family":"epoch-ai:epoch-math-level-5:overall","index":5,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0.25422297741566097,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":0,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":8.919090876564738,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":25,"benchmarkCount":3,"categoryCount":2,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"llama-3-70b","providerId":"meta","slug":"llama-3-70b","name":"Llama 3 70B","provider":"Meta","description":"Llama 3 70B is a standard model from Meta. It has 3 scored results in 2 categories. It has no index.","logoUrl":null,"releaseDate":"2024-04-18","contextWindowTokens":128000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":14.411338341129948,"multilingual":null,"instruction":null,"math":10.68902185179224},"capabilityCounts":{"knowledge":1,"math":2},"benchmarks":[{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":40.561868686868685,"unit":"percent","family":"gpqa-diamond","index":14.411338341129948,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.6224849492112066,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-math-level-5:overall","benchmarkName":"MATH level 5","category":"math","domain":null,"eligible":true,"ranked":true,"score":22.5547583081571,"unit":"percent","family":"epoch-ai:epoch-math-level-5:overall","index":9.954006241224512,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0.8257639939780866,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":4.305555555555555,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":11.424037462359976,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.192429710598427,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":25,"benchmarkCount":3,"categoryCount":2,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"l3lunaris8b","providerId":"sao10k","slug":"sao10k-llama-3-8b-lunaris","name":"Llama 3 8B Lunaris","provider":"Sao10K","description":"Llama 3 8B Lunaris is a model from Sao10K. It has no scored result.","logoUrl":null,"releaseDate":"2024-08-13T00:00:00.000Z","contextWindowTokens":8192,"modalities":["text"],"openWeights":true,"details":{"apiId":"sao10k/l3-lunaris-8b","maxOutputTokens":7372,"knowledgeCutoff":"2023-12-31","retiresAt":null,"tools":false,"structuredOutput":true,"cacheReadPerMillion":null,"weightsUrl":"https://huggingface.co/Sao10K/L3-8B-Lunaris-v1"},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":0.04,"outputPerMillion":0.049999999999999996,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":95,"timeToFirstToken":0.156,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":0.04,"outputPerMillion":0.05,"cacheReadPerMillion":null,"contextTokens":8192,"maxOutputTokens":7372,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"llama38b","providerId":"meta","slug":"meta-llama-3-8b","name":"Llama 3-8B","provider":"Meta","description":"Llama 3-8B is a model from Meta. It has 3 scored results in 2 categories. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":5,"multilingual":null,"instruction":null,"math":7.5251785383006435},"capabilityCounts":{"knowledge":1,"math":2},"benchmarks":[{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":26.073232323232325,"unit":"percent","family":"gpqa-diamond","index":5,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.7030433617735758,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-math-level-5:overall","benchmarkName":"MATH level 5","category":"math","domain":null,"eligible":true,"ranked":true,"score":6.12726586102719,"unit":"percent","family":"epoch-ai:epoch-math-level-5:overall","index":5,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0.3743316486385653,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":1.9444444444444444,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":10.050357076601287,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.0477515876628662,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":25,"benchmarkCount":3,"categoryCount":2,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"llama3170binstruct","providerId":"meta","slug":"meta-llama-3-1-70b-instruct","name":"Llama 3.1 70B Instruct","provider":"Meta","description":"Llama 3.1 70B Instruct is a model from Meta. It has no scored result.","logoUrl":null,"releaseDate":"2024-07-23T00:00:00.000Z","contextWindowTokens":131072,"modalities":["text"],"openWeights":true,"details":{"apiId":"meta-llama/llama-3.1-70b-instruct","maxOutputTokens":8192,"knowledgeCutoff":"2023-12-31","retiresAt":null,"tools":false,"structuredOutput":false,"cacheReadPerMillion":null,"weightsUrl":"https://huggingface.co/meta-llama/Meta-Llama-3.1-70B-Instruct"},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":0.72,"outputPerMillion":0.72,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":70,"timeToFirstToken":0.385,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":{"elo":1260.9148355676243,"eloUpper":1264.6289339285577,"eloLower":1257.200737206691,"votes":55240,"rank":295,"rankUpper":null,"rankLower":null,"arenaName":"Llama 3.1 70B Instruct","organization":"meta","updatedAt":"2026-11-09"},"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[{"host":"amazon-bedrock","name":"Amazon Bedrock","kind":"cloud","url":"https://docs.aws.amazon.com/bedrock/latest/userguide/models-supported.html","inputPerMillion":0.72,"outputPerMillion":0.72,"cacheReadPerMillion":null,"contextTokens":128000,"maxOutputTokens":4096,"deprecated":true},{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":0.72,"outputPerMillion":0.72,"cacheReadPerMillion":null,"contextTokens":131072,"maxOutputTokens":8192,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"llama3170blatamgptsft10","providerId":"meta","slug":"meta-llama-3-1-70b-latamgpt-sft-1-0","name":"Llama 3.1 70B LatamGPT SFT-1.0","provider":"Meta","description":"Llama 3.1 70B LatamGPT SFT-1.0 is a model from Meta. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":38.121941744665754,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-french:overall","benchmarkName":"EuroEval French","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":52.207499999999996,"unit":"percent","family":"euroeval","index":43.05380035499422,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/french","measuredAt":null,"stderr":0.6203174590481877,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-portuguese:overall","benchmarkName":"EuroEval Portuguese","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":48.5725,"unit":"percent","family":"euroeval","index":38.121941744665754,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/portuguese","measuredAt":null,"stderr":0.5737826788079264,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":40.736250000000005,"unit":"percent","family":"euroeval","index":27.489953564421903,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.816137051909788,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"llama318binstruct","providerId":"meta","slug":"meta-llama-3-1-8b-instruct","name":"Llama 3.1 8B Instruct","provider":"Meta","description":"Llama 3.1 8B Instruct is a model from Meta. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":"2024-07-23T00:00:00.000Z","contextWindowTokens":131072,"modalities":["text"],"openWeights":true,"details":{"apiId":"meta-llama/llama-3.1-8b-instruct","maxOutputTokens":16384,"knowledgeCutoff":"2023-12-31","retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":null,"weightsUrl":"https://huggingface.co/meta-llama/Meta-Llama-3.1-8B-Instruct"},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":28.725462160578815,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-dutch:overall","benchmarkName":"EuroEval Dutch","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":46.83888888888889,"unit":"percent","family":"euroeval","index":35.769830359440334,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/dutch","measuredAt":null,"stderr":0.3476073419787489,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-french:overall","benchmarkName":"EuroEval French","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":46.66375,"unit":"percent","family":"euroeval","index":35.53220718552836,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/french","measuredAt":null,"stderr":0.4204740628148186,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":42.965,"unit":"percent","family":"euroeval","index":30.513854493656993,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":0.27674446697269306,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-portuguese:overall","benchmarkName":"EuroEval Portuguese","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":42.40375,"unit":"percent","family":"euroeval","index":29.752367383602838,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/portuguese","measuredAt":null,"stderr":0.3982069185737485,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-italian:overall","benchmarkName":"EuroEval Italian","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":40.89000000000001,"unit":"percent","family":"euroeval","index":27.698556937554784,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/italian","measuredAt":null,"stderr":0.299345640522791,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":40.79714285714285,"unit":"percent","family":"euroeval","index":27.572571160517487,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.2967579925165436,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-spanish:overall","benchmarkName":"EuroEval Spanish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":38.58375000000001,"unit":"percent","family":"euroeval","index":24.569506340561652,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/spanish","measuredAt":null,"stderr":0.40462366774572145,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":37.006249999999994,"unit":"percent","family":"euroeval","index":22.429201812970668,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.4740994753213718,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":0.02,"outputPerMillion":0.04,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":28,"timeToFirstToken":0.955,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":{"elo":1186.5849689672357,"eloUpper":1190.686046438871,"eloLower":1182.4838914956003,"votes":49605,"rank":332,"rankUpper":null,"rankLower":null,"arenaName":"Llama 3.1 8B Instruct","organization":"meta","updatedAt":"2026-11-09"},"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":0.02,"outputPerMillion":0.04,"cacheReadPerMillion":null,"contextTokens":131072,"maxOutputTokens":16384,"deprecated":false},{"host":"novita-ai","name":"Novita AI","kind":"host","url":"https://novita.ai/docs/guides/introduction","inputPerMillion":0.02,"outputPerMillion":0.05,"cacheReadPerMillion":null,"contextTokens":16384,"maxOutputTokens":16384,"deprecated":false},{"host":"groq","name":"Groq","kind":"host","url":"https://console.groq.com/docs/models","inputPerMillion":0.05,"outputPerMillion":0.08,"cacheReadPerMillion":null,"contextTokens":131072,"maxOutputTokens":131072,"deprecated":true},{"host":"amazon-bedrock","name":"Amazon Bedrock","kind":"cloud","url":"https://docs.aws.amazon.com/bedrock/latest/userguide/models-supported.html","inputPerMillion":0.22,"outputPerMillion":0.22,"cacheReadPerMillion":null,"contextTokens":128000,"maxOutputTokens":4096,"deprecated":true},{"host":"coreweave","name":"CoreWeave","kind":"host","url":"https://docs.wandb.ai/inference","inputPerMillion":0.22,"outputPerMillion":0.22,"cacheReadPerMillion":0.22,"contextTokens":131072,"maxOutputTokens":131072,"deprecated":false},{"host":"cloudflare","name":"Cloudflare Workers AI","kind":"cloud","url":"https://developers.cloudflare.com/workers-ai/models/","inputPerMillion":0.152,"outputPerMillion":0.287,"cacheReadPerMillion":null,"contextTokens":32000,"maxOutputTokens":32000,"deprecated":true}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"l31euryale70b","providerId":"sao10k","slug":"sao10k-llama-3-1-euryale-70b-v2-2","name":"Llama 3.1 Euryale 70B v2.2","provider":"Sao10K","description":"Llama 3.1 Euryale 70B v2.2 is a model from Sao10K. It has no scored result.","logoUrl":null,"releaseDate":"2024-08-28T00:00:00.000Z","contextWindowTokens":131072,"modalities":["text"],"openWeights":true,"details":{"apiId":"sao10k/l3.1-euryale-70b","maxOutputTokens":16384,"knowledgeCutoff":"2023-12-31","retiresAt":null,"tools":false,"structuredOutput":true,"cacheReadPerMillion":null,"weightsUrl":"https://huggingface.co/Sao10K/L3.1-70B-Euryale-v2.2"},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":0.85,"outputPerMillion":0.85,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":38,"timeToFirstToken":0.321,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":0.85,"outputPerMillion":0.85,"cacheReadPerMillion":null,"contextTokens":131072,"maxOutputTokens":16384,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"llama31tulu38bsft","providerId":"meta","slug":"meta-llama-3-1-tulu-3-8b-sft","name":"Llama 3.1 Tulu 3 8B SFT","provider":"Meta","description":"Llama 3.1 Tulu 3 8B SFT is a model from Meta. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":28.622008455203964,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-dutch:overall","benchmarkName":"EuroEval Dutch","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":46.541111111111114,"unit":"percent","family":"euroeval","index":35.36581443171054,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/dutch","measuredAt":null,"stderr":0.4015863604983013,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-french:overall","benchmarkName":"EuroEval French","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":45.72375,"unit":"percent","family":"euroeval","index":34.25684347336639,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/french","measuredAt":null,"stderr":0.45326075552599965,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-portuguese:overall","benchmarkName":"EuroEval Portuguese","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":43.58625,"unit":"percent","family":"euroeval","index":31.35674779810446,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/portuguese","measuredAt":null,"stderr":0.4415349646404008,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-italian:overall","benchmarkName":"EuroEval Italian","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":41.79,"unit":"percent","family":"euroeval","index":28.91964985345453,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/italian","measuredAt":null,"stderr":0.3698711768981195,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":41.35125,"unit":"percent","family":"euroeval","index":28.3243670569534,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":0.3204074554376037,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":38.03,"unit":"percent","family":"euroeval","index":23.81819500480665,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.5898861118894053,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":36.26285714285714,"unit":"percent","family":"euroeval","index":21.420588755651096,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.37450647796909514,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-spanish:overall","benchmarkName":"EuroEval Spanish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":34.23625,"unit":"percent","family":"euroeval","index":18.670949171812527,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/spanish","measuredAt":null,"stderr":0.4599813175553981,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"llama3211bvisioninstructturbo","providerId":"meta","slug":"meta-llama-3-2-11b-vision-instruct-turbo","name":"Llama 3.2 11B Vision Instruct Turbo","provider":"Meta","description":"Llama 3.2 11B Vision Instruct Turbo is a model from Meta. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":5.674940572026792,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"multimodal":1},"benchmarks":[{"benchmarkId":"vals-ai:mmmu:overall","benchmarkName":"MMMU Pro","category":"multimodal","domain":null,"eligible":true,"ranked":true,"score":38.821,"unit":"percent","family":"mmmu-pro","index":5.674940572026792,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmmu","measuredAt":"2026-09-01","stderr":1.171,"costPerTest":0.000171,"latency":38.07},{"benchmarkId":"vals-ai:mortgage_tax:overall","benchmarkName":"MortgageTax","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":23.291,"unit":"percent","family":"vals-ai:mortgage_tax:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mortgage_tax","measuredAt":"2026-09-01","stderr":1.649,"costPerTest":0.000092,"latency":2.452}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":10,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"llama321b","providerId":"meta","slug":"meta-llama-3-2-1b","name":"Llama 3.2 1B","provider":"Meta","description":"Llama 3.2 1B is a model from Meta. It has 3 scored results in 3 categories. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":5,"multilingual":5,"instruction":null,"math":9.242309790860894},"capabilityCounts":{"multilingual":1,"knowledge":1,"math":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-dutch:overall","benchmarkName":"EuroEval Dutch","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":25.298888888888882,"unit":"percent","family":"euroeval","index":6.545006572239451,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/dutch","measuredAt":null,"stderr":0.4821249261803879,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-french:overall","benchmarkName":"EuroEval French","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":24.195000000000004,"unit":"percent","family":"euroeval","index":5.047283347614275,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/french","measuredAt":null,"stderr":0.37990747228766114,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":23.92676767676768,"unit":"percent","family":"gpqa-diamond","index":5,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.6278803912773236,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":21.786250000000003,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":0.37466234798815856,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":19.864285714285717,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.524398586526528,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-portuguese:overall","benchmarkName":"EuroEval Portuguese","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":19.62625,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/portuguese","measuredAt":null,"stderr":0.4722734245116911,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-spanish:overall","benchmarkName":"EuroEval Spanish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":15.086249999999998,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/spanish","measuredAt":null,"stderr":0.5554854296018933,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-italian:overall","benchmarkName":"EuroEval Italian","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":14.71875,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/italian","measuredAt":null,"stderr":0.5466731770445665,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":13.34875,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.7634144107232979,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":0.5555555555555556,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":9.242309790860894,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0.3883473863441843,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":30,"benchmarkCount":3,"categoryCount":3,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"llama321binstruct","providerId":"meta","slug":"meta-llama-3-2-1b-instruct","name":"Llama 3.2 1B Instruct","provider":"Meta","description":"Llama 3.2 1B Instruct is a model from Meta. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":"2024-09-25T00:00:00.000Z","contextWindowTokens":60000,"modalities":["text"],"openWeights":true,"details":{"apiId":"meta-llama/llama-3.2-1b-instruct","maxOutputTokens":54000,"knowledgeCutoff":"2023-12-31","retiresAt":"2026-11-20","tools":false,"structuredOutput":false,"cacheReadPerMillion":null,"weightsUrl":"https://huggingface.co/meta-llama/Llama-3.2-1B-Instruct"},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":5,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-french:overall","benchmarkName":"EuroEval French","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":15.68,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/french","measuredAt":null,"stderr":0.622640546382903,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-dutch:overall","benchmarkName":"EuroEval Dutch","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":14.984444444444446,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/dutch","measuredAt":null,"stderr":0.5113719141367672,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-portuguese:overall","benchmarkName":"EuroEval Portuguese","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":13.967500000000001,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/portuguese","measuredAt":null,"stderr":0.6657784447547096,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":13.198749999999999,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.4629288552250767,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":12.3375,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":0.6332986558488815,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-spanish:overall","benchmarkName":"EuroEval Spanish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":12.273750000000001,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/spanish","measuredAt":null,"stderr":0.4262280052976341,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-italian:overall","benchmarkName":"EuroEval Italian","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":11.74875,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/italian","measuredAt":null,"stderr":0.3930648801406709,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":9.661428571428571,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.3621027027611624,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":0.027,"outputPerMillion":0.201,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":116,"timeToFirstToken":0.216,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":{"elo":1054.6410197403475,"eloUpper":1062.3355311700025,"eloLower":1046.9465083106925,"votes":8045,"rank":394,"rankUpper":null,"rankLower":null,"arenaName":"Llama 3.2 1B Instruct","organization":"meta","updatedAt":"2026-11-09"},"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":0.027,"outputPerMillion":0.201,"cacheReadPerMillion":null,"contextTokens":60000,"maxOutputTokens":54000,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"llama323binstruct","providerId":"meta","slug":"meta-llama-3-2-3b-instruct","name":"Llama 3.2 3B Instruct","provider":"Meta","description":"Llama 3.2 3B Instruct is a model from Meta. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":"2024-09-25T00:00:00.000Z","contextWindowTokens":131072,"modalities":["text"],"openWeights":true,"details":{"apiId":"meta-llama/llama-3.2-3b-instruct","maxOutputTokens":117964,"knowledgeCutoff":"2023-12-31","retiresAt":null,"tools":false,"structuredOutput":true,"cacheReadPerMillion":null,"weightsUrl":"https://huggingface.co/meta-llama/Llama-3.2-3B-Instruct"},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":16.701088863732572,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-french:overall","benchmarkName":"EuroEval French","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":39.17125,"unit":"percent","family":"euroeval","index":25.366608660662873,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/french","measuredAt":null,"stderr":0.44589096200752937,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-dutch:overall","benchmarkName":"EuroEval Dutch","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":38.173333333333325,"unit":"percent","family":"euroeval","index":24.012665358079563,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/dutch","measuredAt":null,"stderr":0.3575137051676705,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-portuguese:overall","benchmarkName":"EuroEval Portuguese","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":33.5775,"unit":"percent","family":"euroeval","index":17.77717699586924,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/portuguese","measuredAt":null,"stderr":0.352384732926953,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":33.04625,"unit":"percent","family":"euroeval","index":17.05639298301174,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":0.3151041494490353,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-italian:overall","benchmarkName":"EuroEval Italian","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":32.5225,"unit":"percent","family":"euroeval","index":16.34578474445341,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/italian","measuredAt":null,"stderr":0.4703771757855604,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":31.62285714285714,"unit":"percent","family":"euroeval","index":15.12517638923456,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.28859971572814713,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":30.128749999999997,"unit":"percent","family":"euroeval","index":13.09801678063669,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.5794865723207053,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-spanish:overall","benchmarkName":"EuroEval Spanish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":29.56375,"unit":"percent","family":"euroeval","index":12.331441783432957,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/spanish","measuredAt":null,"stderr":0.41535188394901973,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":0.049999999999999996,"outputPerMillion":0.33,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":95,"timeToFirstToken":0.204,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":{"elo":1109.5405651673113,"eloUpper":1117.090782166416,"eloLower":1101.990348168207,"votes":7936,"rank":366,"rankUpper":null,"rankLower":null,"arenaName":"Llama 3.2 3B Instruct","organization":"meta","updatedAt":"2026-11-09"},"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":0.05,"outputPerMillion":0.33,"cacheReadPerMillion":null,"contextTokens":131072,"maxOutputTokens":117964,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"llama3290b","providerId":"meta","slug":"meta-llama-3-2-90b","name":"Llama 3.2 90B","provider":"Meta","description":"Llama 3.2 90B is a model from Meta. It has 3 scored results in 2 categories. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":14.845605706148795,"multilingual":null,"instruction":null,"math":14.304130301604005},"capabilityCounts":{"knowledge":1,"math":2},"benchmarks":[{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":41.035353535353536,"unit":"percent","family":"gpqa-diamond","index":14.845605706148795,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.2948341091025695,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-math-level-5:overall","benchmarkName":"MATH level 5","category":"math","domain":null,"eligible":true,"ranked":true,"score":39.43542296072508,"unit":"percent","family":"epoch-ai:epoch-math-level-5:overall","index":18.153879883736508,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.0293053094701803,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":2.638888888888889,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":10.454380719471487,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0.941990275434065,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":25,"benchmarkCount":3,"categoryCount":2,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"llama3290bvisioninstructturbo","providerId":"meta","slug":"meta-llama-3-2-90b-vision-instruct-turbo","name":"Llama 3.2 90B Vision Instruct Turbo","provider":"Meta","description":"Llama 3.2 90B Vision Instruct Turbo is a model from Meta. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":17.705195550788353,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"multimodal":1},"benchmarks":[{"benchmarkId":"vals-ai:mmmu:overall","benchmarkName":"MMMU Pro","category":"multimodal","domain":null,"eligible":true,"ranked":true,"score":48.065,"unit":"percent","family":"mmmu-pro","index":17.705195550788353,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmmu","measuredAt":"2026-09-01","stderr":1.201,"costPerTest":0.008287,"latency":56.329},{"benchmarkId":"vals-ai:mortgage_tax:overall","benchmarkName":"MortgageTax","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":38.792,"unit":"percent","family":"vals-ai:mortgage_tax:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mortgage_tax","measuredAt":"2026-09-01","stderr":1.901,"costPerTest":0.000576,"latency":5.068}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":10,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"llama3370binstruct","providerId":"meta","slug":"meta-llama-3-3-70b-instruct","name":"Llama 3.3 70B Instruct","provider":"Meta","description":"Llama 3.3 70B Instruct is a model from Meta. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":"2024-12-06T17:28:57.000Z","contextWindowTokens":131072,"modalities":["text"],"openWeights":true,"details":{"apiId":"meta-llama/llama-3.3-70b-instruct","maxOutputTokens":115200,"knowledgeCutoff":"2023-12-31","retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.71,"weightsUrl":"https://huggingface.co/meta-llama/Llama-3.3-70B-Instruct"},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":47.31914574872741,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-french:overall","benchmarkName":"EuroEval French","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":60.0725,"unit":"percent","family":"euroeval","index":53.72479567005155,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/french","measuredAt":null,"stderr":0.281147203436207,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-portuguese:overall","benchmarkName":"EuroEval Portuguese","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":55.35125,"unit":"percent","family":"euroeval","index":47.31914574872741,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/portuguese","measuredAt":null,"stderr":0.28295593738248365,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":54.27857142857143,"unit":"percent","family":"euroeval","index":45.86376774360641,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.24530281161812043,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":0.71,"outputPerMillion":0.71,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":84,"timeToFirstToken":0.208,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":{"elo":1274.2433318364738,"eloUpper":1277.7298887131187,"eloLower":1270.7567749598288,"votes":54482,"rank":281,"rankUpper":null,"rankLower":null,"arenaName":"Llama 3.3 70B Instruct","organization":"meta","updatedAt":"2026-11-09"},"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[{"host":"deepinfra","name":"Deep Infra","kind":"host","url":"https://deepinfra.com/models","inputPerMillion":0.1,"outputPerMillion":0.32,"cacheReadPerMillion":null,"contextTokens":131072,"maxOutputTokens":16384,"deprecated":false},{"host":"novita-ai","name":"Novita AI","kind":"host","url":"https://novita.ai/docs/guides/introduction","inputPerMillion":0.135,"outputPerMillion":0.4,"cacheReadPerMillion":null,"contextTokens":12288,"maxOutputTokens":12288,"deprecated":false},{"host":"coreweave","name":"CoreWeave","kind":"host","url":"https://docs.wandb.ai/inference","inputPerMillion":0.71,"outputPerMillion":0.71,"cacheReadPerMillion":0.71,"contextTokens":128000,"maxOutputTokens":128000,"deprecated":false},{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":0.71,"outputPerMillion":0.71,"cacheReadPerMillion":0.71,"contextTokens":131072,"maxOutputTokens":115200,"deprecated":false},{"host":"amazon-bedrock","name":"Amazon Bedrock","kind":"cloud","url":"https://docs.aws.amazon.com/bedrock/latest/userguide/models-supported.html","inputPerMillion":0.72,"outputPerMillion":0.72,"cacheReadPerMillion":null,"contextTokens":128000,"maxOutputTokens":4096,"deprecated":true},{"host":"oracle-cloud","name":"Oracle Cloud","kind":"cloud","url":"https://docs.oracle.com/en-us/iaas/Content/generative-ai/pretrained-models.htm","inputPerMillion":0.72,"outputPerMillion":0.72,"cacheReadPerMillion":null,"contextTokens":128000,"maxOutputTokens":4096,"deprecated":true},{"host":"ovhcloud","name":"OVHcloud","kind":"cloud","url":"https://www.ovhcloud.com/en/public-cloud/ai-endpoints/catalog//","inputPerMillion":0.74,"outputPerMillion":0.74,"cacheReadPerMillion":null,"contextTokens":131072,"maxOutputTokens":131072,"deprecated":false},{"host":"crusoe","name":"Crusoe","kind":"host","url":"https://docs.crusoecloud.com/managed-inference/overview","inputPerMillion":0.25,"outputPerMillion":0.75,"cacheReadPerMillion":0.13,"contextTokens":128000,"maxOutputTokens":4096,"deprecated":true},{"host":"ibm-watsonx","name":"IBM watsonx","kind":"cloud","url":"https://www.ibm.com/docs/en/watsonx/saas?topic=solutions-supported-foundation-models","inputPerMillion":0.7526,"outputPerMillion":0.7526,"cacheReadPerMillion":null,"contextTokens":131072,"maxOutputTokens":4096,"deprecated":false},{"host":"stackit","name":"STACKIT","kind":"cloud","url":"https://docs.stackit.cloud/products/data-and-ai/ai-model-serving/basics/available-shared-models","inputPerMillion":0.53,"outputPerMillion":0.76,"cacheReadPerMillion":null,"contextTokens":128000,"maxOutputTokens":4096,"deprecated":false},{"host":"groq","name":"Groq","kind":"host","url":"https://console.groq.com/docs/models","inputPerMillion":0.59,"outputPerMillion":0.79,"cacheReadPerMillion":null,"contextTokens":131072,"maxOutputTokens":32768,"deprecated":true},{"host":"scaleway","name":"Scaleway","kind":"cloud","url":"https://www.scaleway.com/en/docs/generative-apis/","inputPerMillion":1.00935,"outputPerMillion":1.00935,"cacheReadPerMillion":null,"contextTokens":100000,"maxOutputTokens":16384,"deprecated":false},{"host":"io-net","name":"IO.NET","kind":"host","url":"https://io.net/docs/guides/intelligence/io-intelligence","inputPerMillion":0.5466,"outputPerMillion":1.0226,"cacheReadPerMillion":0.2733,"contextTokens":128000,"maxOutputTokens":4096,"deprecated":false},{"host":"together-ai","name":"Together AI","kind":"host","url":"https://docs.together.ai/docs/serverless-models","inputPerMillion":1.04,"outputPerMillion":1.04,"cacheReadPerMillion":null,"contextTokens":131072,"maxOutputTokens":131072,"deprecated":true},{"host":"sambanova","name":"SambaNova","kind":"host","url":"https://docs.sambanova.ai/cloud/docs/get-started/supported-models","inputPerMillion":0.6,"outputPerMillion":1.2,"cacheReadPerMillion":null,"contextTokens":131072,"maxOutputTokens":3072,"deprecated":false},{"host":"cloudflare","name":"Cloudflare Workers AI","kind":"cloud","url":"https://developers.cloudflare.com/workers-ai/models/","inputPerMillion":0.293,"outputPerMillion":2.253,"cacheReadPerMillion":null,"contextTokens":24000,"maxOutputTokens":24000,"deprecated":false},{"host":"regolo","name":"Regolo","kind":"host","url":"https://docs.regolo.ai/","inputPerMillion":0.6,"outputPerMillion":2.7,"cacheReadPerMillion":null,"contextTokens":128000,"maxOutputTokens":16384,"deprecated":true},{"host":"tinfoil","name":"Tinfoil","kind":"host","url":"https://docs.tinfoil.sh","inputPerMillion":1.75,"outputPerMillion":2.75,"cacheReadPerMillion":null,"contextTokens":131072,"maxOutputTokens":4096,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"llama3370binstructturbo","providerId":"meta","slug":"meta-llama-3-3-70b-instruct-turbo","name":"Llama 3.3 70B Instruct Turbo","provider":"Meta","description":"Llama 3.3 70B Instruct Turbo is a model from Meta. It has 5 scored results in 3 categories. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":19.497392506687316,"reasoning":null,"multimodal":null,"knowledge":27.781850441868475,"multilingual":null,"instruction":null,"math":22.103245063910244},"capabilityCounts":{"math":2,"knowledge":2,"coding":1},"benchmarks":[{"benchmarkId":"vals-ai:mgsm:overall","benchmarkName":"MGSM","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":91.091,"unit":"percent","family":"vals-ai:mgsm:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mgsm","measuredAt":"2026-01-09","stderr":1.765,"costPerTest":0.000515,"latency":3.008},{"benchmarkId":"vals-ai:legal_bench:overall","benchmarkName":"LegalBench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":77.18,"unit":"percent","family":"vals-ai:legal_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","measuredAt":"2026-10-13","stderr":0.771,"costPerTest":null,"latency":0.548},{"benchmarkId":"vals-ai:math500:overall","benchmarkName":"MATH 500","category":"math","domain":null,"eligible":true,"ranked":true,"score":73.4,"unit":"percent","family":"math-500","index":21.69283604873047,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/math500","measuredAt":"2026-01-09","stderr":3.862,"costPerTest":0.000698,"latency":5.414},{"benchmarkId":"vals-ai:mmlu_pro:overall","benchmarkName":"MMLU Pro","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":69.86,"unit":"percent","family":"mmlu-pro","index":32.495966399898165,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","measuredAt":"2026-09-01","stderr":0.442,"costPerTest":0.001565,"latency":4.179},{"benchmarkId":"vals-ai:tax_eval_v2:overall","benchmarkName":"TaxEval v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":59.444,"unit":"percent","family":"vals-ai:tax_eval_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","measuredAt":"2026-09-01","stderr":0.962,"costPerTest":0.000532,"latency":3.836},{"benchmarkId":"vals-ai:gpqa:overall","benchmarkName":"GPQA Diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":50,"unit":"percent","family":"gpqa-diamond","index":23.06773448383878,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","measuredAt":"2026-09-01","stderr":2.512,"costPerTest":0.001608,"latency":7.342},{"benchmarkId":"vals-ai:lcb:overall","benchmarkName":"LiveCodeBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":36.341,"unit":"percent","family":"livecodebench","index":19.497392506687316,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","measuredAt":"2026-09-01","stderr":1.047,"costPerTest":0.000743,"latency":2.945},{"benchmarkId":"vals-ai:aime:overall","benchmarkName":"AIME","category":"math","domain":null,"eligible":true,"ranked":true,"score":16.042,"unit":"percent","family":"vals-ai:aime:overall","index":22.513654079090017,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/aime","measuredAt":"2026-04-16","stderr":1.077,"costPerTest":0.001136,"latency":11.244}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":45,"benchmarkCount":5,"categoryCount":3,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"l33euryale70b","providerId":"sao10k","slug":"sao10k-llama-3-3-euryale-70b","name":"Llama 3.3 Euryale 70B","provider":"Sao10K","description":"Llama 3.3 Euryale 70B is a model from Sao10K. It has no scored result.","logoUrl":null,"releaseDate":"2024-12-18T15:32:08.000Z","contextWindowTokens":131072,"modalities":["text"],"openWeights":true,"details":{"apiId":"sao10k/l3.3-euryale-70b","maxOutputTokens":16384,"knowledgeCutoff":"2023-12-31","retiresAt":null,"tools":false,"structuredOutput":true,"cacheReadPerMillion":null,"weightsUrl":"https://huggingface.co/Sao10K/L3.3-70B-Euryale-v2.3"},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":0.65,"outputPerMillion":0.75,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":19.5,"timeToFirstToken":1.2805,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"llama33nemotronsuper49bv142e84561","providerId":"nvidia","slug":"nvidia-llama-3-3-nemotron-super-49b-v1-42e84561","name":"Llama 3.3 Nemotron Super 49B V1 42e84561","provider":"NVIDIA","description":"Llama 3.3 Nemotron Super 49B V1 42e84561 is a model from NVIDIA. It has 5 scored results in 3 categories. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":19.47001104613022,"reasoning":null,"multimodal":null,"knowledge":27.317768755934424,"multilingual":null,"instruction":null,"math":19.483918987828172},"capabilityCounts":{"math":2,"knowledge":2,"coding":1},"benchmarks":[{"benchmarkId":"vals-ai:math500:overall","benchmarkName":"MATH 500","category":"math","domain":null,"eligible":true,"ranked":true,"score":71.2,"unit":"percent","family":"math-500","index":19.36796798138473,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/math500","measuredAt":"2026-01-09","stderr":2.025,"costPerTest":null,"latency":12.55},{"benchmarkId":"vals-ai:mmlu_pro:overall","benchmarkName":"MMLU Pro","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":70.778,"unit":"percent","family":"mmlu-pro","index":33.652536518122744,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","measuredAt":"2026-09-01","stderr":0.45,"costPerTest":null,"latency":17.45},{"benchmarkId":"vals-ai:tax_eval_v2:overall","benchmarkName":"TaxEval v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":60.22,"unit":"percent","family":"vals-ai:tax_eval_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","measuredAt":"2026-09-01","stderr":0.959,"costPerTest":null,"latency":18.308},{"benchmarkId":"vals-ai:gpqa:overall","benchmarkName":"GPQA Diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":47.727,"unit":"percent","family":"gpqa-diamond","index":20.983000993746106,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","measuredAt":"2026-09-01","stderr":2.468,"costPerTest":null,"latency":16.482},{"benchmarkId":"vals-ai:lcb:overall","benchmarkName":"LiveCodeBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":36.308,"unit":"percent","family":"livecodebench","index":19.47001104613022,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","measuredAt":"2026-09-01","stderr":1.039,"costPerTest":null,"latency":44.228},{"benchmarkId":"vals-ai:aime:overall","benchmarkName":"AIME","category":"math","domain":null,"eligible":true,"ranked":true,"score":9.375,"unit":"percent","family":"vals-ai:aime:overall","index":19.599869994271614,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/aime","measuredAt":"2026-04-16","stderr":0.54,"costPerTest":0,"latency":15.851}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":45,"benchmarkCount":5,"categoryCount":3,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"llama33nemotronsuper49bv142e84561thinking","providerId":"nvidia","slug":"nvidia-llama-3-3-nemotron-super-49b-v1-42e84561-thinking","name":"Llama 3.3 Nemotron Super 49B V1 42e84561 Thinking","provider":"NVIDIA","description":"Llama 3.3 Nemotron Super 49B V1 42e84561 Thinking is a model from NVIDIA. It has 5 scored results in 3 categories. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":37.77493229915966,"reasoning":null,"multimodal":null,"knowledge":33.27812471871181,"multilingual":null,"instruction":null,"math":39.808676983930525},"capabilityCounts":{"math":2,"knowledge":2,"coding":1},"benchmarks":[{"benchmarkId":"vals-ai:math500:overall","benchmarkName":"MATH 500","category":"math","domain":null,"eligible":true,"ranked":true,"score":91.4,"unit":"percent","family":"math-500","index":40.71448387246832,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/math500","measuredAt":"2026-01-09","stderr":1.254,"costPerTest":null,"latency":42.828},{"benchmarkId":"vals-ai:mmlu_pro:overall","benchmarkName":"MMLU Pro","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":69.577,"unit":"percent","family":"mmlu-pro","index":32.13942027412741,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","measuredAt":"2026-09-01","stderr":0.441,"costPerTest":null,"latency":36.351},{"benchmarkId":"vals-ai:tax_eval_v2:overall","benchmarkName":"TaxEval v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":63.736,"unit":"percent","family":"vals-ai:tax_eval_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","measuredAt":"2026-09-01","stderr":0.943,"costPerTest":null,"latency":32.67},{"benchmarkId":"vals-ai:gpqa:overall","benchmarkName":"GPQA Diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":62.374,"unit":"percent","family":"gpqa-diamond","index":34.41682916329621,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","measuredAt":"2026-09-01","stderr":2.451,"costPerTest":null,"latency":77.589},{"benchmarkId":"vals-ai:lcb:overall","benchmarkName":"LiveCodeBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":58.369,"unit":"percent","family":"livecodebench","index":37.77493229915966,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","measuredAt":"2026-09-01","stderr":1.172,"costPerTest":null,"latency":152.4},{"benchmarkId":"vals-ai:aime:overall","benchmarkName":"AIME","category":"math","domain":null,"eligible":true,"ranked":true,"score":53.542,"unit":"percent","family":"vals-ai:aime:overall","index":38.90287009539273,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/aime","measuredAt":"2026-04-16","stderr":1.596,"costPerTest":0,"latency":167.893}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":45,"benchmarkCount":5,"categoryCount":3,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"llama-4-behemoth","providerId":"meta","slug":"llama-4-behemoth","name":"Llama 4 Behemoth","provider":"Meta","description":"Llama 4 Behemoth is a standard model from Meta. It has no scored result.","logoUrl":null,"releaseDate":"2026-02-28","contextWindowTokens":32000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"llama4maverickinstruct","providerId":"meta","slug":"meta-llama-4-maverick-instruct","name":"Llama 4 Maverick Instruct","provider":"Meta","description":"Llama 4 Maverick Instruct is a model from Meta. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":16.746138966040526,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"coding":1},"benchmarks":[{"benchmarkId":"swe-bench:swe-bench-verified:overall","benchmarkName":"SWE-bench Verified","category":"coding","domain":null,"eligible":true,"ranked":true,"score":21.04,"unit":"percent","family":"swe-bench-verified","index":16.746138966040526,"aboveFit":false,"run":"mini-SWE-agent","source":"SWE-bench team","sourceUrl":"https://www.swebench.com/","measuredAt":"2026-02-26","stderr":null,"costPerTest":0.31370609759999996,"latency":null},{"benchmarkId":"swebench-experiments:swe-bench-verified:overall","benchmarkName":"SWE-bench Verified","category":"coding","domain":null,"eligible":true,"ranked":true,"score":21.04,"unit":"percent","family":"swe-bench-verified","index":16.746138966040526,"aboveFit":false,"run":"mini-SWE-agent","source":"SWE-bench team","sourceUrl":"https://github.com/SWE-bench/experiments/tree/main/evaluation/verified","measuredAt":"2026-09-01","stderr":null,"costPerTest":0.31370609759999996,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":20,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:swe-bench-team","name":"SWE-bench team","url":"https://www.swebench.com","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"llama4scout17b16einstruct","providerId":"meta","slug":"meta-llama-4-scout-17b-16e-instruct","name":"Llama 4 Scout 17B 16E Instruct","provider":"Meta","description":"Llama 4 Scout 17B 16E Instruct is a model from Meta. It has 6 scored results in 4 categories. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":21.322823210493336,"reasoning":null,"multimodal":31.613388428116874,"knowledge":26.248707484814236,"multilingual":null,"instruction":null,"math":25.805056598670745},"capabilityCounts":{"math":2,"knowledge":2,"multimodal":1,"coding":1},"benchmarks":[{"benchmarkId":"vals-ai:mgsm:overall","benchmarkName":"MGSM","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":87.964,"unit":"percent","family":"vals-ai:mgsm:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mgsm","measuredAt":"2026-01-09","stderr":1.993,"costPerTest":0.000147,"latency":3.499},{"benchmarkId":"vals-ai:math500:overall","benchmarkName":"MATH 500","category":"math","domain":null,"eligible":true,"ranked":true,"score":79.2,"unit":"percent","family":"math-500","index":27.822033680823775,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/math500","measuredAt":"2026-01-09","stderr":1.815,"costPerTest":0.00043,"latency":10.523},{"benchmarkId":"vals-ai:legal_bench:overall","benchmarkName":"LegalBench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":72.036,"unit":"percent","family":"vals-ai:legal_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","measuredAt":"2026-10-13","stderr":0.479,"costPerTest":null,"latency":0.392},{"benchmarkId":"vals-ai:mmlu_pro:overall","benchmarkName":"MMLU Pro","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":69.632,"unit":"percent","family":"mmlu-pro","index":32.20871369079664,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","measuredAt":"2026-09-01","stderr":0.445,"costPerTest":0.000507,"latency":5.342},{"benchmarkId":"vals-ai:mmmu:overall","benchmarkName":"MMMU Pro","category":"multimodal","domain":null,"eligible":true,"ranked":true,"score":58.752,"unit":"percent","family":"mmmu-pro","index":31.613388428116874,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmmu","measuredAt":"2026-09-01","stderr":1.183,"costPerTest":0.000835,"latency":10.506},{"benchmarkId":"vals-ai:mortgage_tax:overall","benchmarkName":"MortgageTax","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":57.75,"unit":"percent","family":"vals-ai:mortgage_tax:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mortgage_tax","measuredAt":"2026-09-01","stderr":0.979,"costPerTest":0.000572,"latency":2.426},{"benchmarkId":"vals-ai:tax_eval_v2:overall","benchmarkName":"TaxEval v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":55.192,"unit":"percent","family":"vals-ai:tax_eval_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","measuredAt":"2026-09-01","stderr":0.975,"costPerTest":0.000378,"latency":7.042},{"benchmarkId":"vals-ai:medqa:overall","benchmarkName":"MedQA","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":50.9,"unit":"percent","family":"vals-ai:medqa:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medqa","measuredAt":"2026-04-16","stderr":0.456,"costPerTest":0.000326,"latency":4.802},{"benchmarkId":"vals-ai:medscribe:overall","benchmarkName":"MedScribe","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":50.593,"unit":"percent","family":"vals-ai:medscribe:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medscribe","measuredAt":"2026-10-29","stderr":1.901,"costPerTest":0.0017,"latency":11.323},{"benchmarkId":"vals-ai:gpqa:overall","benchmarkName":"GPQA Diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":46.97,"unit":"percent","family":"gpqa-diamond","index":20.288701278831834,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","measuredAt":"2026-09-01","stderr":2.491,"costPerTest":0.000654,"latency":12.092},{"benchmarkId":"vals-ai:corp_fin_v2:overall","benchmarkName":"CorpFin v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":46.776,"unit":"percent","family":"vals-ai:corp_fin_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","measuredAt":"2026-08-12","stderr":0.98,"costPerTest":0.011639,"latency":9.697},{"benchmarkId":"vals-ai:lcb:overall","benchmarkName":"LiveCodeBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":38.541,"unit":"percent","family":"livecodebench","index":21.322823210493336,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","measuredAt":"2026-09-01","stderr":1.016,"costPerTest":0.000615,"latency":16.762},{"benchmarkId":"vals-ai:medcode:overall","benchmarkName":"MedCode","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":23.311,"unit":"percent","family":"vals-ai:medcode:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medcode","measuredAt":"2026-10-29","stderr":1.749,"costPerTest":0.002176,"latency":10.736},{"benchmarkId":"vals-ai:aime:overall","benchmarkName":"AIME","category":"math","domain":null,"eligible":true,"ranked":true,"score":18.958,"unit":"percent","family":"vals-ai:aime:overall","index":23.788079516517715,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/aime","measuredAt":"2026-04-16","stderr":0.7,"costPerTest":0.000813,"latency":21.69}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":{"elo":1279.2065314031074,"eloUpper":1283.9206873570588,"eloLower":1274.492375449156,"votes":29777,"rank":278,"rankUpper":null,"rankLower":null,"arenaName":"Llama 4 Scout 17B 16E Instruct","organization":"meta","updatedAt":"2026-11-09"},"quality":{"coveragePercent":55.00000000000001,"benchmarkCount":6,"categoryCount":4,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"llama4scoutinstruct","providerId":"meta","slug":"meta-llama-4-scout-instruct","name":"Llama 4 Scout Instruct","provider":"Meta","description":"Llama 4 Scout Instruct is a model from Meta. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":7.910199184191896,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"coding":1},"benchmarks":[{"benchmarkId":"swe-bench:swe-bench-verified:overall","benchmarkName":"SWE-bench Verified","category":"coding","domain":null,"eligible":true,"ranked":true,"score":9.06,"unit":"percent","family":"swe-bench-verified","index":7.910199184191896,"aboveFit":false,"run":"mini-SWE-agent","source":"SWE-bench team","sourceUrl":"https://www.swebench.com/","measuredAt":"2026-02-26","stderr":null,"costPerTest":0.115631523,"latency":null},{"benchmarkId":"swebench-experiments:swe-bench-verified:overall","benchmarkName":"SWE-bench Verified","category":"coding","domain":null,"eligible":true,"ranked":true,"score":9.06,"unit":"percent","family":"swe-bench-verified","index":7.910199184191896,"aboveFit":false,"run":"mini-SWE-agent","source":"SWE-bench team","sourceUrl":"https://github.com/SWE-bench/experiments/tree/main/evaluation/verified","measuredAt":"2026-09-01","stderr":null,"costPerTest":0.115631523,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":20,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:swe-bench-team","name":"SWE-bench team","url":"https://www.swebench.com","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"llamaguard412b","providerId":"meta","slug":"meta-llama-guard-4-12b","name":"Llama Guard 4 12B","provider":"Meta","description":"Llama Guard 4 12B is a model from Meta. It has no scored result.","logoUrl":null,"releaseDate":"2025-04-30T01:06:33.000Z","contextWindowTokens":163840,"modalities":["image","text"],"openWeights":true,"details":{"apiId":"meta-llama/llama-guard-4-12b","maxOutputTokens":16384,"knowledgeCutoff":"2024-08-31","retiresAt":null,"tools":false,"structuredOutput":false,"cacheReadPerMillion":null,"weightsUrl":"https://huggingface.co/meta-llama/Llama-Guard-4-12B"},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":0.18,"outputPerMillion":0.18,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":42,"timeToFirstToken":0.2965,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[{"host":"deepinfra","name":"Deep Infra","kind":"host","url":"https://deepinfra.com/models","inputPerMillion":0.18,"outputPerMillion":0.18,"cacheReadPerMillion":null,"contextTokens":163840,"maxOutputTokens":163840,"deprecated":false},{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":0.18,"outputPerMillion":0.18,"cacheReadPerMillion":null,"contextTokens":163840,"maxOutputTokens":16384,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"llamapllum8bbase2412","providerId":"meta","slug":"meta-llama-pllum-8b-base-2412","name":"Llama PLLuM 8B Base 2412","provider":"Meta","description":"Llama PLLuM 8B Base 2412 is a model from Meta. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":23.71159165500588,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":37.95142857142857,"unit":"percent","family":"euroeval","index":23.71159165500588,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.8371745484006283,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"llamapllum8bchat2412","providerId":"meta","slug":"meta-llama-pllum-8b-chat-2412","name":"Llama PLLuM 8B Chat 2412","provider":"Meta","description":"Llama PLLuM 8B Chat 2412 is a model from Meta. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":28.663801813932686,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":41.60142857142858,"unit":"percent","family":"euroeval","index":28.663801813932686,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.5415925119308721,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"llamapllum8binstruct2412","providerId":"meta","slug":"meta-llama-pllum-8b-instruct-2412","name":"Llama PLLuM 8B Instruct 2412","provider":"Meta","description":"Llama PLLuM 8B Instruct 2412 is a model from Meta. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":27.981540375207743,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":41.09857142857144,"unit":"percent","family":"euroeval","index":27.981540375207743,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.5081137587555907,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"llama-omni","providerId":"ictnlp","slug":"llama-omni","name":"LLaMA-Omni","provider":"ICTNLP","description":"LLaMA-Omni is a standard model from ICTNLP. It has no scored result.","logoUrl":null,"releaseDate":"2024-09-10","contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"llama3swerl70b","providerId":"meta","slug":"meta-llama3-swe-rl-70b","name":"Llama3 SWE RL 70B","provider":"Meta","description":"Llama3 SWE RL 70B is a model from Meta. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":31.615299734159752,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"coding":1},"benchmarks":[{"benchmarkId":"swebench-experiments:swe-bench-verified:overall","benchmarkName":"SWE-bench Verified","category":"coding","domain":null,"eligible":true,"ranked":true,"score":41.199999999999996,"unit":"percent","family":"swe-bench-verified","index":31.615299734159752,"aboveFit":false,"run":"Agentless Mini","source":"SWE-bench team","sourceUrl":"https://github.com/SWE-bench/experiments/tree/main/evaluation/verified","measuredAt":"2026-09-01","stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":20,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:swe-bench-team","name":"SWE-bench team","url":"https://github.com","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"localjev-qwen3-5-4b","providerId":"amith-chandrappa-amithgc","slug":"localjev-qwen3-5-4b","name":"local-jev Qwen3.5-4B","provider":"Amith Chandrappa (amithgc)","description":"local-jev Qwen3.5-4B is a standard model from Amith Chandrappa (amithgc). It has 2 scored results in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":76.59441974115266,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"reasoning":2},"benchmarks":[{"benchmarkId":"jevBench14","benchmarkName":"jevBench14","category":"reasoning","score":46.79864791106764,"unit":"percent","family":"jevBench14","index":86.0790605345139,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"jevBench15","benchmarkName":"jevBench15","category":"reasoning","score":25.82001585149664,"unit":"percent","family":"jevBench15","index":67.10977894779144,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":15,"benchmarkCount":2,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"longcat-2-0","providerId":"meituan","slug":"longcat-2-0","name":"LongCat-2.0","provider":"Meituan","description":"LongCat-2.0 is a reasoning model from Meituan. It has no scored result.","logoUrl":null,"releaseDate":"2026-06-30","contextWindowTokens":1048756,"modalities":["text"],"openWeights":true,"details":{"apiId":"meituan/longcat-2.0","maxOutputTokens":262144,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":false,"cacheReadPerMillion":0.006,"weightsUrl":"https://huggingface.co/meituan-longcat/LongCat-2.0"},"reasoning":true,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":0.3,"outputPerMillion":1.2,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":38,"timeToFirstToken":2.2155,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[{"host":"longcat","name":"LongCat","kind":"maker","url":"https://longcat.chat/platform/docs/","inputPerMillion":0.3,"outputPerMillion":1.2,"cacheReadPerMillion":0.006,"contextTokens":1048576,"maxOutputTokens":131072,"deprecated":false},{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":0.3,"outputPerMillion":1.2,"cacheReadPerMillion":0.006,"contextTokens":1048756,"maxOutputTokens":262144,"deprecated":false},{"host":"siliconflow","name":"SiliconFlow","kind":"host","url":"https://cloud.siliconflow.com/models","inputPerMillion":0.75,"outputPerMillion":2.95,"cacheReadPerMillion":0.015,"contextTokens":1049000,"maxOutputTokens":131072,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"longcat-2-5","providerId":"meituan","slug":"longcat-2-5","name":"LongCat-2.5","provider":"Meituan","description":"LongCat-2.5 is a reasoning model from Meituan. It has 12 scored results in 3 categories. It has no index.","logoUrl":null,"releaseDate":"2026-11-02","contextWindowTokens":1000000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":true,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":72.92042442722186,"coding":62.66107631929548,"reasoning":null,"multimodal":66.87675901751388,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"agentic":9,"multimodal":1,"coding":2},"benchmarks":[{"benchmarkId":"mathVisionPython","benchmarkName":"MathVision with Python","category":"multimodal","score":96.5,"unit":"percent","family":"mathVisionPython","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"browseComp","benchmarkName":"BrowseComp","category":"agentic","score":91.9,"unit":"percent","family":"browseComp","index":72.51181422193514,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"charxiv","benchmarkName":"CharXiv Reasoning","category":"multimodal","score":90.2,"unit":"percent","family":"charxiv","index":66.87675901751388,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"terminalBench21","benchmarkName":"Terminal-Bench 2.1","category":"agentic","score":88.4,"unit":"percent","family":"terminal-bench-2-1","index":72.84233542676623,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"videoMme","benchmarkName":"Video-MME","category":"multimodal","score":85.2,"unit":"percent","family":"videoMme","index":null,"aboveFit":false,"run":null,"source":"Video-MME benchmark team","sourceUrl":"https://mme-benchmark.github.io/","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gdpvalRubrics","benchmarkName":"GDPval rubrics","category":"agentic","score":84.8,"unit":"percent","family":"gdpvalRubrics","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mcpAtlas","benchmarkName":"MCP Atlas","category":"agentic","score":83.1,"unit":"percent","family":"mcpAtlas","index":67.02145734517407,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"paperBench","benchmarkName":"PaperBench","category":"coding","score":82.1,"unit":"percent","family":"paperBench","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vision2Web","benchmarkName":"Vision2Web","category":"multimodal","score":78.6,"unit":"percent","family":"vision2Web","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"clawEval","benchmarkName":"Claw-Eval","category":"agentic","score":78.3,"unit":"percent","family":"clawEval","index":78.36041778855233,"aboveFit":false,"run":null,"source":"Bowen Ye, Rang Li, Qibin Yang, Yuanxin Liu, Linli Yao, Hanglong Lv, Zhihui Xie, Chenxin An, Lei Li, Lingpeng Kong, Qi Liu, Zhifang Sui, Tong Yang","sourceUrl":"https://arxiv.org/abs/2604.06132","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"frontierScience","benchmarkName":"FrontierScience","category":"knowledge","score":77,"unit":"percent","family":"frontierScience","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"jobBench","benchmarkName":"JobBench","category":"agentic","score":65.4,"unit":"percent","family":"jobBench","index":72.66627912569382,"aboveFit":false,"run":null,"source":"Yuetai Li, Yichen Feng, Zhangchen Xu, Zixian Ma, Kaiyuan Zheng, Fengqing Jiang, Xinghua Sun, Rulin Shao, Zichen Chen, Yue Huang, Xinyang Han, Brian Lee, Kayla Xu, Shenglai Zeng, Hang Hua, Xiangliang Zhang, Basel Alomair, Ranjay Krishna, Luke Zettlemoyer, Pang Wei Koh, Bhaskar Ramasubramanian, Luyao Niu, Xiang Yue, Radha Poovendran","sourceUrl":"https://arxiv.org/abs/2605.26329","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"deepSwe","benchmarkName":"DeepSWE","category":"agentic","score":64.6,"unit":"percent","family":"deepSwe","index":68.03911725887562,"aboveFit":false,"run":null,"source":"Datacurve AI","sourceUrl":"https://deepswe.datacurve.ai/blog","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hleWithTools","benchmarkName":"Humanity's Last Exam with tools","category":"agentic","score":63.1,"unit":"percent","family":"hleWithTools","index":71.20382431927493,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"nl2Repo","benchmarkName":"NL2Repo","category":"coding","score":60.5,"unit":"percent","family":"nl2Repo","index":69.43766646831837,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"sweAtlasCodebaseQna","benchmarkName":"SWE-Atlas Codebase QnA","category":"coding","score":58.3,"unit":"percent","family":"sweAtlasCodebaseQna","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"automationBench","benchmarkName":"AutomationBench","category":"agentic","score":51.8,"unit":"percent","family":"automationBench","index":80.61877554284692,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"agentsLastExam","benchmarkName":"Agents' Last Exam","category":"agentic","score":38.5,"unit":"percent","family":"agentsLastExam","index":73.01979881587766,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"spreadsheetBench2","benchmarkName":"SpreadsheetBench 2","category":"agentic","score":34.9,"unit":"percent","family":"spreadsheetBench2","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"programBench","benchmarkName":"ProgramBench: Can Language Models Rebuild Programs From Scratch?","category":"coding","score":25.5,"unit":"percent","family":"programBench","index":55.884486170272595,"aboveFit":false,"run":null,"source":"John Yang, Kilian Lieret, Jeffrey Ma, Parth Thakkar, Dmitrii Pedchenko, Sten Sootla, Emily McMilin, Pengcheng Yin, Rui Hou, Gabriel Synnaeve, Diyi Yang, Ofir Press","sourceUrl":"https://programbench.com/static/paper.pdf","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":0.3,"outputPerMillion":1.2,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":50,"benchmarkCount":12,"categoryCount":3,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"lucie7binstructhumandata","providerId":"unknown","slug":"unknown-lucie-7b-instruct-human-data","name":"Lucie 7B Instruct Human Data","provider":"Unknown","description":"Lucie 7B Instruct Human Data is a model. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":5,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-french:overall","benchmarkName":"EuroEval French","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":30.4875,"unit":"percent","family":"euroeval","index":13.584757984613404,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/french","measuredAt":null,"stderr":0.6272272813741443,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-portuguese:overall","benchmarkName":"EuroEval Portuguese","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":21.689999999999998,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/portuguese","measuredAt":null,"stderr":0.6074305515859406,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-italian:overall","benchmarkName":"EuroEval Italian","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":19.555,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/italian","measuredAt":null,"stderr":0.9056187456650839,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-spanish:overall","benchmarkName":"EuroEval Spanish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":19.06625,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/spanish","measuredAt":null,"stderr":0.7914217823006895,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"lucie7binstructv11","providerId":"unknown","slug":"unknown-lucie-7b-instruct-v1-1","name":"Lucie 7B Instruct V1.1","provider":"Unknown","description":"Lucie 7B Instruct V1.1 is a model. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":7.966034609091324,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-french:overall","benchmarkName":"EuroEval French","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":37.0275,"unit":"percent","family":"euroeval","index":22.45803317348498,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/french","measuredAt":null,"stderr":0.5931141331986618,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-portuguese:overall","benchmarkName":"EuroEval Portuguese","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":27.413750000000004,"unit":"percent","family":"euroeval","index":9.41438648433909,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/portuguese","measuredAt":null,"stderr":0.6689917787237748,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-italian:overall","benchmarkName":"EuroEval Italian","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":25.278750000000002,"unit":"percent","family":"euroeval","index":6.517682733843557,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/italian","measuredAt":null,"stderr":0.6420852260409049,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-spanish:overall","benchmarkName":"EuroEval Spanish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":22.35375,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/spanish","measuredAt":null,"stderr":0.8665097013305737,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"luciole1bbase","providerId":"unknown","slug":"unknown-luciole-1b-base","name":"Luciole 1B Base","provider":"Unknown","description":"Luciole 1B Base is a model. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":5,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-dutch:overall","benchmarkName":"EuroEval Dutch","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":23.332222222222224,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/dutch","measuredAt":null,"stderr":0.862914241193365,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-french:overall","benchmarkName":"EuroEval French","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":20.807500000000005,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/french","measuredAt":null,"stderr":0.8167752903951001,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-portuguese:overall","benchmarkName":"EuroEval Portuguese","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":17.28125,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/portuguese","measuredAt":null,"stderr":0.7784218811158895,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-spanish:overall","benchmarkName":"EuroEval Spanish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":15.250000000000002,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/spanish","measuredAt":null,"stderr":0.8990889486029734,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-italian:overall","benchmarkName":"EuroEval Italian","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":14.1325,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/italian","measuredAt":null,"stderr":0.7536048782352726,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":13.165000000000001,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.8404686192833138,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":11.94714285714286,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.9898783681207856,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":10.39875,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":0.9244533857907602,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"luciole1binstruct11","providerId":"unknown","slug":"unknown-luciole-1b-instruct-1-1","name":"Luciole 1B Instruct 1.1","provider":"Unknown","description":"Luciole 1B Instruct 1.1 is a model. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":5,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-french:overall","benchmarkName":"EuroEval French","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":25.810000000000006,"unit":"percent","family":"euroeval","index":7.238466746701057,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/french","measuredAt":null,"stderr":0.68097356776897,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-dutch:overall","benchmarkName":"EuroEval Dutch","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":24.198888888888888,"unit":"percent","family":"euroeval","index":5.05255967502864,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/dutch","measuredAt":null,"stderr":0.6774853649733598,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-portuguese:overall","benchmarkName":"EuroEval Portuguese","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":22.3925,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/portuguese","measuredAt":null,"stderr":0.6638653383781986,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-italian:overall","benchmarkName":"EuroEval Italian","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":17.9625,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/italian","measuredAt":null,"stderr":0.6905059286494215,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":16.462500000000002,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.7648641627766332,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":16.34625,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":0.92660671268883,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-spanish:overall","benchmarkName":"EuroEval Spanish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":13.9975,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/spanish","measuredAt":null,"stderr":0.6324839424839179,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":12.531428571428572,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.6610103583738692,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"luciole23bbase","providerId":"unknown","slug":"unknown-luciole-23b-base","name":"Luciole 23B Base","provider":"Unknown","description":"Luciole 23B Base is a model. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":10.099555287149506,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-dutch:overall","benchmarkName":"EuroEval Dutch","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":39.97999999999999,"unit":"percent","family":"euroeval","index":26.46389632258945,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/dutch","measuredAt":null,"stderr":0.6043259687810958,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-portuguese:overall","benchmarkName":"EuroEval Portuguese","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":37.0575,"unit":"percent","family":"euroeval","index":22.49873627068163,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/portuguese","measuredAt":null,"stderr":0.7336168959341108,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-french:overall","benchmarkName":"EuroEval French","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":36.972500000000004,"unit":"percent","family":"euroeval","index":22.38341082862444,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/french","measuredAt":null,"stderr":0.7101166277450486,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-italian:overall","benchmarkName":"EuroEval Italian","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":29.014999999999993,"unit":"percent","family":"euroeval","index":11.586914297210733,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/italian","measuredAt":null,"stderr":1.0998700207297225,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-spanish:overall","benchmarkName":"EuroEval Spanish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":26.8225,"unit":"percent","family":"euroeval","index":8.61219627708828,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/spanish","measuredAt":null,"stderr":0.7571121119094583,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":26.7625,"unit":"percent","family":"euroeval","index":8.530790082694956,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.8998619685818486,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":26.46142857142857,"unit":"percent","family":"euroeval","index":8.122305428685635,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.7115289659513203,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":23.68125,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":0.9316073140009153,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"luciole23binstruct11","providerId":"unknown","slug":"unknown-luciole-23b-instruct-1-1","name":"Luciole 23B Instruct 1.1","provider":"Unknown","description":"Luciole 23B Instruct 1.1 is a model. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":31.168495973569915,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-french:overall","benchmarkName":"EuroEval French","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":49.955000000000005,"unit":"percent","family":"euroeval","index":39.99767614047845,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/french","measuredAt":null,"stderr":0.5522708800941799,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-dutch:overall","benchmarkName":"EuroEval Dutch","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":44.593333333333334,"unit":"percent","family":"euroeval","index":32.72312815816451,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/dutch","measuredAt":null,"stderr":0.6379722893475144,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-portuguese:overall","benchmarkName":"EuroEval Portuguese","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":44.415000000000006,"unit":"percent","family":"euroeval","index":32.48117085816216,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/portuguese","measuredAt":null,"stderr":0.6264283678123141,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-italian:overall","benchmarkName":"EuroEval Italian","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":43.4475,"unit":"percent","family":"euroeval","index":31.168495973569915,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/italian","measuredAt":null,"stderr":0.6304202467402201,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":34.79875,"unit":"percent","family":"euroeval","index":19.43413224424988,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.6991490363291649,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-spanish:overall","benchmarkName":"EuroEval Spanish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":34.19625,"unit":"percent","family":"euroeval","index":18.61667837555032,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/spanish","measuredAt":null,"stderr":0.7509493990942399,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":29.83285714285714,"unit":"percent","family":"euroeval","index":12.696558256500602,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.8033272644928411,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"luciole8bbase","providerId":"unknown","slug":"unknown-luciole-8b-base","name":"Luciole 8B Base","provider":"Unknown","description":"Luciole 8B Base is a model. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":5,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-dutch:overall","benchmarkName":"EuroEval Dutch","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":33.56111111111111,"unit":"percent","family":"euroeval","index":17.75494104462291,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/dutch","measuredAt":null,"stderr":0.6722910892180376,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-french:overall","benchmarkName":"EuroEval French","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":32.53,"unit":"percent","family":"euroeval","index":16.355960518752575,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/french","measuredAt":null,"stderr":0.6405381038626821,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-portuguese:overall","benchmarkName":"EuroEval Portuguese","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":28.891250000000003,"unit":"percent","family":"euroeval","index":11.419014021274528,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/portuguese","measuredAt":null,"stderr":0.8324652770536438,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-italian:overall","benchmarkName":"EuroEval Italian","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":23.611250000000002,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/italian","measuredAt":null,"stderr":0.9707656449936823,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":22.48,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":1.2131835334005674,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-spanish:overall","benchmarkName":"EuroEval Spanish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":22.42375,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/spanish","measuredAt":null,"stderr":0.8696281532931186,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":21.665,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":1.0741842835379785,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":21.19875,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.8831813941654341,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"luciole8binstruct11","providerId":"unknown","slug":"unknown-luciole-8b-instruct-1-1","name":"Luciole 8B Instruct 1.1","provider":"Unknown","description":"Luciole 8B Instruct 1.1 is a model. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":16.686673183475428,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-french:overall","benchmarkName":"EuroEval French","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":42.58625000000001,"unit":"percent","family":"euroeval","index":29.999977891549186,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/french","measuredAt":null,"stderr":0.5219509675247282,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-portuguese:overall","benchmarkName":"EuroEval Portuguese","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":37.642500000000005,"unit":"percent","family":"euroeval","index":23.292446666016485,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/portuguese","measuredAt":null,"stderr":0.6472682983122223,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-dutch:overall","benchmarkName":"EuroEval Dutch","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":37.50888888888889,"unit":"percent","family":"euroeval","index":23.111167131279508,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/dutch","measuredAt":null,"stderr":0.6617951643115662,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-italian:overall","benchmarkName":"EuroEval Italian","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":34.18875,"unit":"percent","family":"euroeval","index":18.606502601251158,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/italian","measuredAt":null,"stderr":0.874474842405429,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-spanish:overall","benchmarkName":"EuroEval Spanish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":31.35875,"unit":"percent","family":"euroeval","index":14.76684376569969,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/spanish","measuredAt":null,"stderr":0.7114641593221686,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":28.60375,"unit":"percent","family":"euroeval","index":11.02894267313988,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":1.124348422420737,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":28.423750000000002,"unit":"percent","family":"euroeval","index":10.784724089959937,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.6914545357433126,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":24.34857142857143,"unit":"percent","family":"euroeval","index":5.255644440406684,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.7260052847392969,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"lynxinstruct30b","providerId":"unknown","slug":"unknown-lynx-instruct-30b","name":"Lynx Instruct 30B","provider":"Unknown","description":"Lynx Instruct 30B is a model. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":30.094951785008046,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":42.65625,"unit":"percent","family":"euroeval","index":30.094951785008046,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":0.9735028569552325,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"lynxinstruct30bnothinking","providerId":"unknown","slug":"unknown-lynx-instruct-30b-no-thinking","name":"Lynx Instruct 30B No Thinking","provider":"Unknown","description":"Lynx Instruct 30B No Thinking is a model. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":30.196709527999698,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":42.73125,"unit":"percent","family":"euroeval","index":30.196709527999698,"aboveFit":false,"run":"no-thinking effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":0.30027071119241716,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"lynxinstruct30bthinking","providerId":"unknown","slug":"unknown-lynx-instruct-30b-thinking","name":"Lynx Instruct 30B Thinking","provider":"Unknown","description":"Lynx Instruct 30B Thinking is a model. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":31.858752663529913,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":43.95625,"unit":"percent","family":"euroeval","index":31.858752663529913,"aboveFit":false,"run":"thinking effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":0.3332322763478952,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"lyra-base","providerId":"dvl-lab","slug":"lyra-base","name":"Lyra Base","provider":"DVL Lab","description":"Lyra Base is a standard model from DVL Lab. It has no scored result.","logoUrl":null,"releaseDate":"2024-12-12","contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"lyra-mini","providerId":"dvl-lab","slug":"lyra-mini","name":"Lyra Mini","provider":"DVL Lab","description":"Lyra Mini is a standard model from DVL Lab. It has no scored result.","logoUrl":null,"releaseDate":"2024-12-12","contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"lyria3clippreview","providerId":"google","slug":"google-lyria-3-clip-preview","name":"Lyria 3 Clip Preview","provider":"Google","description":"Lyria 3 Clip Preview is a model from Google. It has no scored result.","logoUrl":null,"releaseDate":"2026-03-30T21:47:35.000Z","contextWindowTokens":1048576,"modalities":["text","image"],"openWeights":null,"details":{"apiId":"google/lyria-3-clip-preview","maxOutputTokens":65536,"knowledgeCutoff":null,"retiresAt":null,"tools":false,"structuredOutput":false,"cacheReadPerMillion":null,"weightsUrl":null},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":0,"outputPerMillion":0,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":8,"timeToFirstToken":2.532,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"lyria3propreview","providerId":"google","slug":"google-lyria-3-pro-preview","name":"Lyria 3 Pro Preview","provider":"Google","description":"Lyria 3 Pro Preview is a model from Google. It has no scored result.","logoUrl":null,"releaseDate":"2026-03-30T21:48:06.000Z","contextWindowTokens":1048576,"modalities":["text","image"],"openWeights":null,"details":{"apiId":"google/lyria-3-pro-preview","maxOutputTokens":65536,"knowledgeCutoff":null,"retiresAt":null,"tools":false,"structuredOutput":false,"cacheReadPerMillion":null,"weightsUrl":null},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":0,"outputPerMillion":0,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":10,"timeToFirstToken":6.025,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"macaw-v1","providerId":"bad-theory-labs","slug":"macaw-v1","name":"Macaw","provider":"Bad Theory Labs","description":"Macaw is a reasoning model from Bad Theory Labs. It has no scored result.","logoUrl":null,"releaseDate":"2026-08-05","contextWindowTokens":128000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":true,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"magistralmediumthinking","providerId":"mistral","slug":"mistral-magistral-medium-thinking","name":"Magistral Medium (Thinking)","provider":"Mistral","description":"Magistral Medium (Thinking) is a model from Mistral. It has 2 scored results in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":34.49472960631833,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"reasoning":2},"benchmarks":[{"benchmarkId":"arc-prize:v1-public-eval:overall","benchmarkName":"ARC-AGI-1 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":7.99,"unit":"percent","family":"arc-prize:v1-public-eval:overall","index":null,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.1059,"latency":null},{"benchmarkId":"arc-prize:v1-semi-private:overall","benchmarkName":"ARC-AGI-1 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":6.12,"unit":"percent","family":"arc-prize:v1-semi-private:overall","index":29.800943810394116,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.0989,"latency":null},{"benchmarkId":"arc-prize:v2-public-eval:overall","benchmarkName":"ARC-AGI-2 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":0,"unit":"percent","family":"arc-prize:v2-public-eval:overall","index":null,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.1242,"latency":null},{"benchmarkId":"arc-prize:v2-semi-private:overall","benchmarkName":"ARC-AGI-2 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":0,"unit":"percent","family":"arc-prize:v2-semi-private:overall","index":39.188515402242544,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.123,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":15,"benchmarkCount":2,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:arc-prize-foundation","name":"ARC Prize Foundation","url":"https://arcprize.org","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"magistralsmall10","providerId":"mistral","slug":"mistral-magistral-small-1-0","name":"Magistral Small 1.0","provider":"Mistral","description":"Magistral Small 1.0 is a model from Mistral. It has 2 scored results in 2 categories. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":28.626356756079897,"multilingual":null,"instruction":null,"math":26.37291224855732},"capabilityCounts":{"knowledge":1,"math":1},"benchmarks":[{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":56.060606060606055,"unit":"percent","family":"gpqa-diamond","index":28.626356756079897,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":3.5360859475294797,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":30,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":26.37291224855732,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":5.924610544247621,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-2025-02-28-public:overall","benchmarkName":"FrontierMath-2025-02-28-Public","category":"math","domain":null,"eligible":false,"ranked":true,"score":0,"unit":"percent","family":"epoch-ai:epoch-frontiermath-2025-02-28-public:overall","index":null,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":25,"benchmarkCount":2,"categoryCount":2,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"magistralsmall12","providerId":"mistral","slug":"mistral-magistral-small-1-2","name":"Magistral Small 1.2","provider":"Mistral","description":"Magistral Small 1.2 is a model from Mistral. It has 2 scored results in 2 categories. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":20.867446501076667,"multilingual":null,"instruction":null,"math":25.241646048520764},"capabilityCounts":{"knowledge":1,"math":1},"benchmarks":[{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":47.6010101010101,"unit":"percent","family":"gpqa-diamond","index":20.867446501076667,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.5993462698453396,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":28.055555555555557,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":25.241646048520764,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":5.56407175551872,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":25,"benchmarkCount":2,"categoryCount":2,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"magnumv472b","providerId":"anthracite-org","slug":"anthracite-org-magnum-v4-72b","name":"Magnum v4 72B","provider":"Anthracite Org","description":"Magnum v4 72B is a model from Anthracite Org. It has no scored result.","logoUrl":null,"releaseDate":"2024-10-22T00:00:00.000Z","contextWindowTokens":32768,"modalities":["text"],"openWeights":true,"details":{"apiId":"anthracite-org/magnum-v4-72b","maxOutputTokens":4096,"knowledgeCutoff":"2024-06-30","retiresAt":null,"tools":false,"structuredOutput":true,"cacheReadPerMillion":2.5,"weightsUrl":"https://huggingface.co/anthracite-org/magnum-v4-72b"},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":2.5,"outputPerMillion":5,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":16,"timeToFirstToken":1.527,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":2.5,"outputPerMillion":5,"cacheReadPerMillion":2.5,"contextTokens":32768,"maxOutputTokens":4096,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"mai-code-1-1-flash","providerId":"microsoft","slug":"mai-code-1-1-flash","name":"MAI-Code-1.1-Flash","provider":"Microsoft","description":"MAI-Code-1.1-Flash is a reasoning model from Microsoft in the MAI-Code family. It has 2 scored results in 2 categories. It has no index.","logoUrl":null,"releaseDate":"2026-08-11","contextWindowTokens":256000,"modalities":["text"],"openWeights":false,"details":null,"reasoning":true,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":57.92852363270531,"coding":54.774607676567676,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"coding":1,"agentic":1},"benchmarks":[{"benchmarkId":"sweVerified","benchmarkName":"Software Engineering Benchmark Verified","category":"coding","score":72.6,"unit":"percent","family":"swe-bench-verified","index":54.774607676567676,"aboveFit":false,"run":null,"source":"Carlos E. Jimenez, John Yang, Alexander Wettig, Shunyu Yao, Kexin Pei, Ofir Press, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2310.06770","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"terminalBench21","benchmarkName":"Terminal-Bench 2.1","category":"agentic","score":62.9,"unit":"percent","family":"terminal-bench-2-1","index":57.92852363270531,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":40,"benchmarkCount":2,"categoryCount":2,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"mai-cyber-1-flash","providerId":"microsoft","slug":"mai-cyber-1-flash","name":"MAI-Cyber-1-Flash","provider":"Microsoft","description":"MAI-Cyber-1-Flash is a reasoning model from Microsoft in the MAI-Cyber family. It has no scored result.","logoUrl":null,"releaseDate":"2026-07-27","contextWindowTokens":256000,"modalities":["text"],"openWeights":false,"details":null,"reasoning":true,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"mair-hub-0-5b-omni","providerId":"mair-hub","slug":"mair-hub-0-5b-omni","name":"Mair-hub 0.5B Omni","provider":"Mair Hub","description":"Mair-hub 0.5B Omni is a standard model from Mair Hub. It has no scored result.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"malkuth-2b","providerId":"newfull5-dhtocks","slug":"malkuth-2b","name":"Malkuth-2B","provider":"newfull5 (dhtocks)","description":"Malkuth-2B is a standard model from newfull5 (dhtocks). It has 2 scored results in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":74.20287104235496,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"reasoning":2},"benchmarks":[{"benchmarkId":"jevBench14","benchmarkName":"jevBench14","category":"reasoning","score":38.930553392688644,"unit":"percent","family":"jevBench14","index":78.8275487843235,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"jevBench15","benchmarkName":"jevBench15","category":"reasoning","score":29.896192877577803,"unit":"percent","family":"jevBench15","index":69.5781933003864,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":15,"benchmarkCount":2,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"malkuth-4b","providerId":"newfull5-dhtocks","slug":"malkuth-4b","name":"Malkuth-4B","provider":"newfull5 (dhtocks)","description":"Malkuth-4B is a standard model from newfull5 (dhtocks). It has 2 scored results in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":87.91419173728843,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"reasoning":2},"benchmarks":[{"benchmarkId":"jevBench15","benchmarkName":"jevBench15","category":"reasoning","score":66.77408647378738,"unit":"percent","family":"jevBench15","index":91.91037398437143,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"jevBench14","benchmarkName":"jevBench14","category":"reasoning","score":44.453846652872855,"unit":"percent","family":"jevBench14","index":83.91800949020542,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":15,"benchmarkCount":2,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"mamaylmgemma29bitv01","providerId":"unknown","slug":"unknown-mamaylm-gemma-2-9b-it-v0-1","name":"MamayLM Gemma 2 9B IT V0.1","provider":"Unknown","description":"MamayLM Gemma 2 9B IT V0.1 is a model. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":36.773409369685986,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":47.57857142857143,"unit":"percent","family":"euroeval","index":36.773409369685986,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.7086520691439776,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"mamaylmgemma34bitv10","providerId":"unknown","slug":"unknown-mamaylm-gemma-3-4b-it-v1-0","name":"MamayLM Gemma 3 4B IT V1.0","provider":"Unknown","description":"MamayLM Gemma 3 4B IT V1.0 is a model. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":29.63873790392883,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":42.32,"unit":"percent","family":"euroeval","index":29.63873790392883,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.672995132953866,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"mammoth-1","providerId":"baseten","slug":"mammoth-1","name":"mammoth-1","provider":"Baseten","description":"mammoth-1 is a model from Baseten. It has no scored result.","logoUrl":null,"releaseDate":"2026-10-21","contextWindowTokens":262000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"manchego-v2-1","providerId":"oraculumai","slug":"manchego-v2-1","name":"Manchego v2.1","provider":"oraculumai","description":"Manchego v2.1 is a standard model from oraculumai. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":93.14182091379308,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"reasoning":1},"benchmarks":[{"benchmarkId":"jevBench15","benchmarkName":"jevBench15","category":"reasoning","score":68.80761689677726,"unit":"percent","family":"jevBench15","index":93.14182091379308,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":15,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"marcominiglobalbase","providerId":"unknown","slug":"unknown-marco-mini-global-base","name":"Marco Mini Global Base","provider":"Unknown","description":"Marco Mini Global Base is a model. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":5,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":4.61625,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.44939716565639354,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-portuguese:overall","benchmarkName":"EuroEval Portuguese","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":4.1450000000000005,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/portuguese","measuredAt":null,"stderr":0.6380108247514301,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-spanish:overall","benchmarkName":"EuroEval Spanish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":4.0287500000000005,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/spanish","measuredAt":null,"stderr":0.8047243316813528,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-italian:overall","benchmarkName":"EuroEval Italian","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":3.8100000000000005,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/italian","measuredAt":null,"stderr":0.594394387170673,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-french:overall","benchmarkName":"EuroEval French","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":3.565,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/french","measuredAt":null,"stderr":0.39119208836580527,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-dutch:overall","benchmarkName":"EuroEval Dutch","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":3.5,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/dutch","measuredAt":null,"stderr":0.5984331393081621,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":1.937142857142857,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.5127596405525613,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":1.6099999999999999,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":1.3512690794212674,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"mctsrefine7b","providerId":"unknown","slug":"unknown-mcts-refine-7b","name":"MCTS Refine 7B","provider":"Unknown","description":"MCTS Refine 7B is a model. It has 2 scored results in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":17.97848543297144,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"coding":2},"benchmarks":[{"benchmarkId":"swebench-experiments:swe-bench-verified:overall","benchmarkName":"SWE-bench Verified","category":"coding","domain":null,"eligible":true,"ranked":true,"score":23.200000000000003,"unit":"percent","family":"swe-bench-verified","index":18.339263334053307,"aboveFit":false,"run":"MCTS-Refine-7B","source":"SWE-bench team","sourceUrl":"https://github.com/SWE-bench/experiments/tree/main/evaluation/verified","measuredAt":"2026-09-01","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"swebench-experiments:swe-bench-lite:overall","benchmarkName":"SWE-bench Lite","category":"coding","domain":null,"eligible":true,"ranked":true,"score":16.333333333333332,"unit":"percent","family":"swe-bench-lite","index":17.617707531889565,"aboveFit":false,"run":"MCTS-Refine-7B","source":"SWE-bench team","sourceUrl":"https://github.com/SWE-bench/experiments/tree/main/evaluation/lite","measuredAt":"2025-09-11","stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":20,"benchmarkCount":2,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:swe-bench-team","name":"SWE-bench team","url":"https://github.com","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"megrez-3b-omni","providerId":"infinigence-ai","slug":"megrez-3b-omni","name":"Megrez-3B-Omni","provider":"Infinigence AI","description":"Megrez-3B-Omni is a standard model from Infinigence AI. It has no scored result.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"mellum212ba25bbase","providerId":"unknown","slug":"unknown-mellum2-12b-a2-5b-base","name":"Mellum2 12B A2.5B Base","provider":"Unknown","description":"Mellum2 12B A2.5B Base is a model. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":8.75804904204297,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":26.93,"unit":"percent","family":"euroeval","index":8.75804904204297,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.6145221720979642,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"mellum212ba25bbasepretrain","providerId":"unknown","slug":"unknown-mellum2-12b-a2-5b-base-pretrain","name":"Mellum2 12B A2.5B Base Pretrain","provider":"Unknown","description":"Mellum2 12B A2.5B Base Pretrain is a model. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":5.279159093449856,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-dutch:overall","benchmarkName":"EuroEval Dutch","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":27.555555555555557,"unit":"percent","family":"euroeval","index":9.606783994699228,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/dutch","measuredAt":null,"stderr":0.9989000122974778,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":21.17625,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.9202258554289812,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"mellum212ba25binstructsft","providerId":"unknown","slug":"unknown-mellum2-12b-a2-5b-instruct-sft","name":"Mellum2 12B A2.5B Instruct SFT","provider":"Unknown","description":"Mellum2 12B A2.5B Instruct SFT is a model. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":17.698408965183113,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-dutch:overall","benchmarkName":"EuroEval Dutch","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":35.10888888888889,"unit":"percent","family":"euroeval","index":19.85491935554682,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/dutch","measuredAt":null,"stderr":0.6039550715037185,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":31.93,"unit":"percent","family":"euroeval","index":15.541898574819406,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.744034609410073,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"mellum212ba25bthinkingsft","providerId":"unknown","slug":"unknown-mellum2-12b-a2-5b-thinking-sft","name":"Mellum2 12B A2.5B Thinking SFT","provider":"Unknown","description":"Mellum2 12B A2.5B Thinking SFT is a model. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":14.859744816245772,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-dutch:overall","benchmarkName":"EuroEval Dutch","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":33.42444444444444,"unit":"percent","family":"euroeval","index":17.569515824060353,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/dutch","measuredAt":null,"stderr":0.8839431857057078,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":29.43,"unit":"percent","family":"euroeval","index":12.149973808431184,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.7996952544563459,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"mellum2-12b-a2-5b-instruct","providerId":"jetbrains","slug":"mellum2-12b-a2-5b-instruct","name":"Mellum2-12B-A2.5B-Instruct","provider":"JetBrains","description":"Mellum2-12B-A2.5B-Instruct is a standard model from JetBrains in the Mellum2 12B-A2.5B family. It has 6 scored results in 5 categories. It has no index.","logoUrl":null,"releaseDate":"2026-05-28","contextWindowTokens":128000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":26.185089032058755,"coding":14.96039698017848,"reasoning":null,"multimodal":null,"knowledge":18.68527426470533,"multilingual":19.060737859549977,"instruction":15.837414206641995,"math":null},"capabilityCounts":{"knowledge":2,"instruction":1,"agentic":1,"coding":1,"multilingual":1},"benchmarks":[{"benchmarkId":"mmluRedux","benchmarkName":"MMLU-Redux","category":"knowledge","score":78.1,"unit":"percent","family":"mmluRedux","index":22.649085387341923,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"ifeval","benchmarkName":"Instruction-Following Eval","category":"instruction","score":75.8,"unit":"percent","family":"ifeval","index":15.837414206641995,"aboveFit":false,"run":null,"source":"Jeffrey Zhou, Tianjian Lu, Swaroop Mishra, Siddhartha Brahma, Sujoy Basu, Yi Luan, Denny Zhou, Le Hou","sourceUrl":"https://arxiv.org/abs/2311.07911","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"bfclV4","benchmarkName":"Berkeley Function Calling Leaderboard v4","category":"agentic","score":44.2,"unit":"percent","family":"bfclV4","index":26.185089032058755,"aboveFit":false,"run":null,"source":"Shishir G. Patil et al.","sourceUrl":"https://gorilla.cs.berkeley.edu/leaderboard.html","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gpqa","benchmarkName":"Graduate-Level Google-Proof Q&A","category":"knowledge","score":40.9,"unit":"percent","family":"gpqa-diamond","index":14.721463142068735,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gpqaDiamond","benchmarkName":"GPQA Diamond","category":"knowledge","score":40.9,"unit":"percent","family":"gpqa-diamond","index":14.721463142068735,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"liveCodeBenchV6","benchmarkName":"LiveCodeBench v6","category":"coding","score":37.2,"unit":"percent","family":"liveCodeBenchV6","index":14.96039698017848,"aboveFit":false,"run":null,"source":"LiveCodeBench maintainers","sourceUrl":"https://github.com/LiveCodeBench/LiveCodeBench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-dutch:overall","benchmarkName":"EuroEval Dutch","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":37.123333333333335,"unit":"percent","family":"euroeval","index":22.588056956196528,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/dutch","measuredAt":null,"stderr":0.579113839850312,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":31.92375,"unit":"percent","family":"euroeval","index":15.53341876290343,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.7120667542442914,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":65,"benchmarkCount":6,"categoryCount":5,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"mellum2-12b-a2-5b-thinking","providerId":"jetbrains","slug":"mellum2-12b-a2-5b-thinking","name":"Mellum2-12B-A2.5B-Thinking","provider":"JetBrains","description":"Mellum2-12B-A2.5B-Thinking is a reasoning model from JetBrains in the Mellum2 12B-A2.5B family. It has 6 scored results in 5 categories. It has no index.","logoUrl":null,"releaseDate":"2026-05-28","contextWindowTokens":128000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":true,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":27.485907448768636,"coding":41.808965688835755,"reasoning":null,"multimodal":null,"knowledge":33.21389419565345,"multilingual":11.85694919666821,"instruction":17.247143424076008,"math":null},"capabilityCounts":{"knowledge":2,"instruction":1,"coding":1,"agentic":1,"multilingual":1},"benchmarks":[{"benchmarkId":"mmluRedux","benchmarkName":"MMLU-Redux","category":"knowledge","score":86.2,"unit":"percent","family":"mmluRedux","index":36.38954157807777,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"ifeval","benchmarkName":"Instruction-Following Eval","category":"instruction","score":76.5,"unit":"percent","family":"ifeval","index":17.247143424076008,"aboveFit":false,"run":null,"source":"Jeffrey Zhou, Tianjian Lu, Swaroop Mishra, Siddhartha Brahma, Sujoy Basu, Yi Luan, Denny Zhou, Le Hou","sourceUrl":"https://arxiv.org/abs/2311.07911","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"liveCodeBenchV6","benchmarkName":"LiveCodeBench v6","category":"coding","score":69.9,"unit":"percent","family":"liveCodeBenchV6","index":41.808965688835755,"aboveFit":false,"run":null,"source":"LiveCodeBench maintainers","sourceUrl":"https://github.com/LiveCodeBench/LiveCodeBench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gpqa","benchmarkName":"Graduate-Level Google-Proof Q&A","category":"knowledge","score":57.6,"unit":"percent","family":"gpqa-diamond","index":30.038246813229147,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gpqaDiamond","benchmarkName":"GPQA Diamond","category":"knowledge","score":57.6,"unit":"percent","family":"gpqa-diamond","index":30.038246813229147,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"bfclV4","benchmarkName":"Berkeley Function Calling Leaderboard v4","category":"agentic","score":45.6,"unit":"percent","family":"bfclV4","index":27.485907448768636,"aboveFit":false,"run":null,"source":"Shishir G. Patil et al.","sourceUrl":"https://gorilla.cs.berkeley.edu/leaderboard.html","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-dutch:overall","benchmarkName":"EuroEval Dutch","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":31.15555555555556,"unit":"percent","family":"euroeval","index":14.491155658298261,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/dutch","measuredAt":null,"stderr":0.9160401899641902,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":27.2725,"unit":"percent","family":"euroeval","index":9.222742735038153,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.9437963564773917,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":65,"benchmarkCount":6,"categoryCount":5,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"mellum2-1-12b-a2-5b-thinking","providerId":"jetbrains","slug":"mellum2-1-12b-a2-5b-thinking","name":"Mellum2.1-12B-A2.5B-Thinking","provider":"JetBrains","description":"Mellum2.1-12B-A2.5B-Thinking is a reasoning model from JetBrains in the Mellum2.1 12B-A2.5B family. It has 7 scored results in 4 categories. It has no index.","logoUrl":null,"releaseDate":"2026-10-08","contextWindowTokens":128000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":true,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":31.31760454918484,"coding":40.10711800022274,"reasoning":null,"multimodal":null,"knowledge":37.78108065078488,"multilingual":null,"instruction":45.64311766096081,"math":null},"capabilityCounts":{"instruction":1,"knowledge":2,"coding":3,"agentic":1},"benchmarks":[{"benchmarkId":"ifeval","benchmarkName":"Instruction-Following Eval","category":"instruction","score":90.6,"unit":"percent","family":"ifeval","index":45.64311766096081,"aboveFit":false,"run":null,"source":"Jeffrey Zhou, Tianjian Lu, Swaroop Mishra, Siddhartha Brahma, Sujoy Basu, Yi Luan, Denny Zhou, Le Hou","sourceUrl":"https://arxiv.org/abs/2311.07911","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmluRedux","benchmarkName":"MMLU-Redux","category":"knowledge","score":87.8,"unit":"percent","family":"mmluRedux","index":39.10370576390212,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"liveCodeBenchV6","benchmarkName":"LiveCodeBench v6","category":"coding","score":82,"unit":"percent","family":"liveCodeBenchV6","index":51.743757168185994,"aboveFit":false,"run":null,"source":"LiveCodeBench maintainers","sourceUrl":"https://github.com/LiveCodeBench/LiveCodeBench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gpqaDiamond","benchmarkName":"GPQA Diamond","category":"knowledge","score":64.6,"unit":"percent","family":"gpqa-diamond","index":36.45845553766764,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"sweVerified","benchmarkName":"Software Engineering Benchmark Verified","category":"coding","score":47,"unit":"percent","family":"swe-bench-verified","index":35.89313368530517,"aboveFit":false,"run":null,"source":"Carlos E. Jimenez, John Yang, Alexander Wettig, Shunyu Yao, Kexin Pei, Ofir Press, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2310.06770","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"swePro","benchmarkName":"SWE-bench Pro","category":"coding","score":28,"unit":"percent","family":"swePro","index":32.684463147177055,"aboveFit":false,"run":null,"source":"Xiang Deng, Jeff Da, Edwin Pan, Yannis Yiming He, Charles Ide, Kanak Garg, Niklas Lauffer, Andrew Park, Nitin Pasari, Chetan Rane, Karmini Sampath, Maya Krishnan, Srivatsa Kundurthy, Sean Hendryx, Zifan Wang, Chen Bo Calvin Zhang, Noah Jacobson, Bing Liu, Brad Kenstler","sourceUrl":"https://arxiv.org/abs/2509.16941","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"terminalBench21","benchmarkName":"Terminal-Bench 2.1","category":"agentic","score":17.4,"unit":"percent","family":"terminal-bench-2-1","index":31.31760454918484,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":60,"benchmarkCount":7,"categoryCount":4,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"meralion-audiollm","providerId":"ai-singapore","slug":"meralion-audiollm","name":"MERaLiON-AudioLLM","provider":"AI Singapore","description":"MERaLiON-AudioLLM is a standard model from AI Singapore. It has no scored result.","logoUrl":null,"releaseDate":"2024-12-13","contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"mercury-2","providerId":"inception","slug":"mercury-2","name":"Mercury 2","provider":"Inception","description":"Mercury 2 is a reasoning model from Inception. It has no scored result.","logoUrl":null,"releaseDate":"2026-02-24","contextWindowTokens":128000,"modalities":["text"],"openWeights":false,"details":{"apiId":"inception/mercury-2","maxOutputTokens":50000,"knowledgeCutoff":null,"retiresAt":"2026-11-08","tools":true,"structuredOutput":true,"cacheReadPerMillion":0.024999999999999998,"weightsUrl":null},"reasoning":true,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":0.25,"outputPerMillion":0.75,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":412,"timeToFirstToken":0.467,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":{"elo":1354.8026721854653,"eloUpper":1365.2242119980176,"eloLower":1344.3811323729133,"votes":3158,"rank":207,"rankUpper":null,"rankLower":null,"arenaName":"Mercury 2","organization":"inception-ai","updatedAt":"2026-11-09"},"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"mercury-2-5-preview","providerId":"inception","slug":"mercury-2-5-preview","name":"Mercury 2.5 Preview","provider":"Inception","description":"Mercury 2.5 Preview is a reasoning model from Inception in the Mercury 2.5 family. It has no scored result.","logoUrl":null,"releaseDate":"2026-08-31","contextWindowTokens":260000,"modalities":["text"],"openWeights":false,"details":{"apiId":"inception/mercury-2.5","maxOutputTokens":65536,"knowledgeCutoff":null,"retiresAt":"2026-11-08","tools":true,"structuredOutput":true,"cacheReadPerMillion":0.004,"weightsUrl":null},"reasoning":true,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":0.04,"outputPerMillion":0.15,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":125,"timeToFirstToken":0.726,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"mercury-decide","providerId":"inception","slug":"mercury-decide","name":"Mercury Decide","provider":"Inception","description":"Mercury Decide is a model from Inception. It has no scored result.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":false,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":0.04,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"mercury-edit-2","providerId":"inception","slug":"mercury-edit-2","name":"Mercury Edit 2","provider":"Inception","description":"Mercury Edit 2 is a standard model from Inception in the Mercury Edit family. It has no scored result.","logoUrl":null,"releaseDate":"2026-03-30","contextWindowTokens":128000,"modalities":["text"],"openWeights":false,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":0.25,"outputPerMillion":0.75,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"metallama31405binstructturbo","providerId":"meta","slug":"meta-meta-llama-3-1-405b-instruct-turbo","name":"Meta Llama 3.1 405B Instruct Turbo","provider":"Meta","description":"Meta Llama 3.1 405B Instruct Turbo is a model from Meta. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":19.57931962387071},"capabilityCounts":{"math":1},"benchmarks":[{"benchmarkId":"vals-ai:medqa:overall","benchmarkName":"MedQA","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":88.242,"unit":"percent","family":"vals-ai:medqa:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medqa","measuredAt":"2026-04-16","stderr":0.576,"costPerTest":0.002624,"latency":8.747},{"benchmarkId":"vals-ai:math500:overall","benchmarkName":"MATH 500","category":"math","domain":null,"eligible":true,"ranked":true,"score":71.4,"unit":"percent","family":"math-500","index":19.57931962387071,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/math500","measuredAt":"2026-01-09","stderr":3.949,"costPerTest":0.002967,"latency":45.304},{"benchmarkId":"vals-ai:tax_eval_v2:overall","benchmarkName":"TaxEval v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":60.875,"unit":"percent","family":"vals-ai:tax_eval_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","measuredAt":"2026-09-01","stderr":0.959,"costPerTest":0.001904,"latency":23.551}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":10,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"metallama3170binstructturbo","providerId":"meta","slug":"meta-meta-llama-3-1-70b-instruct-turbo","name":"Meta Llama 3.1 70B Instruct Turbo","provider":"Meta","description":"Meta Llama 3.1 70B Instruct Turbo is a model from Meta. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":12.81606706431947},"capabilityCounts":{"math":1},"benchmarks":[{"benchmarkId":"vals-ai:medqa:overall","benchmarkName":"MedQA","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":84.784,"unit":"percent","family":"vals-ai:medqa:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medqa","measuredAt":"2026-04-16","stderr":0.642,"costPerTest":0.000618,"latency":4.802},{"benchmarkId":"vals-ai:math500:overall","benchmarkName":"MATH 500","category":"math","domain":null,"eligible":true,"ranked":true,"score":65,"unit":"percent","family":"math-500","index":12.81606706431947,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/math500","measuredAt":"2026-01-09","stderr":4.166,"costPerTest":0.001035,"latency":9.201},{"benchmarkId":"vals-ai:tax_eval_v2:overall","benchmarkName":"TaxEval v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":56.174,"unit":"percent","family":"vals-ai:tax_eval_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","measuredAt":"2026-09-01","stderr":0.973,"costPerTest":0.000488,"latency":4.327},{"benchmarkId":"vals-ai:corp_fin_v2:overall","benchmarkName":"CorpFin v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":38.85,"unit":"percent","family":"vals-ai:corp_fin_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","measuredAt":"2026-08-12","stderr":0.95,"costPerTest":0.053099,"latency":0.062}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":10,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"metallama318binstructturbo","providerId":"meta","slug":"meta-meta-llama-3-1-8b-instruct-turbo","name":"Meta Llama 3.1 8B Instruct Turbo","provider":"Meta","description":"Meta Llama 3.1 8B Instruct Turbo is a model from Meta. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":5},"capabilityCounts":{"math":1},"benchmarks":[{"benchmarkId":"vals-ai:medqa:overall","benchmarkName":"MedQA","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":62.614,"unit":"percent","family":"vals-ai:medqa:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medqa","measuredAt":"2026-04-16","stderr":0.865,"costPerTest":0.000122,"latency":2.372},{"benchmarkId":"vals-ai:math500:overall","benchmarkName":"MATH 500","category":"math","domain":null,"eligible":true,"ranked":true,"score":44.4,"unit":"percent","family":"math-500","index":5,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/math500","measuredAt":"2026-01-09","stderr":4.339,"costPerTest":0.000351,"latency":5.841},{"benchmarkId":"vals-ai:corp_fin_v2:overall","benchmarkName":"CorpFin v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":37.801,"unit":"percent","family":"vals-ai:corp_fin_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","measuredAt":"2026-08-12","stderr":0.934,"costPerTest":0.010777,"latency":0.038},{"benchmarkId":"vals-ai:tax_eval_v2:overall","benchmarkName":"TaxEval v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":32.338,"unit":"percent","family":"vals-ai:tax_eval_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","measuredAt":"2026-09-01","stderr":0.92,"costPerTest":0.000107,"latency":2.458}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":10,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"metask-jev-4b","providerId":"wayfind-metask-ai","slug":"metask-jev-4b","name":"metask-jev-4b","provider":"Wayfind (metask-ai)","description":"metask-jev-4b is a standard model from Wayfind (metask-ai). It has 2 scored results in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":89.66066937047289,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"reasoning":2},"benchmarks":[{"benchmarkId":"jevBench15","benchmarkName":"jevBench15","category":"reasoning","score":67.47568667137665,"unit":"percent","family":"jevBench15","index":92.33524267541006,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"jevBench14","benchmarkName":"jevBench14","category":"reasoning","score":47.78280701270882,"unit":"percent","family":"jevBench14","index":86.98609606553572,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":15,"benchmarkCount":2,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"midmk30pro","providerId":"kt","slug":"kt-mi-dm-k-3-0-pro","name":"Mi:dm K 3.0 Pro","provider":"KT","description":"Mi:dm K 3.0 Pro is a model from KT. It has no scored result.","logoUrl":null,"releaseDate":"2026-11-05T16:55:01.000Z","contextWindowTokens":261795,"modalities":["text"],"openWeights":null,"details":{"apiId":"kt/midm-k-3.0-pro","maxOutputTokens":235615,"knowledgeCutoff":null,"retiresAt":null,"tools":false,"structuredOutput":true,"cacheReadPerMillion":0.04,"weightsUrl":null},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":0.19999999999999998,"outputPerMillion":0.7999999999999999,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":267,"timeToFirstToken":0.247,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"mica-v0-1-4b","providerId":"sky7350","slug":"mica-v0-1-4b","name":"Mica v0.1 4B","provider":"sky7350","description":"Mica v0.1 4B is a standard model from sky7350. It has no scored result.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"microsoft-decision-1","providerId":"microsoft","slug":"microsoft-decision-1","name":"Microsoft-Decision-1","provider":"Microsoft","description":"Microsoft-Decision-1 is a model from Microsoft. It has no scored result.","logoUrl":null,"releaseDate":"2026-10-12","contextWindowTokens":null,"modalities":["text"],"openWeights":false,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":0.042,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"mimov26proultraspeed","providerId":"xiaomi","slug":"xiaomi-mimo-v2-6-pro-ultraspeed","name":"MiMo V2.6 Pro UltraSpeed","provider":"Xiaomi","description":"MiMo V2.6 Pro UltraSpeed is a model from Xiaomi. It has no scored result.","logoUrl":null,"releaseDate":"2026-09-21T20:07:47.000Z","contextWindowTokens":1048576,"modalities":["text","image","video","audio"],"openWeights":null,"details":{"apiId":"xiaomi/mimo-v2.6-pro-ultraspeed","maxOutputTokens":131072,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.036,"weightsUrl":null},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":4.35,"outputPerMillion":8.7,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":105,"timeToFirstToken":3.694,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":4.35,"outputPerMillion":8.7,"cacheReadPerMillion":0.036,"contextTokens":1048576,"maxOutputTokens":131072,"deprecated":false},{"host":"xiaomi","name":"Xiaomi","kind":"maker","url":"https://platform.xiaomimimo.com/#/docs","inputPerMillion":4.35,"outputPerMillion":8.7,"cacheReadPerMillion":0.036,"contextTokens":1048576,"maxOutputTokens":131072,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"ming-3-0-asr-tiny","providerId":"inclusionai","slug":"ming-3-0-asr-tiny","name":"Ming-3.0-asr-tiny","provider":"InclusionAI","description":"Ming-3.0-asr-tiny is a standard model from InclusionAI. It has no scored result.","logoUrl":null,"releaseDate":"2026-10-14","contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"ming-3-0-audio-flash","providerId":"inclusionai","slug":"ming-3-0-audio-flash","name":"Ming-3.0-audio-flash","provider":"InclusionAI","description":"Ming-3.0-audio-flash is a standard model from InclusionAI. It has no scored result.","logoUrl":null,"releaseDate":"2026-10-14","contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"ming-3-0-audio-tiny","providerId":"inclusionai","slug":"ming-3-0-audio-tiny","name":"Ming-3.0-audio-tiny","provider":"InclusionAI","description":"Ming-3.0-audio-tiny is a standard model from InclusionAI. It has no scored result.","logoUrl":null,"releaseDate":"2026-10-14","contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"mini-omni","providerId":"gpt-omni","slug":"mini-omni","name":"Mini-Omni 0.5B","provider":"gpt-omni","description":"Mini-Omni 0.5B is a standard model from gpt-omni. It has no scored result.","logoUrl":null,"releaseDate":"2024-08-29","contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"mini-omni2","providerId":"gpt-omni","slug":"mini-omni2","name":"Mini-Omni2","provider":"gpt-omni","description":"Mini-Omni2 is a standard model from gpt-omni. It has no scored result.","logoUrl":null,"releaseDate":"2024-10-15","contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"minicpm-o-2-6","providerId":"openbmb","slug":"minicpm-o-2-6","name":"MiniCPM-o 2.6","provider":"OpenBMB","description":"MiniCPM-o 2.6 is a standard model from OpenBMB. It has no scored result.","logoUrl":null,"releaseDate":"2025-01-24","contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"minimax01","providerId":"minimax","slug":"minimax-minimax-01","name":"MiniMax 01","provider":"MiniMax","description":"MiniMax 01 is a model from MiniMax. It has no scored result.","logoUrl":null,"releaseDate":"2025-01-15T04:31:02.000Z","contextWindowTokens":1000192,"modalities":["text","image"],"openWeights":true,"details":{"apiId":"minimax/minimax-01","maxOutputTokens":900172,"knowledgeCutoff":"2024-03-31","retiresAt":"2026-11-05","tools":false,"structuredOutput":false,"cacheReadPerMillion":null,"weightsUrl":"https://huggingface.co/MiniMaxAI/MiniMax-Text-01"},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":0.19999999999999998,"outputPerMillion":1.1,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":35,"timeToFirstToken":0.91,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"minimax25","providerId":"minimax","slug":"minimax-minimax-2-5","name":"MiniMax 2.5","provider":"MiniMax","description":"MiniMax 2.5 is a model from MiniMax. It has no scored result.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[{"benchmarkId":"swebench-experiments:swe-bench-multilingual:overall","benchmarkName":"SWE-bench Multilingual","category":"coding","domain":null,"eligible":true,"ranked":true,"score":68.3,"unit":"percent","family":"swe-bench-multilingual","index":null,"aboveFit":false,"run":"mini-SWE-agent","source":"SWE-bench team","sourceUrl":"https://github.com/SWE-bench/experiments/tree/main/evaluation/multilingual","measuredAt":"2026-09-02","stderr":null,"costPerTest":0.10121049956228956,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:swe-bench-team","name":"SWE-bench team","url":"https://github.com","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"minimaxm1","providerId":"minimax","slug":"minimax-minimax-m1","name":"MiniMax M1","provider":"MiniMax","description":"MiniMax M1 is a model from MiniMax. It has no scored result.","logoUrl":null,"releaseDate":"2025-06-17T22:46:54.000Z","contextWindowTokens":1000000,"modalities":["text"],"openWeights":null,"details":{"apiId":"minimax/minimax-m1","maxOutputTokens":40000,"knowledgeCutoff":"2024-06-30","retiresAt":null,"tools":true,"structuredOutput":false,"cacheReadPerMillion":null,"weightsUrl":null},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":0.55,"outputPerMillion":2.2,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":44,"timeToFirstToken":1.8755,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":{"elo":1342.5812187542356,"eloUpper":1346.8553911391505,"eloLower":1338.3070463693207,"votes":34242,"rank":215,"rankUpper":null,"rankLower":null,"arenaName":"MiniMax-M1","organization":"minimax","updatedAt":"2026-11-09"},"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"minimax-m1-80k","providerId":"minimax","slug":"minimax-m1-80k","name":"MiniMax M1 80k","provider":"MiniMax","description":"MiniMax M1 80k is a standard model from MiniMax. It has 6 scored results in 3 categories. It has no index.","logoUrl":null,"releaseDate":"2025-06-16","contextWindowTokens":80000,"modalities":["text"],"openWeights":false,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":42.90404913701761,"reasoning":53.26413846881225,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":43.32714462629887},"capabilityCounts":{"math":3,"coding":2,"reasoning":1},"benchmarks":[{"benchmarkId":"math500","benchmarkName":"MATH-500 Problem Set","category":"math","score":96.8,"unit":"percent","family":"math-500","index":46.420978219589664,"aboveFit":false,"run":null,"source":"Dan Hendrycks, Collin Burns, Saurav Kadavath, Akul Arora, Steven Basart, Eric Tang, Dawn Song, Jacob Steinhardt","sourceUrl":"https://arxiv.org/abs/2103.03874","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aime2024","benchmarkName":"American Invitational Mathematics Examination 2024","category":"math","score":86,"unit":"percent","family":"aime2024","index":42.825308623365494,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aime2025","benchmarkName":"American Invitational Mathematics Examination 2025","category":"math","score":76.9,"unit":"percent","family":"aime2025","index":40.73514703594144,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"liveCodeBench","benchmarkName":"LiveCodeBench: Holistic and Contamination Free Evaluation of Large Language Models for Code","category":"coding","score":65,"unit":"percent","family":"livecodebench","index":43.27694638867682,"aboveFit":false,"run":null,"source":"Naman Jain, King Han, Alex Gu, Wen-Ding Li, Fanjia Yan, Tianjun Zhang, Sida Wang, Armando Solar-Lezama, Koushik Sen, Ion Stoica","sourceUrl":"https://arxiv.org/abs/2403.07974","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"longBenchV2","benchmarkName":"LongBench v2","category":"reasoning","score":61.5,"unit":"percent","family":"longBenchV2","index":53.26413846881225,"aboveFit":false,"run":null,"source":"LongBench v2 authors","sourceUrl":"https://arxiv.org/abs/2412.15204","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"sweVerified","benchmarkName":"Software Engineering Benchmark Verified","category":"coding","score":56,"unit":"percent","family":"swe-bench-verified","index":42.5311518853584,"aboveFit":false,"run":null,"source":"Carlos E. Jimenez, John Yang, Alexander Wettig, Shunyu Yao, Kexin Pei, Ofir Press, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2310.06770","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":71,"timeToFirstToken":29.98,"source":"BenchLM","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":45,"benchmarkCount":6,"categoryCount":3,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"minimaxm2","providerId":"minimax","slug":"minimax-minimax-m2","name":"MiniMax M2","provider":"MiniMax","description":"MiniMax M2 is a model from MiniMax. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":"2025-10-23T20:41:33.000Z","contextWindowTokens":204800,"modalities":["text"],"openWeights":true,"details":{"apiId":"minimax/minimax-m2","maxOutputTokens":131072,"knowledgeCutoff":null,"retiresAt":"2026-11-05","tools":true,"structuredOutput":false,"cacheReadPerMillion":null,"weightsUrl":"https://huggingface.co/MiniMaxAI/MiniMax-M2"},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":46.218939774276855,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"coding":1},"benchmarks":[{"benchmarkId":"swe-bench:swe-bench-verified:overall","benchmarkName":"SWE-bench Verified","category":"coding","domain":null,"eligible":true,"ranked":true,"score":61,"unit":"percent","family":"swe-bench-verified","index":46.218939774276855,"aboveFit":false,"run":"mini-SWE-agent","source":"SWE-bench team","sourceUrl":"https://www.swebench.com/","measuredAt":"2026-02-26","stderr":null,"costPerTest":0.42825617729,"latency":null},{"benchmarkId":"swebench-experiments:swe-bench-verified:overall","benchmarkName":"SWE-bench Verified","category":"coding","domain":null,"eligible":true,"ranked":true,"score":61,"unit":"percent","family":"swe-bench-verified","index":46.218939774276855,"aboveFit":false,"run":"mini-SWE-agent","source":"SWE-bench team","sourceUrl":"https://github.com/SWE-bench/experiments/tree/main/evaluation/verified","measuredAt":"2026-09-01","stderr":null,"costPerTest":0.42825617729,"latency":null}],"price":{"inputPerMillion":0.255,"outputPerMillion":1.02,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":26,"timeToFirstToken":1.038,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":{"elo":1340.2714662690128,"eloUpper":1347.9633390050842,"eloLower":1332.5795935329415,"votes":6832,"rank":217,"rankUpper":null,"rankLower":null,"arenaName":"MiniMax-M2","organization":"minimax","updatedAt":"2026-11-09"},"quality":{"coveragePercent":20,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:swe-bench-team","name":"SWE-bench team","url":"https://www.swebench.com","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"minimaxm2her","providerId":"minimax","slug":"minimax-minimax-m2-her","name":"MiniMax M2-her","provider":"MiniMax","description":"MiniMax M2-her is a model from MiniMax. It has no scored result.","logoUrl":null,"releaseDate":"2026-01-23T14:07:19.000Z","contextWindowTokens":65536,"modalities":["text"],"openWeights":null,"details":{"apiId":"minimax/minimax-m2-her","maxOutputTokens":2048,"knowledgeCutoff":null,"retiresAt":"2026-11-05","tools":false,"structuredOutput":false,"cacheReadPerMillion":0.03,"weightsUrl":null},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":0.3,"outputPerMillion":1.2,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":50,"timeToFirstToken":0.658,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"minimax-m3-1-flash-preview","providerId":"minimax","slug":"minimax-m3-1-flash-preview","name":"MiniMax M3.1 Flash Preview","provider":"MiniMax","description":"MiniMax M3.1 Flash Preview is a reasoning model from MiniMax in the MiniMax M3.1 Flash family. It has no scored result.","logoUrl":null,"releaseDate":"2026-09-27","contextWindowTokens":1048576,"modalities":["text","image","video"],"openWeights":true,"details":{"apiId":"minimax/minimax-m3.1-flash","maxOutputTokens":524288,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.015,"weightsUrl":"https://huggingface.co/MiniMaxAI/MiniMax-M3.1-Flash"},"reasoning":true,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":0.15,"outputPerMillion":0.6,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":99.5,"timeToFirstToken":1.371,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"ministral-3-14b","providerId":"mistral","slug":"ministral-3-14b","name":"Ministral 3 14B","provider":"Mistral","description":"Ministral 3 14B is a standard model from Mistral. It has no scored result.","logoUrl":null,"releaseDate":"2025-12-02","contextWindowTokens":262144,"modalities":["text","image"],"openWeights":true,"details":{"apiId":"mistralai/ministral-14b-2512","maxOutputTokens":209715,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.02,"weightsUrl":"https://huggingface.co/mistralai/Ministral-3-14B-Instruct-2512"},"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":0.19999999999999998,"outputPerMillion":0.19999999999999998,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":40,"timeToFirstToken":1.1595,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":0.2,"outputPerMillion":0.2,"cacheReadPerMillion":0.02,"contextTokens":262144,"maxOutputTokens":209715,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"ministral-3-14b-reasoning","providerId":"mistral","slug":"ministral-3-14b-reasoning","name":"Ministral 3 14B (Reasoning)","provider":"Mistral","description":"Ministral 3 14B (Reasoning) is a reasoning model from Mistral in the Ministral 3 14B family. It has 4 scored results in 3 categories. It has no index.","logoUrl":null,"releaseDate":"2025-12-02","contextWindowTokens":128000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":true,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":42.51179519213823,"multilingual":36.246207348853076,"instruction":null,"math":44.85999251720233},"capabilityCounts":{"math":2,"knowledge":1,"multilingual":1},"benchmarks":[{"benchmarkId":"aime2024","benchmarkName":"American Invitational Mathematics Examination 2024","category":"math","score":89.8,"unit":"percent","family":"aime2024","index":44.5895688861137,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aime2025","benchmarkName":"American Invitational Mathematics Examination 2025","category":"math","score":85,"unit":"percent","family":"aime2025","index":45.13041614829095,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gpqa","benchmarkName":"Graduate-Level Google-Proof Q&A","category":"knowledge","score":71.2,"unit":"percent","family":"gpqa-diamond","index":42.51179519213823,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-french:overall","benchmarkName":"EuroEval French","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":57.238749999999996,"unit":"percent","family":"euroeval","index":49.8800489473505,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/french","measuredAt":null,"stderr":0.629769302999122,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-dutch:overall","benchmarkName":"EuroEval Dutch","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":50.74,"unit":"percent","family":"euroeval","index":41.06274051712435,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/dutch","measuredAt":null,"stderr":0.6187622052668268,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":49.81375,"unit":"percent","family":"euroeval","index":39.80603239117751,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":0.9199787701354852,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-italian:overall","benchmarkName":"EuroEval Italian","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":47.8225,"unit":"percent","family":"euroeval","index":37.10436431474929,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/italian","measuredAt":null,"stderr":0.46921976460929266,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-portuguese:overall","benchmarkName":"EuroEval Portuguese","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":46.557500000000005,"unit":"percent","family":"euroeval","index":35.38805038295686,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/portuguese","measuredAt":null,"stderr":0.5685810188530743,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":46.327142857142846,"unit":"percent","family":"euroeval","index":35.07550874376821,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.7574446837753843,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":45.058749999999996,"unit":"percent","family":"euroeval","index":33.354591485507115,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.7778927866357934,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-spanish:overall","benchmarkName":"EuroEval Spanish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":40.364999999999995,"unit":"percent","family":"euroeval","index":26.98625273661324,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/spanish","measuredAt":null,"stderr":0.4980932392634937,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":0.2,"outputPerMillion":0.2,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":30,"benchmarkCount":4,"categoryCount":3,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"ministral314bbase2512","providerId":"mistral","slug":"mistral-ministral-3-14b-base","name":"Ministral 3 14B Base","provider":"Mistral","description":"Ministral 3 14B Base is a model from Mistral. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":46.53504580689733,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-french:overall","benchmarkName":"EuroEval French","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":59.683749999999996,"unit":"percent","family":"euroeval","index":53.19735136887818,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/french","measuredAt":null,"stderr":0.4788283225750123,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-portuguese:overall","benchmarkName":"EuroEval Portuguese","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":57.44625,"unit":"percent","family":"euroeval","index":50.16157870296073,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/portuguese","measuredAt":null,"stderr":0.6154368367265645,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-italian:overall","benchmarkName":"EuroEval Italian","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":56.17500000000001,"unit":"percent","family":"euroeval","index":48.43678495925234,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/italian","measuredAt":null,"stderr":0.6724000855145693,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-dutch:overall","benchmarkName":"EuroEval Dutch","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":55.49666666666666,"unit":"percent","family":"euroeval","index":47.51644270597232,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/dutch","measuredAt":null,"stderr":0.604037853445512,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":54.050000000000004,"unit":"percent","family":"euroeval","index":45.55364890782235,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.7110770789758998,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":53.81625,"unit":"percent","family":"euroeval","index":45.23650394216504,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":0.7422737449890034,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":50.1375,"unit":"percent","family":"euroeval","index":40.24528664842479,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.7707078726729084,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-spanish:overall","benchmarkName":"EuroEval Spanish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":44.48125,"unit":"percent","family":"euroeval","index":32.571056864471444,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/spanish","measuredAt":null,"stderr":0.5183643144546121,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"ministral314binstruct2512","providerId":"mistral","slug":"mistral-ministral-3-14b-instruct","name":"Ministral 3 14B Instruct","provider":"Mistral","description":"Ministral 3 14B Instruct is a model from Mistral. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":39.00844551039536,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-french:overall","benchmarkName":"EuroEval French","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":58.51875,"unit":"percent","family":"euroeval","index":51.61671442774127,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/french","measuredAt":null,"stderr":0.4848920627314908,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-dutch:overall","benchmarkName":"EuroEval Dutch","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":52.55555555555556,"unit":"percent","family":"euroeval","index":43.52603165858139,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/dutch","measuredAt":null,"stderr":0.4681326181202803,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":50.38250000000001,"unit":"percent","family":"euroeval","index":40.577695275530836,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":0.874646357106688,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-portuguese:overall","benchmarkName":"EuroEval Portuguese","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":49.397499999999994,"unit":"percent","family":"euroeval","index":39.24127691757386,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/portuguese","measuredAt":null,"stderr":0.5534465873054056,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":49.05428571428571,"unit":"percent","family":"euroeval","index":38.77561410321685,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.6555259611291422,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-italian:overall","benchmarkName":"EuroEval Italian","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":47.996249999999996,"unit":"percent","family":"euroeval","index":37.34010308601327,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/italian","measuredAt":null,"stderr":0.44135444939413493,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":46.475,"unit":"percent","family":"euroeval","index":35.27611686566605,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.5865351119072072,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-spanish:overall","benchmarkName":"EuroEval Spanish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":42.52,"unit":"percent","family":"euroeval","index":29.91009188523989,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/spanish","measuredAt":null,"stderr":0.5167869483646041,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"ministral-3-3b","providerId":"mistral","slug":"ministral-3-3b","name":"Ministral 3 3B","provider":"Mistral","description":"Ministral 3 3B is a standard model from Mistral. It has no scored result.","logoUrl":null,"releaseDate":"2025-12-02","contextWindowTokens":131072,"modalities":["text","image"],"openWeights":true,"details":{"apiId":"mistralai/ministral-3b-2512","maxOutputTokens":104857,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.01,"weightsUrl":"https://huggingface.co/mistralai/Ministral-3-3B-Instruct-2512"},"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":0.09999999999999999,"outputPerMillion":0.09999999999999999,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":147,"timeToFirstToken":0.335,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":0.1,"outputPerMillion":0.1,"cacheReadPerMillion":0.01,"contextTokens":131072,"maxOutputTokens":104857,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"ministral-3-3b-reasoning","providerId":"mistral","slug":"ministral-3-3b-reasoning","name":"Ministral 3 3B (Reasoning)","provider":"Mistral","description":"Ministral 3 3B (Reasoning) is a reasoning model from Mistral in the Ministral 3 3B family. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":"2025-12-02","contextWindowTokens":128000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":true,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":26.429977074925574,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-french:overall","benchmarkName":"EuroEval French","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":48.67875,"unit":"percent","family":"euroeval","index":38.26609854723726,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/french","measuredAt":null,"stderr":0.4661377210653521,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-portuguese:overall","benchmarkName":"EuroEval Portuguese","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":43.603750000000005,"unit":"percent","family":"euroeval","index":31.380491271469186,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/portuguese","measuredAt":null,"stderr":0.3871732493341966,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-dutch:overall","benchmarkName":"EuroEval Dutch","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":41.516666666666666,"unit":"percent","family":"euroeval","index":28.548799412329416,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/dutch","measuredAt":null,"stderr":0.3929329412206315,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-italian:overall","benchmarkName":"EuroEval Italian","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":40.667500000000004,"unit":"percent","family":"euroeval","index":27.396675633346224,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/italian","measuredAt":null,"stderr":0.4233239008135496,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":39.24249999999999,"unit":"percent","family":"euroeval","index":25.46327851650493,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":0.34838018026288464,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":38.707142857142856,"unit":"percent","family":"euroeval","index":24.736922055816944,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.29063617764329003,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-spanish:overall","benchmarkName":"EuroEval Spanish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":37.33999999999999,"unit":"percent","family":"euroeval","index":22.88202376928349,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/spanish","measuredAt":null,"stderr":0.3668893736536942,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":34.486250000000005,"unit":"percent","family":"euroeval","index":19.010141648451363,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.6124693869900764,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":0.1,"outputPerMillion":0.1,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"ministral33binstruct2512","providerId":"mistral","slug":"mistral-ministral-3-3b-instruct","name":"Ministral 3 3B Instruct","provider":"Mistral","description":"Ministral 3 3B Instruct is a model from Mistral. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":29.333841555483495,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-french:overall","benchmarkName":"EuroEval French","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":48.497499999999995,"unit":"percent","family":"euroeval","index":38.02018400167411,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/french","measuredAt":null,"stderr":0.46770717334674267,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-portuguese:overall","benchmarkName":"EuroEval Portuguese","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":44.7125,"unit":"percent","family":"euroeval","index":32.88480990536235,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/portuguese","measuredAt":null,"stderr":0.3730365263616956,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-italian:overall","benchmarkName":"EuroEval Italian","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":43.725,"unit":"percent","family":"euroeval","index":31.544999622639008,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/italian","measuredAt":null,"stderr":0.31170649255349175,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-dutch:overall","benchmarkName":"EuroEval Dutch","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":40.465555555555554,"unit":"percent","family":"euroeval","index":27.12268348832797,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/dutch","measuredAt":null,"stderr":0.44190804026333663,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":39.215714285714284,"unit":"percent","family":"euroeval","index":25.426936465436487,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.5973307974921879,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":33.74125,"unit":"percent","family":"euroeval","index":17.99934806806767,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.6665852814906731,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"ministral-3-8b","providerId":"mistral","slug":"ministral-3-8b","name":"Ministral 3 8B","provider":"Mistral","description":"Ministral 3 8B is a standard model from Mistral. It has no scored result.","logoUrl":null,"releaseDate":"2025-12-02","contextWindowTokens":262144,"modalities":["text","image"],"openWeights":true,"details":{"apiId":"mistralai/ministral-8b-2512","maxOutputTokens":209715,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.015,"weightsUrl":"https://huggingface.co/mistralai/Ministral-3-8B-Instruct-2512"},"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":0.15,"outputPerMillion":0.15,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":64,"timeToFirstToken":0.566,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":0.15,"outputPerMillion":0.15,"cacheReadPerMillion":0.015,"contextTokens":262144,"maxOutputTokens":209715,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"ministral-3-8b-reasoning","providerId":"mistral","slug":"ministral-3-8b-reasoning","name":"Ministral 3 8B (Reasoning)","provider":"Mistral","description":"Ministral 3 8B (Reasoning) is a reasoning model from Mistral in the Ministral 3 8B family. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":"2025-12-02","contextWindowTokens":128000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":true,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":40.29531753872902,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-french:overall","benchmarkName":"EuroEval French","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":59.169999999999995,"unit":"percent","family":"euroeval","index":52.5003108293854,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/french","measuredAt":null,"stderr":0.3971952448104081,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-portuguese:overall","benchmarkName":"EuroEval Portuguese","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":53.11625000000001,"unit":"percent","family":"euroeval","index":44.28676500757635,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/portuguese","measuredAt":null,"stderr":0.3420046600559706,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-dutch:overall","benchmarkName":"EuroEval Dutch","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":50.68444444444444,"unit":"percent","family":"euroeval","index":40.987364411204595,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/dutch","measuredAt":null,"stderr":0.3497653534430416,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":50.42625000000001,"unit":"percent","family":"euroeval","index":40.63705395894264,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":0.3296778730821952,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-italian:overall","benchmarkName":"EuroEval Italian","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":49.92250000000001,"unit":"percent","family":"euroeval","index":39.9535811185154,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/italian","measuredAt":null,"stderr":0.4372749421130829,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-spanish:overall","benchmarkName":"EuroEval Spanish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":46.36625,"unit":"percent","family":"euroeval","index":35.12856813832816,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/spanish","measuredAt":null,"stderr":0.3835382151754894,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":44.892857142857146,"unit":"percent","family":"euroeval","index":33.12951304922322,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.3358358442538769,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":44.325,"unit":"percent","family":"euroeval","index":32.35906156657218,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.5793342083806203,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":0.15,"outputPerMillion":0.15,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"ministral38bbase2512","providerId":"mistral","slug":"mistral-ministral-3-8b-base","name":"Ministral 3 8B Base","provider":"Mistral","description":"Ministral 3 8B Base is a model from Mistral. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":40.336007175906744,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-french:overall","benchmarkName":"EuroEval French","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":57.31125,"unit":"percent","family":"euroeval","index":49.97841476557577,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/french","measuredAt":null,"stderr":0.5845724933658784,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-portuguese:overall","benchmarkName":"EuroEval Portuguese","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":53.743750000000006,"unit":"percent","family":"euroeval","index":45.13813812393979,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/portuguese","measuredAt":null,"stderr":0.7530147740914517,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-italian:overall","benchmarkName":"EuroEval Italian","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":50.52875,"unit":"percent","family":"euroeval","index":40.77612287436454,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/italian","measuredAt":null,"stderr":0.8120662303654794,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":50.26428571428572,"unit":"percent","family":"euroeval","index":40.417305690148766,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.6198419811796011,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-dutch:overall","benchmarkName":"EuroEval Dutch","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":50.14444444444444,"unit":"percent","family":"euroeval","index":40.25470866166474,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/dutch","measuredAt":null,"stderr":0.6588607826120761,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":48.116249999999994,"unit":"percent","family":"euroeval","index":37.502915474799906,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":0.8130056119117506,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":45.925,"unit":"percent","family":"euroeval","index":34.529893417060634,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.8419796241596349,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-spanish:overall","benchmarkName":"EuroEval Spanish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":40.188750000000006,"unit":"percent","family":"euroeval","index":26.747122040582884,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/spanish","measuredAt":null,"stderr":0.5877366544635446,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"ministral38binstruct2512","providerId":"mistral","slug":"mistral-ministral-3-8b-instruct","name":"Ministral 3 8B Instruct","provider":"Mistral","description":"Ministral 3 8B Instruct is a model from Mistral. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":34.08446101113173,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-french:overall","benchmarkName":"EuroEval French","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":56.88625,"unit":"percent","family":"euroeval","index":49.40178755528977,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/french","measuredAt":null,"stderr":0.5063518416081845,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":49.87500000000001,"unit":"percent","family":"euroeval","index":39.88913454795403,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":0.7926044883799234,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-dutch:overall","benchmarkName":"EuroEval Dutch","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":48.5711111111111,"unit":"percent","family":"euroeval","index":38.12005734201775,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/dutch","measuredAt":null,"stderr":0.5895530845531832,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":46.037142857142854,"unit":"percent","family":"euroeval","index":34.68204547086719,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.6360929851163946,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-portuguese:overall","benchmarkName":"EuroEval Portuguese","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":45.15625,"unit":"percent","family":"euroeval","index":33.486876551396264,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/portuguese","measuredAt":null,"stderr":0.5177037642320171,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-italian:overall","benchmarkName":"EuroEval Italian","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":44.22625,"unit":"percent","family":"euroeval","index":32.22508053829985,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/italian","measuredAt":null,"stderr":0.6450096898496952,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":43.29375000000001,"unit":"percent","family":"euroeval","index":30.959892600437055,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.6454007960174825,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-spanish:overall","benchmarkName":"EuroEval Spanish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":39.9975,"unit":"percent","family":"euroeval","index":26.48763979595418,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/spanish","measuredAt":null,"stderr":0.45719764599131524,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"ministral3b","providerId":"mistral","slug":"mistral-ministral-3b","name":"Ministral 3B","provider":"Mistral","description":"Ministral 3B is a model from Mistral. It has 2 scored results in 2 categories. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":5,"multilingual":null,"instruction":null,"math":6.014581487914775},"capabilityCounts":{"knowledge":1,"math":1},"benchmarks":[{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":25.252525252525253,"unit":"percent","family":"gpqa-diamond","index":5,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.9305614722797293,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-math-level-5:overall","benchmarkName":"MATH level 5","category":"math","domain":null,"eligible":true,"ranked":true,"score":14.444864048338369,"unit":"percent","family":"epoch-ai:epoch-math-level-5:overall","index":6.014581487914775,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0.7446287617112978,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":25,"benchmarkCount":2,"categoryCount":2,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"ministral8b","providerId":"mistral","slug":"mistral-ministral-8b","name":"Ministral 8B","provider":"Mistral","description":"Ministral 8B is a model from Mistral. It has 2 scored results in 2 categories. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":5,"multilingual":null,"instruction":null,"math":6.253056560292087},"capabilityCounts":{"knowledge":1,"math":1},"benchmarks":[{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":27.146464646464647,"unit":"percent","family":"gpqa-diamond","index":5,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.0863706671553732,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-math-level-5:overall","benchmarkName":"MATH level 5","category":"math","domain":null,"eligible":true,"ranked":true,"score":14.935800604229607,"unit":"percent","family":"epoch-ai:epoch-math-level-5:overall","index":6.253056560292087,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0.7454534670584494,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":{"elo":1190.9257950419073,"eloUpper":1199.896844124978,"eloLower":1181.954745958837,"votes":4781,"rank":329,"rankUpper":null,"rankLower":null,"arenaName":"Ministral 8B-2410","organization":"mistral","updatedAt":"2026-11-09"},"quality":{"coveragePercent":25,"benchmarkCount":2,"categoryCount":2,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"mirror","providerId":"bluusun","slug":"mirror","name":"Mirror","provider":"Bluusun","description":"Mirror is a standard model from Bluusun. It has 2 scored results in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":48.249875125864754,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"reasoning":2},"benchmarks":[{"benchmarkId":"jevBench14","benchmarkName":"jevBench14","category":"reasoning","score":2.110814199749038,"unit":"percent","family":"jevBench14","index":44.893186306487486,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"jevBench15","benchmarkName":"jevBench15","category":"reasoning","score":0.21902679657747523,"unit":"percent","family":"jevBench15","index":51.606563945242016,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":15,"benchmarkCount":2,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"mistral7b","providerId":"mistral","slug":"mistral-mistral-7b","name":"Mistral 7B","provider":"Mistral","description":"Mistral 7B is a model from Mistral. It has 3 scored results in 2 categories. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":5,"multilingual":null,"instruction":null,"math":7.040350166856406},"capabilityCounts":{"knowledge":1,"math":2},"benchmarks":[{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":14.204545454545457,"unit":"percent","family":"gpqa-diamond","index":5,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.0148891037227303,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-math-level-5:overall","benchmarkName":"MATH level 5","category":"math","domain":null,"eligible":true,"ranked":true,"score":3.6395392749244713,"unit":"percent","family":"epoch-ai:epoch-math-level-5:overall","index":5,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0.2666943446495743,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":0.2777777777777778,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":9.080700333712812,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0.2777777777777778,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":25,"benchmarkCount":3,"categoryCount":2,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"mistral7bv01","providerId":"mistral","slug":"mistral-mistral-7b-v0-1","name":"Mistral 7B V0.1","provider":"Mistral","description":"Mistral 7B V0.1 is a model from Mistral. It has no scored result.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[{"benchmarkId":"vals-ai:legal_bench:overall","benchmarkName":"LegalBench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":53.77,"unit":"percent","family":"vals-ai:legal_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","measuredAt":"2026-10-13","stderr":0.779,"costPerTest":null,"latency":1.715}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"mistral-7b-v0-3","providerId":"mistral","slug":"mistral-7b-v0-3","name":"Mistral 7B v0.3","provider":"Mistral","description":"Mistral 7B v0.3 is a standard model from Mistral in the Mistral 7B family. It has no scored result.","logoUrl":null,"releaseDate":"2024-05-22","contextWindowTokens":32000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"mistral-8x7b","providerId":"mistral","slug":"mistral-8x7b","name":"Mistral 8x7B","provider":"Mistral","description":"Mistral 8x7B is a standard model from Mistral. It has no scored result.","logoUrl":null,"releaseDate":"2023-12-11","contextWindowTokens":32000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"mistral-8x7b-v0-2","providerId":"mistral","slug":"mistral-8x7b-v0-2","name":"Mistral 8x7B v0.2","provider":"Mistral","description":"Mistral 8x7B v0.2 is a standard model from Mistral in the Mistral 8x7B family. It has no scored result.","logoUrl":null,"releaseDate":"2023-12-11","contextWindowTokens":32000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"mistrallarge","providerId":"mistral","slug":"mistral-mistral-large","name":"Mistral Large","provider":"Mistral","description":"Mistral Large is a model from Mistral. It has 3 scored results in 2 categories. It has no index.","logoUrl":null,"releaseDate":"2024-02-26T00:00:00.000Z","contextWindowTokens":128000,"modalities":["text","file"],"openWeights":null,"details":{"apiId":"mistralai/mistral-large","maxOutputTokens":102400,"knowledgeCutoff":"2024-11-30","retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.19999999999999998,"weightsUrl":null},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":12.761122354058372,"multilingual":null,"instruction":null,"math":10.46537362641498},"capabilityCounts":{"knowledge":1,"math":2},"benchmarks":[{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":38.76262626262626,"unit":"percent","family":"gpqa-diamond","index":12.761122354058372,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.4776598535417285,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-math-level-5:overall","benchmarkName":"MATH level 5","category":"math","domain":null,"eligible":true,"ranked":true,"score":24.4618580060423,"unit":"percent","family":"epoch-ai:epoch-math-level-5:overall","index":10.880390176228666,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0.892334786643176,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":1.9444444444444444,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":10.050357076601287,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0.9489457497216071,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":2,"outputPerMillion":6,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":90,"timeToFirstToken":1.138,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":{"elo":1176.484039149017,"eloUpper":1181.1661049636605,"eloLower":1171.8019733343735,"votes":62436,"rank":335,"rankUpper":null,"rankLower":null,"arenaName":"Mistral Large-2402","organization":"mistral","updatedAt":"2026-11-09"},"quality":{"coveragePercent":25,"benchmarkCount":3,"categoryCount":2,"confidence":"unranked","state":"partial"},"hosts":[{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":2,"outputPerMillion":6,"cacheReadPerMillion":0.2,"contextTokens":128000,"maxOutputTokens":102400,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"mistrallarge2411","providerId":"mistral","slug":"mistral-mistral-large-2411","name":"Mistral Large 2411","provider":"Mistral","description":"Mistral Large 2411 is a model from Mistral. It has 5 scored results in 3 categories. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":20.117209204752363,"reasoning":null,"multimodal":null,"knowledge":26.642931499822684,"multilingual":null,"instruction":null,"math":21.12927936863077},"capabilityCounts":{"math":2,"knowledge":2,"coding":1},"benchmarks":[{"benchmarkId":"vals-ai:mgsm:overall","benchmarkName":"MGSM","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":87.236,"unit":"percent","family":"vals-ai:mgsm:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mgsm","measuredAt":"2026-01-09","stderr":2.045,"costPerTest":0.002954,"latency":8.041},{"benchmarkId":"vals-ai:medqa:overall","benchmarkName":"MedQA","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":76.225,"unit":"percent","family":"vals-ai:medqa:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medqa","measuredAt":"2026-04-16","stderr":0.761,"costPerTest":0.002428,"latency":4.86},{"benchmarkId":"vals-ai:math500:overall","benchmarkName":"MATH 500","category":"math","domain":null,"eligible":true,"ranked":true,"score":74.4,"unit":"percent","family":"math-500","index":22.74959426116035,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/math500","measuredAt":"2026-01-09","stderr":3.815,"costPerTest":0.003323,"latency":9.928},{"benchmarkId":"vals-ai:mmlu_pro:overall","benchmarkName":"MMLU Pro","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":69.706,"unit":"percent","family":"mmlu-pro","index":32.30194483322433,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","measuredAt":"2026-09-01","stderr":0.442,"costPerTest":0.004997,"latency":7.192},{"benchmarkId":"vals-ai:tax_eval_v2:overall","benchmarkName":"TaxEval v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":63.778,"unit":"percent","family":"vals-ai:tax_eval_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","measuredAt":"2026-09-01","stderr":0.95,"costPerTest":0.004079,"latency":12.019},{"benchmarkId":"vals-ai:gpqa:overall","benchmarkName":"GPQA Diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":47.728,"unit":"percent","family":"gpqa-diamond","index":20.98391816642103,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","measuredAt":"2026-09-01","stderr":2.5,"costPerTest":0.006714,"latency":15.802},{"benchmarkId":"vals-ai:lcb:overall","benchmarkName":"LiveCodeBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":37.088,"unit":"percent","family":"livecodebench","index":20.117209204752363,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","measuredAt":"2026-09-01","stderr":1.091,"costPerTest":0.003291,"latency":5.349},{"benchmarkId":"vals-ai:aime:overall","benchmarkName":"AIME","category":"math","domain":null,"eligible":true,"ranked":true,"score":9.167,"unit":"percent","family":"vals-ai:aime:overall","index":19.508964476101184,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/aime","measuredAt":"2026-04-16","stderr":1.091,"costPerTest":0.006338,"latency":19.676}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":{"elo":1265.3205246983084,"eloUpper":1269.7590866034793,"eloLower":1260.8819627931375,"votes":28073,"rank":289,"rankUpper":null,"rankLower":null,"arenaName":"Mistral Large-2411","organization":"mistral","updatedAt":"2026-11-09"},"quality":{"coveragePercent":45,"benchmarkCount":5,"categoryCount":3,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"mistrallarge4oss","providerId":"mistral","slug":"mistral-mistral-large-4-oss","name":"Mistral Large 4 OSS","provider":"Mistral","description":"Mistral Large 4 OSS is a model from Mistral. It has 2 scored results in 2 categories. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":66.98670044280914,"coding":71.67584791611017,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"coding":1,"agentic":1},"benchmarks":[{"benchmarkId":"vals-ai:vibe-code:overall","benchmarkName":"Vibe Code Bench v1.1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":81.555,"unit":"percent","family":"vals-ai:vibe-code:overall","index":71.67584791611017,"aboveFit":false,"run":"OpenHands","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","measuredAt":"2026-11-09","stderr":3.289,"costPerTest":11.723179,"latency":3012.646},{"benchmarkId":"vals-ai:legal_research:overall","benchmarkName":"Legal Research Bench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":35.577,"unit":"percent","family":"vals-ai:legal_research:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_research","measuredAt":"2026-11-09","stderr":3.328,"costPerTest":1.09052,"latency":1704.14},{"benchmarkId":"vals-ai:tax_agent_bench:overall","benchmarkName":"Tax Agent Bench","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":30.668,"unit":"percent","family":"vals-ai:tax_agent_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench","measuredAt":"2026-11-09","stderr":3.302,"costPerTest":0.503196,"latency":1023.271},{"benchmarkId":"vals-ai:terminal-bench-4:overall","benchmarkName":"Terminal-Bench 4.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":26.263,"unit":"percent","family":"terminal-bench-4","index":66.98670044280914,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-4","measuredAt":"2026-11-09","stderr":5.126,"costPerTest":2.728364,"latency":3872.678},{"benchmarkId":"vals-ai:hlab:overall","benchmarkName":"HLAB-Vals-1.1","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":15.833,"unit":"percent","family":"vals-ai:hlab:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/hlab","measuredAt":"2026-11-09","stderr":3.13,"costPerTest":0.733095,"latency":688.685}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":40,"benchmarkCount":2,"categoryCount":2,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"mistrallarge401029","providerId":"mistral","slug":"mistral-mistral-large-4-0","name":"Mistral Large 4.0","provider":"Mistral","description":"Mistral Large 4.0 is a model from Mistral. It has 3 scored results in 3 categories. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":53.51053136482202,"coding":77.27253112962576,"reasoning":66.42033162367584,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"coding":1,"reasoning":1,"agentic":1},"benchmarks":[{"benchmarkId":"livebench:livebench-coding-2:overall","benchmarkName":"LiveBench Coding 2.0","category":"coding","domain":null,"eligible":true,"ranked":true,"score":95.4,"unit":"percent","family":"livebench:livebench-coding-2:overall","index":77.27253112962576,"aboveFit":false,"run":"high effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Coding","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.35108965517241375,"latency":null},{"benchmarkId":"livebench:livebench-reasoning:overall","benchmarkName":"LiveBench Reasoning","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":85.2,"unit":"percent","family":"livebench","index":71.99069780710602,"aboveFit":false,"run":"high effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Reasoning","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.0542119504950495,"latency":null},{"benchmarkId":"livebench:livebench-research:overall","benchmarkName":"LiveBench Research","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":79.7,"unit":"percent","family":"livebench-2-agentic","index":76.87655984753063,"aboveFit":false,"run":"high effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Research","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.8233545454545456,"latency":null},{"benchmarkId":"livebench:livebench-data-analysis:overall","benchmarkName":"LiveBench Data Analysis","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":76.6,"unit":"percent","family":"livebench","index":60.849965440245626,"aboveFit":false,"run":"high effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Data%20Analysis","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.06326571333333333,"latency":null},{"benchmarkId":"livebench:livebench-design:overall","benchmarkName":"LiveBench Design","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":76.3,"unit":"percent","family":"livebench-2-agentic","index":74.39003412480106,"aboveFit":false,"run":"high effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Design","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.206156,"latency":null},{"benchmarkId":"livebench:livebench-long-horizon:overall","benchmarkName":"LiveBench Long Agentic","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":19.2,"unit":"percent","family":"livebench-2-agentic","index":32.63102860484297,"aboveFit":false,"run":"high effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Long%20Agentic","measuredAt":"2026-10-14","stderr":null,"costPerTest":1.799777777777778,"latency":null},{"benchmarkId":"livebench:livebench-browser:overall","benchmarkName":"LiveBench Browser Use","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":12.5,"unit":"percent","family":"livebench-2-agentic","index":27.731110268875913,"aboveFit":false,"run":"high effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Browser","measuredAt":"2026-10-14","stderr":null,"costPerTest":1.044644,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":55.00000000000001,"benchmarkCount":3,"categoryCount":3,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:livebench","name":"LiveBench","url":"https://livebench.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"mistralmedium2505","providerId":"mistral","slug":"mistral-mistral-medium","name":"Mistral Medium","provider":"Mistral","description":"Mistral Medium is a model from Mistral. It has 5 scored results in 4 categories. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":26.55351191812661,"reasoning":null,"multimodal":37.10144395012439,"knowledge":39.33711644560559,"multilingual":null,"instruction":null,"math":35.025426514139376},"capabilityCounts":{"math":2,"knowledge":1,"multimodal":1,"coding":1},"benchmarks":[{"benchmarkId":"vals-ai:mgsm:overall","benchmarkName":"MGSM","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":90.873,"unit":"percent","family":"vals-ai:mgsm:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mgsm","measuredAt":"2026-01-09","stderr":1.728,"costPerTest":0.000492,"latency":5.006},{"benchmarkId":"vals-ai:math500:overall","benchmarkName":"MATH 500","category":"math","domain":null,"eligible":true,"ranked":true,"score":87,"unit":"percent","family":"math-500","index":36.06474773777684,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/math500","measuredAt":"2026-01-09","stderr":1.504,"costPerTest":0.001841,"latency":14.13},{"benchmarkId":"vals-ai:medqa:overall","benchmarkName":"MedQA","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":78.233,"unit":"percent","family":"vals-ai:medqa:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medqa","measuredAt":"2026-04-16","stderr":0.377,"costPerTest":0.000637,"latency":7.344},{"benchmarkId":"vals-ai:mmlu_pro:overall","benchmarkName":"MMLU Pro","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":75.29,"unit":"percent","family":"mmlu-pro","index":39.33711644560559,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","measuredAt":"2026-09-01","stderr":0.421,"costPerTest":0.000209,"latency":8.831},{"benchmarkId":"vals-ai:tax_eval_v2:overall","benchmarkName":"TaxEval v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":70.319,"unit":"percent","family":"vals-ai:tax_eval_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","measuredAt":"2026-09-01","stderr":0.894,"costPerTest":0.001422,"latency":11.052},{"benchmarkId":"vals-ai:mmmu:overall","benchmarkName":"MMMU Pro","category":"multimodal","domain":null,"eligible":true,"ranked":true,"score":62.969,"unit":"percent","family":"mmmu-pro","index":37.10144395012439,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmmu","measuredAt":"2026-09-01","stderr":1.161,"costPerTest":0.001567,"latency":12.56},{"benchmarkId":"vals-ai:legal_bench:overall","benchmarkName":"LegalBench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":61.979,"unit":"percent","family":"vals-ai:legal_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","measuredAt":"2026-10-13","stderr":0.473,"costPerTest":null,"latency":0.676},{"benchmarkId":"vals-ai:corp_fin_v2:overall","benchmarkName":"CorpFin v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":50.738,"unit":"percent","family":"vals-ai:corp_fin_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","measuredAt":"2026-08-12","stderr":0.974,"costPerTest":0.024082,"latency":30.346},{"benchmarkId":"vals-ai:lcb:overall","benchmarkName":"LiveCodeBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":44.845,"unit":"percent","family":"livecodebench","index":26.55351191812661,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","measuredAt":"2026-09-01","stderr":1.145,"costPerTest":0.00161,"latency":12.483},{"benchmarkId":"vals-ai:aime:overall","benchmarkName":"AIME","category":"math","domain":null,"eligible":true,"ranked":true,"score":42.292,"unit":"percent","family":"vals-ai:aime:overall","index":33.98610529050191,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/aime","measuredAt":"2026-04-16","stderr":1.334,"costPerTest":0,"latency":65.954},{"benchmarkId":"vals-ai:mortgage_tax:overall","benchmarkName":"MortgageTax","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":36.446,"unit":"percent","family":"vals-ai:mortgage_tax:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mortgage_tax","measuredAt":"2026-09-01","stderr":0.756,"costPerTest":0.00269,"latency":7.138}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":55.00000000000001,"benchmarkCount":5,"categoryCount":4,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"mistralmedium31","providerId":"mistral","slug":"mistral-mistral-medium-3-1","name":"Mistral Medium 3.1","provider":"Mistral","description":"Mistral Medium 3.1 is a model from Mistral. It has no scored result.","logoUrl":null,"releaseDate":"2025-08-13T14:33:59.000Z","contextWindowTokens":131072,"modalities":["text","image","file"],"openWeights":null,"details":{"apiId":"mistralai/mistral-medium-3.1","maxOutputTokens":104857,"knowledgeCutoff":"2025-06-30","retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.04,"weightsUrl":null},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":0.39999999999999997,"outputPerMillion":2,"batch":{"inputPerMillion":0.19999999999999998,"outputPerMillion":1},"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":83,"timeToFirstToken":0.755,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":{"elo":1424.538396338714,"eloUpper":1427.1059469903385,"eloLower":1421.9708456870894,"votes":94801,"rank":108,"rankUpper":null,"rankLower":null,"arenaName":"Mistral Medium 3.1-2508","organization":"mistral","updatedAt":"2026-11-09"},"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":0.4,"outputPerMillion":2,"cacheReadPerMillion":0.04,"contextTokens":131072,"maxOutputTokens":104857,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"mistralnemo","providerId":"mistral","slug":"mistral-mistral-nemo","name":"Mistral Nemo","provider":"Mistral","description":"Mistral Nemo is a model from Mistral. It has 2 scored results in 2 categories. It has no index.","logoUrl":null,"releaseDate":"2024-07-19T00:00:00.000Z","contextWindowTokens":131072,"modalities":["text"],"openWeights":true,"details":{"apiId":"mistralai/mistral-nemo","maxOutputTokens":16384,"knowledgeCutoff":"2024-04-30","retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":null,"weightsUrl":"https://huggingface.co/mistralai/Mistral-Nemo-Instruct-2407"},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":5,"multilingual":null,"instruction":null,"math":5},"capabilityCounts":{"knowledge":1,"math":1},"benchmarks":[{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":29.892676767676768,"unit":"percent","family":"gpqa-diamond","index":5,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.096191098761666,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-math-level-5:overall","benchmarkName":"MATH level 5","category":"math","domain":null,"eligible":true,"ranked":true,"score":10.82892749244713,"unit":"percent","family":"epoch-ai:epoch-math-level-5:overall","index":5,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0.6092598712054612,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":0.029,"outputPerMillion":0.03,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":20,"timeToFirstToken":0.868,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":25,"benchmarkCount":2,"categoryCount":2,"confidence":"unranked","state":"partial"},"hosts":[{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":0.029,"outputPerMillion":0.03,"cacheReadPerMillion":null,"contextTokens":131072,"maxOutputTokens":16384,"deprecated":false},{"host":"io-net","name":"IO.NET","kind":"host","url":"https://io.net/docs/guides/intelligence/io-intelligence","inputPerMillion":0.0514,"outputPerMillion":0.082,"cacheReadPerMillion":0.0257,"contextTokens":128000,"maxOutputTokens":4096,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"mistralsmall2402","providerId":"mistral","slug":"mistral-mistral-small-2402","name":"Mistral Small 2402","provider":"Mistral","description":"Mistral Small 2402 is a model from Mistral. It has 5 scored results in 3 categories. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":5,"reasoning":null,"multimodal":null,"knowledge":24.717073084779365,"multilingual":null,"instruction":null,"math":18.34743072328407},"capabilityCounts":{"math":2,"knowledge":2,"coding":1},"benchmarks":[{"benchmarkId":"vals-ai:mgsm:overall","benchmarkName":"MGSM","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":83.964,"unit":"percent","family":"vals-ai:mgsm:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mgsm","measuredAt":"2026-01-09","stderr":2.238,"costPerTest":0.000208,"latency":2.953},{"benchmarkId":"vals-ai:math500:overall","benchmarkName":"MATH 500","category":"math","domain":null,"eligible":true,"ranked":true,"score":70.6,"unit":"percent","family":"math-500","index":18.73391305392679,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/math500","measuredAt":"2026-01-09","stderr":3.981,"costPerTest":0.000327,"latency":4.885},{"benchmarkId":"vals-ai:mmlu_pro:overall","benchmarkName":"MMLU Pro","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":64.443,"unit":"percent","family":"mmlu-pro","index":25.67119479813075,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","measuredAt":"2026-09-01","stderr":0.461,"costPerTest":0.000502,"latency":4.665},{"benchmarkId":"vals-ai:medqa:overall","benchmarkName":"MedQA","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":56.983,"unit":"percent","family":"vals-ai:medqa:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medqa","measuredAt":"2026-04-16","stderr":0.885,"costPerTest":0.00021,"latency":2.763},{"benchmarkId":"vals-ai:gpqa:overall","benchmarkName":"GPQA Diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":50.758,"unit":"percent","family":"gpqa-diamond","index":23.76295137142798,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","measuredAt":"2026-09-01","stderr":2.495,"costPerTest":0.000586,"latency":7.986},{"benchmarkId":"vals-ai:tax_eval_v2:overall","benchmarkName":"TaxEval v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":49.142,"unit":"percent","family":"vals-ai:tax_eval_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","measuredAt":"2026-09-01","stderr":0.987,"costPerTest":0.000355,"latency":9.018},{"benchmarkId":"vals-ai:lcb:overall","benchmarkName":"LiveCodeBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":15.781,"unit":"percent","family":"livecodebench","index":5,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","measuredAt":"2026-09-01","stderr":0.986,"costPerTest":0.000382,"latency":5.04},{"benchmarkId":"vals-ai:aime:overall","benchmarkName":"AIME","category":"math","domain":null,"eligible":true,"ranked":true,"score":5.625,"unit":"percent","family":"vals-ai:aime:overall","index":17.960948392641342,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/aime","measuredAt":"2026-04-16","stderr":0.625,"costPerTest":0.000782,"latency":13.228}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":45,"benchmarkCount":5,"categoryCount":3,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"mistralsmall3","providerId":"mistral","slug":"mistral-mistral-small-3","name":"Mistral Small 3","provider":"Mistral","description":"Mistral Small 3 is a model from Mistral. It has 3 scored results in 2 categories. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":19.665973457857888,"multilingual":null,"instruction":null,"math":16.580813883483486},"capabilityCounts":{"knowledge":1,"math":2},"benchmarks":[{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":46.29103535353535,"unit":"percent","family":"gpqa-diamond","index":19.665973457857888,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.4794688677057852,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-math-level-5:overall","benchmarkName":"MATH level 5","category":"math","domain":null,"eligible":true,"ranked":true,"score":44.81684290030212,"unit":"percent","family":"epoch-ai:epoch-math-level-5:overall","index":20.767933561718518,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.0854927233993668,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":5.972222222222222,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":12.393694205248451,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.682716849943299,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":25,"benchmarkCount":3,"categoryCount":2,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"mistralsmall3124binstruct","providerId":"mistral","slug":"mistral-mistral-small-3-1-24b","name":"Mistral Small 3.1 24B","provider":"Mistral","description":"Mistral Small 3.1 24B is a model from Mistral. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":"2025-03-17T19:15:37.000Z","contextWindowTokens":128000,"modalities":["text","image"],"openWeights":true,"details":{"apiId":"mistralai/mistral-small-3.1-24b-instruct","maxOutputTokens":102400,"knowledgeCutoff":"2023-10-31","retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":null,"weightsUrl":"https://huggingface.co/mistralai/Mistral-Small-3.1-24B-Instruct-2503"},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":49.70131335618833,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-french:overall","benchmarkName":"EuroEval French","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":62.89999999999999,"unit":"percent","family":"euroeval","index":57.56106258083662,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/french","measuredAt":null,"stderr":0.3439658413273039,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":60.00875,"unit":"percent","family":"euroeval","index":53.638301588508654,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":0.2750823740263996,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-portuguese:overall","benchmarkName":"EuroEval Portuguese","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":58.29624999999999,"unit":"percent","family":"euroeval","index":51.314833123532715,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/portuguese","measuredAt":null,"stderr":0.42246856391925774,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-dutch:overall","benchmarkName":"EuroEval Dutch","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":57.56777777777778,"unit":"percent","family":"euroeval","index":50.32646393466016,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/dutch","measuredAt":null,"stderr":0.28713785565151134,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-italian:overall","benchmarkName":"EuroEval Italian","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":56.646249999999995,"unit":"percent","family":"euroeval","index":49.0761627777165,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/italian","measuredAt":null,"stderr":0.39222920594978644,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":55.028571428571425,"unit":"percent","family":"euroeval","index":46.88134517352287,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.3211713256319176,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-spanish:overall","benchmarkName":"EuroEval Spanish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":53.221250000000005,"unit":"percent","family":"euroeval","index":44.429225847764656,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/spanish","measuredAt":null,"stderr":0.2999244696586126,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":51.28125,"unit":"percent","family":"euroeval","index":41.797092229047394,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.46387969884443103,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":0.351,"outputPerMillion":0.5549999999999999,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":46.5,"timeToFirstToken":0.5345,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":{"elo":1277.1483244931092,"eloUpper":1281.6817870995353,"eloLower":1272.6148618866828,"votes":32666,"rank":280,"rankUpper":null,"rankLower":null,"arenaName":"Mistral Small 3.1 24B Instruct-2503","organization":"mistral","updatedAt":"2026-11-09"},"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":0.351,"outputPerMillion":0.555,"cacheReadPerMillion":null,"contextTokens":128000,"maxOutputTokens":102400,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"mistralsmall3124bbase2503","providerId":"mistral","slug":"mistral-mistral-small-3-1-24b-base","name":"Mistral Small 3.1 24B Base","provider":"Mistral","description":"Mistral Small 3.1 24B Base is a model from Mistral. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":49.482816869153496,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-french:overall","benchmarkName":"EuroEval French","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":62.097500000000004,"unit":"percent","family":"euroeval","index":56.472254730826016,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/french","measuredAt":null,"stderr":0.4258245677506173,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-portuguese:overall","benchmarkName":"EuroEval Portuguese","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":60.84875,"unit":"percent","family":"euroeval","index":54.7779883100151,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/portuguese","measuredAt":null,"stderr":0.3943527133163915,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-italian:overall","benchmarkName":"EuroEval Italian","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":58.19,"unit":"percent","family":"euroeval","index":51.17067632096123,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/italian","measuredAt":null,"stderr":0.4716245196127954,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-dutch:overall","benchmarkName":"EuroEval Dutch","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":57.24444444444445,"unit":"percent","family":"euroeval","index":49.88777499820729,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/dutch","measuredAt":null,"stderr":0.4771326393735347,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":56.6475,"unit":"percent","family":"euroeval","index":49.0778587400997,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":0.37558912058258553,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":56.34285714285714,"unit":"percent","family":"euroeval","index":48.664528479281245,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.3730596056768343,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":53.2325,"unit":"percent","family":"euroeval","index":44.4444895092134,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.5309146000064416,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-spanish:overall","benchmarkName":"EuroEval Spanish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":48.235,"unit":"percent","family":"euroeval","index":37.664031901203344,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/spanish","measuredAt":null,"stderr":0.34238592990951017,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"mistralsmall32","providerId":"mistral","slug":"mistral-mistral-small-3-2","name":"Mistral Small 3.2","provider":"Mistral","description":"Mistral Small 3.2 is a model from Mistral. It has 2 scored results in 2 categories. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":22.199199753801107,"multilingual":null,"instruction":null,"math":26.534521705705394},"capabilityCounts":{"knowledge":1,"math":1},"benchmarks":[{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":49.053030303030305,"unit":"percent","family":"gpqa-diamond","index":22.199199753801107,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.771744326195863,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":30.277777777777775,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":26.534521705705394,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":5.777449971275807,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":{"elo":1338.3063106729169,"eloUpper":1343.4999888926359,"eloLower":1333.112632453198,"votes":17437,"rank":222,"rankUpper":null,"rankLower":null,"arenaName":"Mistral Small 3.2-2506","organization":"mistral","updatedAt":"2026-11-09"},"quality":{"coveragePercent":25,"benchmarkCount":2,"categoryCount":2,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"mistralsmall3224binstruct","providerId":"mistral","slug":"mistral-mistral-small-3-2-24b","name":"Mistral Small 3.2 24B","provider":"Mistral","description":"Mistral Small 3.2 24B is a model from Mistral. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":"2025-06-20T18:10:16.000Z","contextWindowTokens":256000,"modalities":["image","text"],"openWeights":true,"details":{"apiId":"mistralai/mistral-small-3.2-24b-instruct","maxOutputTokens":26214,"knowledgeCutoff":"2023-10-31","retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.01,"weightsUrl":"https://huggingface.co/mistralai/Mistral-Small-3.2-24B-Instruct-2506"},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":50.07734590459541,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-french:overall","benchmarkName":"EuroEval French","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":63.260000000000005,"unit":"percent","family":"euroeval","index":58.049499747196535,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/french","measuredAt":null,"stderr":0.3310305877105619,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":59.38499999999999,"unit":"percent","family":"euroeval","index":52.792016359294784,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":0.2922434772582615,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-portuguese:overall","benchmarkName":"EuroEval Portuguese","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":58.262499999999996,"unit":"percent","family":"euroeval","index":51.26904213918648,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/portuguese","measuredAt":null,"stderr":0.3634341412140582,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-dutch:overall","benchmarkName":"EuroEval Dutch","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":57.44333333333332,"unit":"percent","family":"euroeval","index":50.157621457399934,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/dutch","measuredAt":null,"stderr":0.3387668268336836,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-italian:overall","benchmarkName":"EuroEval Italian","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":57.325,"unit":"percent","family":"euroeval","index":49.99707035179091,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/italian","measuredAt":null,"stderr":0.42545012927486575,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":55.00857142857142,"unit":"percent","family":"euroeval","index":46.85420977539176,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.32235105720570034,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-spanish:overall","benchmarkName":"EuroEval Spanish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":52.394999999999996,"unit":"percent","family":"euroeval","index":43.30819471247334,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/spanish","measuredAt":null,"stderr":0.3798375475120911,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":51.068749999999994,"unit":"percent","family":"euroeval","index":41.508778623904384,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.5353094899214846,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":0.09999999999999999,"outputPerMillion":0.3,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":75,"timeToFirstToken":0.597,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":0.1,"outputPerMillion":0.3,"cacheReadPerMillion":0.01,"contextTokens":256000,"maxOutputTokens":26214,"deprecated":false},{"host":"ovhcloud","name":"OVHcloud","kind":"cloud","url":"https://www.ovhcloud.com/en/public-cloud/ai-endpoints/catalog//","inputPerMillion":0.1,"outputPerMillion":0.31,"cacheReadPerMillion":null,"contextTokens":131072,"maxOutputTokens":131072,"deprecated":false},{"host":"berget-ai","name":"Berget AI","kind":"host","url":"https://api.berget.ai","inputPerMillion":0.33645,"outputPerMillion":0.33645,"cacheReadPerMillion":null,"contextTokens":32000,"maxOutputTokens":8192,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"mistral-small-4-reasoning","providerId":"mistral","slug":"mistral-small-4-reasoning","name":"Mistral Small 4 (Reasoning)","provider":"Mistral","description":"Mistral Small 4 (Reasoning) is a reasoning model from Mistral in the Mistral Small 4 family. It has no scored result.","logoUrl":null,"releaseDate":"2026-03-16","contextWindowTokens":256000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":true,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":76.9,"unit":"percent","family":"gpqa-diamond","index":null,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaMmmuPro","benchmarkName":"Artificial Analysis MMMU-Pro","category":"multimodal","score":56.8,"unit":"percent","family":"mmmu-pro","index":null,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmmu-pro","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":49.7,"unit":"percent","family":"lcr","index":null,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaIfBench","benchmarkName":"Artificial Analysis IFBench","category":"instruction","score":48.2,"unit":"percent","family":"ifbench-family","index":null,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"tau2Bench","benchmarkName":"τ²-Bench Tool-Agent-User Evaluation","category":"agentic","score":41.2,"unit":"percent","family":"tau2-bench","index":null,"aboveFit":false,"run":null,"source":"Victor Barres, Honghua Dong, Soham Ray, Xujie Si, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2506.07982","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaSciCode","benchmarkName":"Artificial Analysis SciCode","category":"coding","score":38.8,"unit":"percent","family":"scicode-family","index":null,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/scicode","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"omniscienceAccuracy","benchmarkName":"Artificial Analysis Omniscience Accuracy","category":"knowledge","score":21.4,"unit":"percent","family":"omniscienceAccuracy","index":null,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":9.9,"unit":"percent","family":"hle","index":null,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":0.3,"unit":"percent","family":"critpt","index":null,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gdpvalAaNormalized","benchmarkName":"GDPval-AA normalized","category":"agentic","score":0,"unit":"percent","family":"gdpvalAaNormalized","index":null,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":0.15,"outputPerMillion":0.6,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"mixtral8x22b","providerId":"mistral","slug":"mistral-mixtral-8x22b","name":"Mixtral 8x22B","provider":"Mistral","description":"Mixtral 8x22B is a model from Mistral. It has 2 scored results in 2 categories. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":8.447399861537917,"multilingual":null,"instruction":null,"math":10.774910817292536},"capabilityCounts":{"knowledge":1,"math":1},"benchmarks":[{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":34.05934343434344,"unit":"percent","family":"gpqa-diamond","index":8.447399861537917,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.068134779158647,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-math-level-5:overall","benchmarkName":"MATH level 5","category":"math","domain":null,"eligible":true,"ranked":true,"score":24.244712990936556,"unit":"percent","family":"epoch-ai:epoch-math-level-5:overall","index":10.774910817292536,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0.8296884112232159,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":25,"benchmarkCount":2,"categoryCount":2,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"mixtral8x22binstruct","providerId":"mistral","slug":"mistral-mixtral-8x22b-instruct","name":"Mixtral 8x22B Instruct","provider":"Mistral","description":"Mixtral 8x22B Instruct is a model from Mistral. It has no scored result.","logoUrl":null,"releaseDate":"2024-04-17T00:00:00.000Z","contextWindowTokens":65536,"modalities":["text","file"],"openWeights":true,"details":{"apiId":"mistralai/mixtral-8x22b-instruct","maxOutputTokens":52428,"knowledgeCutoff":"2024-01-31","retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.19999999999999998,"weightsUrl":"https://huggingface.co/mistralai/Mixtral-8x22B-Instruct-v0.1"},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":2,"outputPerMillion":6,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":113,"timeToFirstToken":0.498,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":2,"outputPerMillion":6,"cacheReadPerMillion":0.2,"contextTokens":65536,"maxOutputTokens":52428,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"mixtral-8x22b-instruct-v0-1","providerId":"mistral","slug":"mixtral-8x22b-instruct-v0-1","name":"Mixtral 8x22B Instruct v0.1","provider":"Mistral","description":"Mixtral 8x22B Instruct v0.1 is a standard model from Mistral in the Mixtral 8x22B family. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":"2024-04-17","contextWindowTokens":64000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":24.424929265623696,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"knowledge":1},"benchmarks":[{"benchmarkId":"mmlu","benchmarkName":"Massive Multitask Language Understanding","category":"knowledge","score":77.8,"unit":"percent","family":"mmlu","index":24.424929265623696,"aboveFit":false,"run":null,"source":"Dan Hendrycks, Collin Burns, Steven Basart, Andy Zou, Mantas Mazeika, Dawn Song, Jacob Steinhardt","sourceUrl":"https://arxiv.org/abs/2009.03300","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:medqa:overall","benchmarkName":"MedQA","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":62.139,"unit":"percent","family":"vals-ai:medqa:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medqa","measuredAt":"2026-04-16","stderr":0.867,"costPerTest":0.000918,"latency":8.946}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":{"elo":1161.9933731289793,"eloUpper":1166.5098024255178,"eloLower":1157.4769438324406,"votes":51416,"rank":344,"rankUpper":null,"rankLower":null,"arenaName":"Mixtral 8x22B Instruct v0.1","organization":"mistral","updatedAt":"2026-11-09"},"quality":{"coveragePercent":15,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"mixtral8x7b","providerId":"mistral","slug":"mistral-mixtral-8x7b","name":"Mixtral 8x7B","provider":"Mistral","description":"Mixtral 8x7B is a model from Mistral. It has 2 scored results in 2 categories. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":5,"multilingual":null,"instruction":null,"math":5},"capabilityCounts":{"knowledge":1,"math":1},"benchmarks":[{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":30.208333333333332,"unit":"percent","family":"gpqa-diamond","index":5,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.937059603941351,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-math-level-5:overall","benchmarkName":"MATH level 5","category":"math","domain":null,"eligible":true,"ranked":true,"score":9.62046827794562,"unit":"percent","family":"epoch-ai:epoch-math-level-5:overall","index":5,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0.5102464546317853,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":25,"benchmarkCount":2,"categoryCount":2,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"mixtral8x7bv01","providerId":"mistral","slug":"mistral-mixtral-8x7b-v0-1","name":"Mixtral 8x7B V0.1","provider":"Mistral","description":"Mixtral 8x7B V0.1 is a model from Mistral. It has no scored result.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[{"benchmarkId":"vals-ai:legal_bench:overall","benchmarkName":"LegalBench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":55.836,"unit":"percent","family":"vals-ai:legal_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","measuredAt":"2026-10-13","stderr":0.742,"costPerTest":null,"latency":14.142},{"benchmarkId":"vals-ai:medqa:overall","benchmarkName":"MedQA","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":53.218,"unit":"percent","family":"vals-ai:medqa:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medqa","measuredAt":"2026-04-16","stderr":0.892,"costPerTest":0.00045,"latency":3.488}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"monad","providerId":"pleias","slug":"pleias-monad","name":"Monad","provider":"PleIAs","description":"Monad is a model from PleIAs. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":5,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":2.8200000000000003,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":0.5550520808897125,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-dutch:overall","benchmarkName":"EuroEval Dutch","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":2.118888888888889,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/dutch","measuredAt":null,"stderr":0.2677708620591517,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-italian:overall","benchmarkName":"EuroEval Italian","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":2.0162500000000003,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/italian","measuredAt":null,"stderr":0.27673882091242635,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-french:overall","benchmarkName":"EuroEval French","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":1.8287499999999999,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/french","measuredAt":null,"stderr":0.18539232049898938,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":1.5214285714285711,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.2504770957796423,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-portuguese:overall","benchmarkName":"EuroEval Portuguese","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":1.49625,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/portuguese","measuredAt":null,"stderr":0.29170833035756794,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-spanish:overall","benchmarkName":"EuroEval Spanish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":1.4612500000000002,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/spanish","measuredAt":null,"stderr":0.2987970674220214,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":1.2162499999999998,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.39348403716034025,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"moondream3preview","providerId":"moondream","slug":"moondream-moondream-3-preview","name":"Moondream 3 Preview","provider":"Moondream","description":"Moondream 3 Preview is a model from Moondream. It has no scored result.","logoUrl":null,"releaseDate":"2026-10-15T22:13:05.000Z","contextWindowTokens":32000,"modalities":["text","image"],"openWeights":true,"details":{"apiId":"moondream/moondream3-preview","maxOutputTokens":4096,"knowledgeCutoff":null,"retiresAt":null,"tools":false,"structuredOutput":false,"cacheReadPerMillion":null,"weightsUrl":"https://huggingface.co/moondream/moondream3-preview"},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":0.3,"outputPerMillion":2.5,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":190.5,"timeToFirstToken":0.2995,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"moondream319ba2b","providerId":"moondream","slug":"moondream-moondream-3-1","name":"Moondream 3.1","provider":"Moondream","description":"Moondream 3.1 is a model from Moondream. It has no scored result.","logoUrl":null,"releaseDate":"2026-10-15T22:12:53.000Z","contextWindowTokens":32000,"modalities":["text","image"],"openWeights":true,"details":{"apiId":"moondream/moondream3.1-9b-a2b","maxOutputTokens":4096,"knowledgeCutoff":null,"retiresAt":null,"tools":false,"structuredOutput":false,"cacheReadPerMillion":null,"weightsUrl":"https://huggingface.co/moondream/moondream3.1-9B-A2B"},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":0.3,"outputPerMillion":2.5,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":200,"timeToFirstToken":0.3815,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"moonshine-tiny","providerId":"useful-sensors","slug":"moonshine-tiny","name":"Moonshine Tiny","provider":"Useful Sensors","description":"Moonshine Tiny is a standard model from Useful Sensors. It has no scored result.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"moonshot-v1","providerId":"moonshot-ai","slug":"moonshot-v1","name":"Moonshot v1","provider":"Moonshot AI","description":"Moonshot v1 is a standard model from Moonshot AI in the Moonshot family. It has no scored result.","logoUrl":null,"releaseDate":null,"contextWindowTokens":128000,"modalities":["text"],"openWeights":false,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"morphv3fast","providerId":"morph","slug":"morph-morph-v3-fast","name":"Morph V3 Fast","provider":"Morph","description":"Morph V3 Fast is a model from Morph. It has no scored result.","logoUrl":null,"releaseDate":"2025-07-07T17:40:02.000Z","contextWindowTokens":262144,"modalities":["text"],"openWeights":null,"details":{"apiId":"morph/morph-v3-fast","maxOutputTokens":131072,"knowledgeCutoff":null,"retiresAt":null,"tools":false,"structuredOutput":true,"cacheReadPerMillion":null,"weightsUrl":null},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":0.7999999999999999,"outputPerMillion":1.2,"batch":{"inputPerMillion":0.39999999999999997,"outputPerMillion":0.6},"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":6041,"timeToFirstToken":0.3625,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":0.8,"outputPerMillion":1.2,"cacheReadPerMillion":null,"contextTokens":262144,"maxOutputTokens":131072,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"morphv3large","providerId":"morph","slug":"morph-morph-v3-large","name":"Morph V3 Large","provider":"Morph","description":"Morph V3 Large is a model from Morph. It has no scored result.","logoUrl":null,"releaseDate":"2025-07-07T17:54:18.000Z","contextWindowTokens":262144,"modalities":["text"],"openWeights":null,"details":{"apiId":"morph/morph-v3-large","maxOutputTokens":131072,"knowledgeCutoff":null,"retiresAt":null,"tools":false,"structuredOutput":true,"cacheReadPerMillion":null,"weightsUrl":null},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":0.8999999999999999,"outputPerMillion":1.9,"batch":{"inputPerMillion":0.44999999999999996,"outputPerMillion":0.95},"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":3112.5,"timeToFirstToken":0.3885,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":0.9,"outputPerMillion":1.9,"cacheReadPerMillion":null,"contextTokens":262144,"maxOutputTokens":131072,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"motif3","providerId":"motif-technologies","slug":"motif-technologies-motif-3","name":"Motif 3","provider":"Motif Technologies","description":"Motif 3 is a model from Motif Technologies. It has no scored result.","logoUrl":null,"releaseDate":"2026-11-02T21:35:34.000Z","contextWindowTokens":262144,"modalities":["text"],"openWeights":true,"details":{"apiId":"motif-technologies/motif-3","maxOutputTokens":235929,"knowledgeCutoff":null,"retiresAt":null,"tools":false,"structuredOutput":true,"cacheReadPerMillion":0.06,"weightsUrl":"https://huggingface.co/Motif-Technologies/Motif-3"},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":0.3,"outputPerMillion":1.2,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":82,"timeToFirstToken":0.374,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"murf-falcon-2","providerId":"murf-ai","slug":"murf-falcon-2","name":"Murf Falcon 2","provider":"Murf AI","description":"Murf Falcon 2 is a standard model from Murf AI. It has no scored result.","logoUrl":null,"releaseDate":"2026-03-05","contextWindowTokens":null,"modalities":["text"],"openWeights":false,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"muse3b","providerId":"unknown","slug":"unknown-muse-3b","name":"Muse 3B","provider":"Unknown","description":"Muse 3B is a model. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":5,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-portuguese:overall","benchmarkName":"EuroEval Portuguese","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":1.01875,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/portuguese","measuredAt":null,"stderr":0.47288179019708515,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":0.8685714285714287,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.8597341781869888,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-spanish:overall","benchmarkName":"EuroEval Spanish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":0.8250000000000001,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/spanish","measuredAt":null,"stderr":0.3846954964644115,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-french:overall","benchmarkName":"EuroEval French","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":0.58625,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/french","measuredAt":null,"stderr":0.509958638028615,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-dutch:overall","benchmarkName":"EuroEval Dutch","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":0.43222222222222223,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/dutch","measuredAt":null,"stderr":0.5228872833710063,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"musespark12contributor","providerId":"meta","slug":"meta-muse-spark-1-2-contributor","name":"Muse Spark 1.2 Contributor","provider":"Meta","description":"Muse Spark 1.2 Contributor is a model from Meta. It has no scored result.","logoUrl":null,"releaseDate":"2026-08-21T18:21:16.000Z","contextWindowTokens":1048576,"modalities":["text","image","video","file"],"openWeights":null,"details":{"apiId":"meta/muse-spark-1.2-contributor","maxOutputTokens":943718,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.002,"weightsUrl":null},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":0.09999999999999999,"outputPerMillion":0.19999999999999998,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":103,"timeToFirstToken":2.352,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"musespark13contributor","providerId":"meta","slug":"meta-muse-spark-1-3-contributor","name":"Muse Spark 1.3 Contributor","provider":"Meta","description":"Muse Spark 1.3 Contributor is a model from Meta. It has no scored result.","logoUrl":null,"releaseDate":"2026-09-02T20:38:39.000Z","contextWindowTokens":1048576,"modalities":["text","image","video","file"],"openWeights":null,"details":{"apiId":"meta/muse-spark-1.3-contributor","maxOutputTokens":943718,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.002,"weightsUrl":null},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":0.09999999999999999,"outputPerMillion":0.19999999999999998,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":78,"timeToFirstToken":3.2965,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"musespark13max","providerId":"meta","slug":"meta-muse-spark-1-3-max","name":"Muse Spark 1.3 Max","provider":"Meta","description":"Muse Spark 1.3 Max is a model from Meta. It has 7 scored results in 3 categories. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":66.81337315888734,"coding":69.91805389155373,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":67.18081324818813},"capabilityCounts":{"coding":4,"agentic":2,"math":1},"benchmarks":[{"benchmarkId":"vals-ai:vibe-code:overall","benchmarkName":"Vibe Code Bench v1.1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":85.856,"unit":"percent","family":"vals-ai:vibe-code:overall","index":73.22771628988818,"aboveFit":false,"run":"OpenHands","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","measuredAt":"2026-11-09","stderr":2.508,"costPerTest":2.543084,"latency":977.668},{"benchmarkId":"vals-ai:terminal-bench-2-1:overall","benchmarkName":"Terminal-Bench 2.1","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":79.026,"unit":"percent","family":"terminal-bench-2-1","index":67.35990124019654,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2-1","measuredAt":"2026-10-01","stderr":0.991,"costPerTest":0.59671,"latency":731.78},{"benchmarkId":"vals-ai:cyber:overall","benchmarkName":"CyberBench v1.1","category":null,"domain":"Other","eligible":false,"ranked":true,"score":72.738,"unit":"percent","family":"vals-ai:cyber:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/cyber","measuredAt":"2026-11-05","stderr":5.67,"costPerTest":3.343249,"latency":978.426},{"benchmarkId":"vals-ai:emb:overall","benchmarkName":"EMB","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":67.426,"unit":"percent","family":"vals-ai:emb:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/emb","measuredAt":"2026-10-20","stderr":3.056,"costPerTest":2.546868,"latency":829.829},{"benchmarkId":"vals-ai:fabv2:overall","benchmarkName":"Finance Agent (v2)","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":59.958,"unit":"percent","family":"vals-ai:fabv2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/fabv2","measuredAt":"2026-10-29","stderr":2.06,"costPerTest":0.756512,"latency":198.929},{"benchmarkId":"vals-ai:proof_bench:overall","benchmarkName":"ProofBench v1.1","category":"math","domain":null,"eligible":true,"ranked":true,"score":58,"unit":"percent","family":"vals-ai:proof_bench:overall","index":67.18081324818813,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/proof_bench","measuredAt":"2026-11-09","stderr":4.96,"costPerTest":0.479019,"latency":568.696},{"benchmarkId":"vals-ai:ioi:overall","benchmarkName":"IOI","category":"coding","domain":null,"eligible":true,"ranked":true,"score":56.556,"unit":"percent","family":"vals-ai:ioi:overall","index":67.34902347286052,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/ioi","measuredAt":"2026-10-29","stderr":2.517,"costPerTest":1.729661,"latency":1737.996},{"benchmarkId":"vals-ai:legal_research:overall","benchmarkName":"Legal Research Bench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":55.288,"unit":"percent","family":"vals-ai:legal_research:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_research","measuredAt":"2026-11-09","stderr":3.456,"costPerTest":0.588812,"latency":317.461},{"benchmarkId":"vals-ai:code-migration:overall","benchmarkName":"Code Migration","category":"coding","domain":null,"eligible":true,"ranked":true,"score":47.414,"unit":"percent","family":"vals-ai:code-migration:overall","index":70.37735524739344,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/code-migration","measuredAt":"2026-10-29","stderr":4.265,"costPerTest":14.210071,"latency":5054.737},{"benchmarkId":"vals-ai:tax_agent_bench:overall","benchmarkName":"Tax Agent Bench","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":42.435,"unit":"percent","family":"vals-ai:tax_agent_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench","measuredAt":"2026-11-09","stderr":3.511,"costPerTest":0.389917,"latency":248.818},{"benchmarkId":"vals-ai:mysterymechanism:overall","benchmarkName":"MysteryMechanism","category":null,"domain":"Science","eligible":false,"ranked":true,"score":36.036,"unit":"percent","family":"vals-ai:mysterymechanism:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mysterymechanism","measuredAt":"2026-11-11","stderr":3.23,"costPerTest":0.851846,"latency":451.803},{"benchmarkId":"vals-ai:terminal-bench-4:overall","benchmarkName":"Terminal-Bench 4.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":24.747,"unit":"percent","family":"terminal-bench-4","index":66.26684507757815,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-4","measuredAt":"2026-11-09","stderr":0.505,"costPerTest":6.654565,"latency":3176.371},{"benchmarkId":"vals-ai:vcb-1-100:overall","benchmarkName":"Vibe Code Bench 1-100","category":"coding","domain":null,"eligible":true,"ranked":true,"score":20.461,"unit":"percent","family":"vals-ai:vcb-1-100:overall","index":68.71812055607276,"aboveFit":false,"run":"OpenHands","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vcb-1-100","measuredAt":"2026-10-29","stderr":3.847,"costPerTest":5.599743,"latency":1201.545},{"benchmarkId":"vals-ai:terminal-bench-science:overall","benchmarkName":"Terminal-Bench-Science","category":null,"domain":"Science","eligible":false,"ranked":true,"score":10,"unit":"percent","family":"vals-ai:terminal-bench-science:overall","index":null,"aboveFit":false,"run":"Mini-SWE-agent","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-science","measuredAt":"2026-11-09","stderr":3.612,"costPerTest":6.094983,"latency":5045.791},{"benchmarkId":"vals-ai:cua_bench:overall","benchmarkName":"CUA-bench","category":null,"domain":"Other","eligible":false,"ranked":true,"score":5.833333,"unit":"percent","family":"vals-ai:cua_bench:overall","index":null,"aboveFit":false,"run":"muse-code","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/cua_bench","measuredAt":"2026-10-13","stderr":0,"costPerTest":143.481477,"latency":36.39},{"benchmarkId":"vals-ai:srebench:overall","benchmarkName":"SRE Bench","category":null,"domain":"Other","eligible":false,"ranked":true,"score":3.817,"unit":"percent","family":"vals-ai:srebench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/srebench","measuredAt":"2026-11-05","stderr":1.186,"costPerTest":25.861222,"latency":6407.234},{"benchmarkId":"vals-ai:programbench:overall","benchmarkName":"ProgramBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":2.5,"unit":"percent","family":"vals-ai:programbench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/programbench","measuredAt":"2026-11-09","stderr":1.107,"costPerTest":54.285113,"latency":19703.41}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":{"elo":1491.468287476569,"eloUpper":1496.5208114214065,"eloLower":1486.415763531731,"votes":26111,"rank":13,"rankUpper":null,"rankLower":null,"arenaName":"Muse Spark 1.3-max","organization":"meta","updatedAt":"2026-11-09"},"quality":{"coveragePercent":50,"benchmarkCount":7,"categoryCount":3,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"mythomaxl213b","providerId":"gryphe","slug":"gryphe-mythomax-13b","name":"MythoMax 13B","provider":"Gryphe","description":"MythoMax 13B is a model from Gryphe. It has no scored result.","logoUrl":null,"releaseDate":"2023-07-02T00:00:00.000Z","contextWindowTokens":8192,"modalities":["text"],"openWeights":true,"details":{"apiId":"gryphe/mythomax-l2-13b","maxOutputTokens":3686,"knowledgeCutoff":"2023-06-30","retiresAt":null,"tools":false,"structuredOutput":true,"cacheReadPerMillion":null,"weightsUrl":"https://huggingface.co/Gryphe/MythoMax-L2-13b"},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":0.08,"outputPerMillion":0.11,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":97,"timeToFirstToken":0.368,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[{"host":"novita-ai","name":"Novita AI","kind":"host","url":"https://novita.ai/docs/guides/introduction","inputPerMillion":0.09,"outputPerMillion":0.09,"cacheReadPerMillion":null,"contextTokens":4096,"maxOutputTokens":3200,"deprecated":false},{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":0.08,"outputPerMillion":0.11,"cacheReadPerMillion":null,"contextTokens":8192,"maxOutputTokens":3686,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gemini25flashimage","providerId":"google","slug":"google-nano-banana-gemini-2-5-flash-image","name":"Nano Banana (Gemini 2.5 Flash Image)","provider":"Google","description":"Nano Banana (Gemini 2.5 Flash Image) is a model from Google. It has no scored result.","logoUrl":null,"releaseDate":"2025-10-07T20:53:51.000Z","contextWindowTokens":32768,"modalities":["image","text"],"openWeights":null,"details":{"apiId":"google/gemini-2.5-flash-image","maxOutputTokens":8192,"knowledgeCutoff":"2025-01-31","retiresAt":null,"tools":false,"structuredOutput":true,"cacheReadPerMillion":0.03,"weightsUrl":null},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":0.3,"outputPerMillion":2.5,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":0.3,"outputPerMillion":2.5,"cacheReadPerMillion":0.03,"contextTokens":32768,"maxOutputTokens":8192,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gemini31flashimagepreview","providerId":"google","slug":"google-nano-banana-2-gemini-3-1-flash-image-preview","name":"Nano Banana 2 (Gemini 3.1 Flash Image Preview)","provider":"Google","description":"Nano Banana 2 (Gemini 3.1 Flash Image Preview) is a model from Google. It has no scored result.","logoUrl":null,"releaseDate":"2026-02-26T15:25:58.000Z","contextWindowTokens":65536,"modalities":["image","text"],"openWeights":null,"details":{"apiId":"google/gemini-3.1-flash-image-preview","maxOutputTokens":58982,"knowledgeCutoff":null,"retiresAt":null,"tools":false,"structuredOutput":true,"cacheReadPerMillion":null,"weightsUrl":null},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":0.5,"outputPerMillion":3,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":0.5,"outputPerMillion":3,"cacheReadPerMillion":null,"contextTokens":65536,"maxOutputTokens":58982,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gemini31flashimage","providerId":"google","slug":"google-nano-banana-2-gemini-3-1-flash-image","name":"Nano Banana 2 (Gemini 3.1 Flash Image)","provider":"Google","description":"Nano Banana 2 (Gemini 3.1 Flash Image) is a model from Google. It has no scored result.","logoUrl":null,"releaseDate":"2026-06-18T03:41:05.000Z","contextWindowTokens":131072,"modalities":["image","text"],"openWeights":null,"details":{"apiId":"google/gemini-3.1-flash-image","maxOutputTokens":32768,"knowledgeCutoff":null,"retiresAt":null,"tools":false,"structuredOutput":true,"cacheReadPerMillion":null,"weightsUrl":null},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":0.5,"outputPerMillion":3,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":0.5,"outputPerMillion":3,"cacheReadPerMillion":null,"contextTokens":131072,"maxOutputTokens":32768,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gemini31flashliteimage","providerId":"google","slug":"google-nano-banana-2-lite-gemini-3-1-flash-lite-image","name":"Nano Banana 2 Lite (Gemini 3.1 Flash Lite Image)","provider":"Google","description":"Nano Banana 2 Lite (Gemini 3.1 Flash Lite Image) is a model from Google. It has no scored result.","logoUrl":null,"releaseDate":"2026-06-30T16:33:45.000Z","contextWindowTokens":65536,"modalities":["image","text"],"openWeights":null,"details":{"apiId":"google/gemini-3.1-flash-lite-image","maxOutputTokens":58982,"knowledgeCutoff":"2025-01-01","retiresAt":null,"tools":false,"structuredOutput":false,"cacheReadPerMillion":null,"weightsUrl":null},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":0.25,"outputPerMillion":1.5,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":0.25,"outputPerMillion":1.5,"cacheReadPerMillion":null,"contextTokens":65536,"maxOutputTokens":58982,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gemininanobanana21","providerId":"google","slug":"google-nano-banana-2-1","name":"Nano Banana 2.1","provider":"Google","description":"Nano Banana 2.1 is a model from Google. It has no scored result.","logoUrl":null,"releaseDate":"2026-10-06T15:33:45.000Z","contextWindowTokens":65536,"modalities":["image","text"],"openWeights":null,"details":{"apiId":"google/gemini-nano-banana-2.1","maxOutputTokens":58982,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":null,"weightsUrl":null},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":1.5,"outputPerMillion":7.5,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":1.5,"outputPerMillion":7.5,"cacheReadPerMillion":null,"contextTokens":65536,"maxOutputTokens":58982,"deprecated":false},{"host":"google-vertex","name":"Google Vertex AI","kind":"maker","url":"https://cloud.google.com/vertex-ai/generative-ai/docs/models","inputPerMillion":1.5,"outputPerMillion":30,"cacheReadPerMillion":0.15,"contextTokens":131072,"maxOutputTokens":32768,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gemini3proimagepreview","providerId":"google","slug":"google-nano-banana-pro-gemini-3-pro-image-preview","name":"Nano Banana Pro (Gemini 3 Pro Image Preview)","provider":"Google","description":"Nano Banana Pro (Gemini 3 Pro Image Preview) is a model from Google. It has no scored result.","logoUrl":null,"releaseDate":"2025-11-20T15:49:57.000Z","contextWindowTokens":65536,"modalities":["image","text"],"openWeights":null,"details":{"apiId":"google/gemini-3-pro-image-preview","maxOutputTokens":32768,"knowledgeCutoff":null,"retiresAt":null,"tools":false,"structuredOutput":true,"cacheReadPerMillion":0.19999999999999998,"weightsUrl":null},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":2,"outputPerMillion":12,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":2,"outputPerMillion":12,"cacheReadPerMillion":0.2,"contextTokens":65536,"maxOutputTokens":32768,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"gemini3proimage","providerId":"google","slug":"google-nano-banana-pro-gemini-3-pro-image","name":"Nano Banana Pro (Gemini 3 Pro Image)","provider":"Google","description":"Nano Banana Pro (Gemini 3 Pro Image) is a model from Google. It has no scored result.","logoUrl":null,"releaseDate":"2026-06-18T03:40:54.000Z","contextWindowTokens":131072,"modalities":["image","text"],"openWeights":null,"details":{"apiId":"google/gemini-3-pro-image","maxOutputTokens":32768,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.19999999999999998,"weightsUrl":null},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":2,"outputPerMillion":12,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":2,"outputPerMillion":12,"cacheReadPerMillion":0.2,"contextTokens":131072,"maxOutputTokens":32768,"deprecated":false},{"host":"google-ai-studio","name":"Google AI Studio","kind":"maker","url":"https://ai.google.dev/gemini-api/docs/models","inputPerMillion":2,"outputPerMillion":120,"cacheReadPerMillion":null,"contextTokens":65536,"maxOutputTokens":32768,"deprecated":false},{"host":"google-vertex","name":"Google Vertex AI","kind":"maker","url":"https://cloud.google.com/vertex-ai/generative-ai/docs/models","inputPerMillion":2,"outputPerMillion":120,"cacheReadPerMillion":0.2,"contextTokens":65536,"maxOutputTokens":32768,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"needle-3","providerId":"cactus-compute","slug":"needle-3","name":"Needle 3 (2-bit)","provider":"Cactus Compute","description":"Needle 3 (2-bit) is a standard model from Cactus Compute. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":51.47392768219078,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"reasoning":1},"benchmarks":[{"benchmarkId":"jevBench15","benchmarkName":"jevBench15","category":"reasoning","score":0,"unit":"percent","family":"jevBench15","index":51.47392768219078,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":15,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"needle-3-tools","providerId":"cactus-compute","slug":"needle-3-tools","name":"Needle 3 (options as tools)","provider":"Cactus Compute","description":"Needle 3 (options as tools) is a standard model from Cactus Compute. It has no scored result.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[{"benchmarkId":"jevBench15","benchmarkName":"jevBench15","category":"reasoning","score":0,"unit":"percent","family":"jevBench15","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"nemotron3nano30ba3b","providerId":"nvidia","slug":"nvidia-nemotron-3-nano-30b-a3b","name":"Nemotron 3 Nano 30B A3B","provider":"NVIDIA","description":"Nemotron 3 Nano 30B A3B is a model from NVIDIA. It has no scored result.","logoUrl":null,"releaseDate":"2025-12-14T16:54:35.000Z","contextWindowTokens":262144,"modalities":["text"],"openWeights":true,"details":{"apiId":"nvidia/nemotron-3-nano-30b-a3b","maxOutputTokens":235929,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.012,"weightsUrl":"https://huggingface.co/nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16"},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":0.06,"outputPerMillion":0.24,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":256,"timeToFirstToken":0.328,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[{"host":"crusoe","name":"Crusoe","kind":"host","url":"https://docs.crusoecloud.com/managed-inference/overview","inputPerMillion":0.05,"outputPerMillion":0.2,"cacheReadPerMillion":0.03,"contextTokens":262144,"maxOutputTokens":262144,"deprecated":false},{"host":"deepinfra","name":"Deep Infra","kind":"host","url":"https://deepinfra.com/models","inputPerMillion":0.05,"outputPerMillion":0.2,"cacheReadPerMillion":0.025,"contextTokens":262144,"maxOutputTokens":262144,"deprecated":true},{"host":"novita-ai","name":"Novita AI","kind":"host","url":"https://novita.ai/docs/guides/introduction","inputPerMillion":0.05,"outputPerMillion":0.2,"cacheReadPerMillion":null,"contextTokens":262144,"maxOutputTokens":32768,"deprecated":false},{"host":"amazon-bedrock","name":"Amazon Bedrock","kind":"cloud","url":"https://docs.aws.amazon.com/bedrock/latest/userguide/models-supported.html","inputPerMillion":0.06,"outputPerMillion":0.24,"cacheReadPerMillion":null,"contextTokens":262144,"maxOutputTokens":8192,"deprecated":false},{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":0.06,"outputPerMillion":0.24,"cacheReadPerMillion":0.012,"contextTokens":262144,"maxOutputTokens":235929,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"nemotron-3-super-100b","providerId":"nvidia","slug":"nemotron-3-super-100b","name":"Nemotron 3 Super 100B","provider":"NVIDIA","description":"Nemotron 3 Super 100B is a standard model from NVIDIA. It has 7 scored results in 4 categories. It has no index.","logoUrl":null,"releaseDate":"2026-03-11","contextWindowTokens":1000000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":5,"coding":45.828034258051495,"reasoning":null,"multimodal":null,"knowledge":47.51798927095867,"multilingual":null,"instruction":null,"math":49.83046246214317},"capabilityCounts":{"math":2,"knowledge":3,"coding":1,"agentic":1},"benchmarks":[{"benchmarkId":"hmmtFeb2025","benchmarkName":"Harvard-MIT Mathematics Tournament February 2025","category":"math","score":93.67,"unit":"percent","family":"hmmtFeb2025","index":51.70342827286688,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aime2025","benchmarkName":"American Invitational Mathematics Examination 2025","category":"math","score":90.21,"unit":"percent","family":"aime2025","index":47.95749665141946,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmluPro","benchmarkName":"Massive Multitask Language Understanding Professional","category":"knowledge","score":83.73,"unit":"percent","family":"mmlu-pro","index":49.97050620357439,"aboveFit":false,"run":null,"source":"Yubo Wang, Xueguang Ma, Ge Zhang, Yuansheng Ni, Abhranil Chandra, Shiguang Guo, Weiming Ren, Aaran Arulraj, Xuan He, Ziyan Jiang, Tianle Li, Max Ku, Kai Wang, Alex Zhuang, Rongqi Fan, Xiang Yue, Wenhu Chen","sourceUrl":"https://arxiv.org/abs/2406.01574","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gpqa","benchmarkName":"Graduate-Level Google-Proof Q&A","category":"knowledge","score":79.23,"unit":"percent","family":"gpqa-diamond","index":49.8766917717441,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"sweVerified","benchmarkName":"Software Engineering Benchmark Verified","category":"coding","score":60.47,"unit":"percent","family":"swe-bench-verified","index":45.828034258051495,"aboveFit":false,"run":null,"source":"Carlos E. Jimenez, John Yang, Alexander Wettig, Shunyu Yao, Kexin Pei, Ofir Press, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2310.06770","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hle","benchmarkName":"Humanity's Last Exam","category":"knowledge","score":18.26,"unit":"percent","family":"hle","index":42.70676983755752,"aboveFit":false,"run":null,"source":"Center for AI Safety, Scale AI, and thousands of expert contributors","sourceUrl":"https://lastexam.ai/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"clawEval","benchmarkName":"Claw-Eval","category":"agentic","score":5.5,"unit":"percent","family":"clawEval","index":5,"aboveFit":false,"run":null,"source":"Bowen Ye, Rang Li, Qibin Yang, Yuanxin Liu, Linli Yao, Hanglong Lv, Zhihui Xie, Chenxin An, Lei Li, Lingpeng Kong, Qi Liu, Zhifang Sui, Tong Yang","sourceUrl":"https://arxiv.org/abs/2604.06132","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":145,"timeToFirstToken":15.08,"source":"BenchLM","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":65,"benchmarkCount":7,"categoryCount":4,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"nemotron-3-super-120b-a12b","providerId":"nvidia","slug":"nemotron-3-super-120b-a12b","name":"Nemotron 3 Super 120B A12B","provider":"NVIDIA","description":"Nemotron 3 Super 120B A12B is a standard model from NVIDIA. It has no scored result.","logoUrl":null,"releaseDate":"2026-03-11","contextWindowTokens":262144,"modalities":["text"],"openWeights":true,"details":{"apiId":"nvidia/nemotron-3-super-120b-a12b","maxOutputTokens":16384,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":null,"weightsUrl":"https://huggingface.co/nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-FP8"},"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":0.075,"outputPerMillion":0.39999999999999997,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":134,"timeToFirstToken":0.721,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":0.075,"outputPerMillion":0.4,"cacheReadPerMillion":null,"contextTokens":262144,"maxOutputTokens":16384,"deprecated":false},{"host":"amazon-bedrock","name":"Amazon Bedrock","kind":"cloud","url":"https://docs.aws.amazon.com/bedrock/latest/userguide/models-supported.html","inputPerMillion":0.15,"outputPerMillion":0.65,"cacheReadPerMillion":null,"contextTokens":262144,"maxOutputTokens":131072,"deprecated":false},{"host":"nebius","name":"Nebius","kind":"host","url":"https://docs.tokenfactory.nebius.com/","inputPerMillion":0.3,"outputPerMillion":0.9,"cacheReadPerMillion":0.06,"contextTokens":262144,"maxOutputTokens":32768,"deprecated":false},{"host":"cloudflare","name":"Cloudflare Workers AI","kind":"cloud","url":"https://developers.cloudflare.com/workers-ai/models/","inputPerMillion":0.5,"outputPerMillion":1.5,"cacheReadPerMillion":null,"contextTokens":256000,"maxOutputTokens":256000,"deprecated":true},{"host":"crusoe","name":"Crusoe","kind":"host","url":"https://docs.crusoecloud.com/managed-inference/overview","inputPerMillion":0.3,"outputPerMillion":2.4,"cacheReadPerMillion":0.15,"contextTokens":262144,"maxOutputTokens":262144,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"nemotron-3-5-asr-streaming-0-6b","providerId":"nvidia","slug":"nemotron-3-5-asr-streaming-0-6b","name":"Nemotron 3.5 ASR Streaming 0.6B","provider":"NVIDIA","description":"Nemotron 3.5 ASR Streaming 0.6B is a standard model from NVIDIA. It has no scored result.","logoUrl":null,"releaseDate":"2026-06-04","contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"nemotron-3-5-bolt-74b-a9b-bf16","providerId":"nvidia","slug":"nemotron-3-5-bolt-74b-a9b-bf16","name":"Nemotron 3.5 Bolt 74B-A9B BF16","provider":"NVIDIA","description":"Nemotron 3.5 Bolt 74B-A9B BF16 is a reasoning model from NVIDIA in the Nemotron 3.5 Bolt 74B-A9B family. It has 9 scored results in 3 categories. It has no index.","logoUrl":null,"releaseDate":"2026-10-15","contextWindowTokens":256000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":true,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":42.32141660301816,"coding":null,"reasoning":53.52615527313249,"multimodal":null,"knowledge":50.018775082705325,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"agentic":5,"knowledge":3,"reasoning":1},"benchmarks":[{"benchmarkId":"pinchBench","benchmarkName":"PinchBench","category":"agentic","score":90.35,"unit":"percent","family":"pinchBench","index":50.21125307795807,"aboveFit":false,"run":null,"source":"Kilo Code","sourceUrl":"https://pinchbench.com/about","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gpqaDiamond","benchmarkName":"GPQA Diamond","category":"knowledge","score":88.83,"unit":"percent","family":"gpqa-diamond","index":58.68154945097403,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"browseComp","benchmarkName":"BrowseComp","category":"agentic","score":70,"unit":"percent","family":"browseComp","index":55.21038047054997,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":67.38,"unit":"percent","family":"lcr","index":53.52615527313249,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mcpAtlas","benchmarkName":"MCP Atlas","category":"agentic","score":55.6,"unit":"percent","family":"mcpAtlas","index":48.50307845320006,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"terminalBench21","benchmarkName":"Terminal-Bench 2.1","category":"agentic","score":53.93,"unit":"percent","family":"terminal-bench-2-1","index":52.6823710133827,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"webArena","benchmarkName":"WebArena Web Agent Benchmark","category":"agentic","score":50.25,"unit":"percent","family":"webArena","index":null,"aboveFit":false,"run":null,"source":"Shuyan Zhou, Frank F. Xu, Hao Zhu, Xuhui Zhou, Robert Lo, Abishek Sridhar, Xianyi Cheng, Tianyue Ou, Yonatan Bisk, Daniel Fried, Uri Alon, Graham Neubig","sourceUrl":"https://arxiv.org/abs/2307.13854","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hleNoTools","benchmarkName":"Humanity's Last Exam without tools","category":"knowledge","score":28.73,"unit":"percent","family":"hle","index":53.21124487801947,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"omniscienceAccuracy","benchmarkName":"Artificial Analysis Omniscience Accuracy","category":"knowledge","score":21.8,"unit":"percent","family":"omniscienceAccuracy","index":38.16353091912248,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"tau3Bench","benchmarkName":"τ³-Bench Tool-Agent-User Evaluation","category":"agentic","score":17.01,"unit":"percent","family":"tau3Bench","index":5,"aboveFit":false,"run":null,"source":"Sierra Research","sourceUrl":"https://github.com/sierra-research/tau2-bench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":50,"benchmarkCount":9,"categoryCount":3,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"nemotron-3-5-bolt-e4b","providerId":"nvidia","slug":"nemotron-3-5-bolt-e4b","name":"Nemotron 3.5 Bolt E4B","provider":"NVIDIA","description":"Nemotron 3.5 Bolt E4B is a reasoning model from NVIDIA. It has 3 scored results in 3 categories. It has no index.","logoUrl":null,"releaseDate":"2026-10-23","contextWindowTokens":64000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":true,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":37.50758536107729,"coding":null,"reasoning":38.0644416516831,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":46.12222795814753,"math":null},"capabilityCounts":{"instruction":1,"agentic":1,"reasoning":1},"benchmarks":[{"benchmarkId":"ruler64k","benchmarkName":"RULER at a 64K-token context","category":"reasoning","score":85.76,"unit":"percent","family":"ruler64k","index":null,"aboveFit":false,"run":null,"source":"RULER authors","sourceUrl":"https://github.com/NVIDIA/RULER","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"brevityBench","benchmarkName":"Brevity Benchmark (NVIDIA)","category":"instruction","score":82.84,"unit":"percent","family":"brevityBench","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"ihEval","benchmarkName":"IHEval: instruction hierarchy evaluation","category":"instruction","score":82.4,"unit":"percent","family":"ihEval","index":null,"aboveFit":false,"run":null,"source":"IHEval authors","sourceUrl":"https://github.com/ytyz1307zzh/IHEval","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"structEvalT","benchmarkName":"StructEval text-format subset","category":"instruction","score":79.99,"unit":"percent","family":"structEvalT","index":null,"aboveFit":false,"run":null,"source":"StructEval authors","sourceUrl":"https://structeval.github.io/","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"roleMrc","benchmarkName":"RoleMRC","category":"instruction","score":79.12,"unit":"percent","family":"roleMrc","index":null,"aboveFit":false,"run":null,"source":"RoleMRC authors","sourceUrl":"https://github.com/LuJunru/RoleMRC","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"ragTruthProvider","benchmarkName":"RAGTruth: provider-reported F1","category":"knowledge","score":77.88,"unit":"percent","family":"ragTruthProvider","index":null,"aboveFit":false,"run":null,"source":"RAGTruth authors","sourceUrl":"https://github.com/ParticleMedia/RAGTruth","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"toolSandbox","benchmarkName":"ToolSandbox","category":"agentic","score":74.13,"unit":"percent","family":"toolSandbox","index":null,"aboveFit":false,"run":null,"source":"Apple","sourceUrl":"https://github.com/apple-aiml-research/ToolSandbox","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"roleBench","benchmarkName":"RoleBench (RoleLLM)","category":"instruction","score":71.21,"unit":"percent","family":"roleBench","index":null,"aboveFit":false,"run":null,"source":"RoleLLM authors","sourceUrl":"https://github.com/InteractiveNLP-Team/RoleLLM-public","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"longMemEval","benchmarkName":"LongMemEval","category":"reasoning","score":69.92,"unit":"percent","family":"longMemEval","index":null,"aboveFit":false,"run":null,"source":"LongMemEval authors","sourceUrl":"https://github.com/xiaowu0162/LongMemEval","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"relyToolBench","benchmarkName":"RelyToolBench","category":"agentic","score":69.29,"unit":"percent","family":"relyToolBench","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"bfclV3","benchmarkName":"Berkeley Function Calling Leaderboard v3","category":"agentic","score":67.31,"unit":"percent","family":"bfclV3","index":null,"aboveFit":false,"run":null,"source":"Shishir G. Patil et al.","sourceUrl":"https://gorilla.cs.berkeley.edu/leaderboard.html","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"sysBench","benchmarkName":"SysBench: system message following","category":"instruction","score":67.22,"unit":"percent","family":"sysBench","index":null,"aboveFit":false,"run":null,"source":"SysBench authors","sourceUrl":"https://github.com/PKU-Baichuan-MLSystemLab/SysBench","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaIfBench","benchmarkName":"Artificial Analysis IFBench","category":"instruction","score":61.1,"unit":"percent","family":"ifbench-family","index":46.12222795814753,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"when2CallMultiTurn","benchmarkName":"When2Call Multi-Turn (NVIDIA)","category":"agentic","score":60.5,"unit":"percent","family":"when2CallMultiTurn","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"multiChallenge","benchmarkName":"MultiChallenge: provider-reported runs","category":"instruction","score":55.6,"unit":"percent","family":"multiChallenge","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"tau2Bench","benchmarkName":"τ²-Bench Tool-Agent-User Evaluation","category":"agentic","score":52.32,"unit":"percent","family":"tau2-bench","index":37.50758536107729,"aboveFit":false,"run":null,"source":"Victor Barres, Honghua Dong, Soham Ray, Xujie Si, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2506.07982","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"longBenchV2","benchmarkName":"LongBench v2","category":"reasoning","score":48.67,"unit":"percent","family":"longBenchV2","index":38.0644416516831,"aboveFit":false,"run":null,"source":"LongBench v2 authors","sourceUrl":"https://arxiv.org/abs/2412.15204","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"inverseIfEval","benchmarkName":"InverseIFEval","category":"instruction","score":45.26,"unit":"percent","family":"inverseIfEval","index":null,"aboveFit":false,"run":null,"source":"InverseIFEval authors","sourceUrl":"https://arxiv.org/abs/2509.04292","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"garage","benchmarkName":"GaRAGe","category":"knowledge","score":40.73,"unit":"percent","family":"garage","index":null,"aboveFit":false,"run":null,"source":"Amazon Science","sourceUrl":"https://github.com/amazon-science/GaRAGe","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"eqBench3","benchmarkName":"EQ-Bench 3","category":"instruction","score":38.15,"unit":"percent","family":"eqBench3","index":null,"aboveFit":false,"run":null,"source":"EQ-Bench","sourceUrl":"https://eqbench.com/eqbench3.html","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"agentIf","benchmarkName":"AgentIF: instruction following in agentic scenarios","category":"instruction","score":22.95,"unit":"percent","family":"agentIf","index":null,"aboveFit":false,"run":null,"source":"AgentIF authors","sourceUrl":"https://agentif.github.io/","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":40,"benchmarkCount":3,"categoryCount":3,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"nemotron-3-5-bolt-e4b-bf16","providerId":"nvidia","slug":"nemotron-3-5-bolt-e4b-bf16","name":"Nemotron 3.5 Bolt E4B BF16","provider":"NVIDIA","description":"Nemotron 3.5 Bolt E4B BF16 is a reasoning model from NVIDIA in the Nemotron 3.5 Bolt E4B family. It has 2 scored results in 2 categories. It has no index.","logoUrl":null,"releaseDate":"2026-10-23","contextWindowTokens":64000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":true,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":39.35231861221936,"coding":null,"reasoning":40.23244205506161,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"agentic":1,"reasoning":1},"benchmarks":[{"benchmarkId":"ruler64k","benchmarkName":"RULER at a 64K-token context","category":"reasoning","score":86.99,"unit":"percent","family":"ruler64k","index":null,"aboveFit":false,"run":null,"source":"RULER authors","sourceUrl":"https://github.com/NVIDIA/RULER","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"ihEval","benchmarkName":"IHEval: instruction hierarchy evaluation","category":"instruction","score":84.79,"unit":"percent","family":"ihEval","index":null,"aboveFit":false,"run":null,"source":"IHEval authors","sourceUrl":"https://github.com/ytyz1307zzh/IHEval","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"brevityBench","benchmarkName":"Brevity Benchmark (NVIDIA)","category":"instruction","score":82.52,"unit":"percent","family":"brevityBench","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"roleMrc","benchmarkName":"RoleMRC","category":"instruction","score":80.63,"unit":"percent","family":"roleMrc","index":null,"aboveFit":false,"run":null,"source":"RoleMRC authors","sourceUrl":"https://github.com/LuJunru/RoleMRC","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"structEvalT","benchmarkName":"StructEval text-format subset","category":"instruction","score":80.4,"unit":"percent","family":"structEvalT","index":null,"aboveFit":false,"run":null,"source":"StructEval authors","sourceUrl":"https://structeval.github.io/","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"ragTruthProvider","benchmarkName":"RAGTruth: provider-reported F1","category":"knowledge","score":77.59,"unit":"percent","family":"ragTruthProvider","index":null,"aboveFit":false,"run":null,"source":"RAGTruth authors","sourceUrl":"https://github.com/ParticleMedia/RAGTruth","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"longMemEval","benchmarkName":"LongMemEval","category":"reasoning","score":72.1,"unit":"percent","family":"longMemEval","index":null,"aboveFit":false,"run":null,"source":"LongMemEval authors","sourceUrl":"https://github.com/xiaowu0162/LongMemEval","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"toolSandbox","benchmarkName":"ToolSandbox","category":"agentic","score":68.77,"unit":"percent","family":"toolSandbox","index":null,"aboveFit":false,"run":null,"source":"Apple","sourceUrl":"https://github.com/apple-aiml-research/ToolSandbox","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"bfclV3","benchmarkName":"Berkeley Function Calling Leaderboard v3","category":"agentic","score":68.51,"unit":"percent","family":"bfclV3","index":null,"aboveFit":false,"run":null,"source":"Shishir G. Patil et al.","sourceUrl":"https://gorilla.cs.berkeley.edu/leaderboard.html","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"sysBench","benchmarkName":"SysBench: system message following","category":"instruction","score":67.88,"unit":"percent","family":"sysBench","index":null,"aboveFit":false,"run":null,"source":"SysBench authors","sourceUrl":"https://github.com/PKU-Baichuan-MLSystemLab/SysBench","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"roleBench","benchmarkName":"RoleBench (RoleLLM)","category":"instruction","score":67.05,"unit":"percent","family":"roleBench","index":null,"aboveFit":false,"run":null,"source":"RoleLLM authors","sourceUrl":"https://github.com/InteractiveNLP-Team/RoleLLM-public","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"when2CallMultiTurn","benchmarkName":"When2Call Multi-Turn (NVIDIA)","category":"agentic","score":63.36,"unit":"percent","family":"when2CallMultiTurn","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"relyToolBench","benchmarkName":"RelyToolBench","category":"agentic","score":63.16,"unit":"percent","family":"relyToolBench","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"tau2Bench","benchmarkName":"τ²-Bench Tool-Agent-User Evaluation","category":"agentic","score":55.1,"unit":"percent","family":"tau2-bench","index":39.35231861221936,"aboveFit":false,"run":null,"source":"Victor Barres, Honghua Dong, Soham Ray, Xujie Si, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2506.07982","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"multiChallenge","benchmarkName":"MultiChallenge: provider-reported runs","category":"instruction","score":53.1,"unit":"percent","family":"multiChallenge","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"longBenchV2","benchmarkName":"LongBench v2","category":"reasoning","score":50.5,"unit":"percent","family":"longBenchV2","index":40.23244205506161,"aboveFit":false,"run":null,"source":"LongBench v2 authors","sourceUrl":"https://arxiv.org/abs/2412.15204","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"inverseIfEval","benchmarkName":"InverseIFEval","category":"instruction","score":46.84,"unit":"percent","family":"inverseIfEval","index":null,"aboveFit":false,"run":null,"source":"InverseIFEval authors","sourceUrl":"https://arxiv.org/abs/2509.04292","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"garage","benchmarkName":"GaRAGe","category":"knowledge","score":42.61,"unit":"percent","family":"garage","index":null,"aboveFit":false,"run":null,"source":"Amazon Science","sourceUrl":"https://github.com/amazon-science/GaRAGe","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"eqBench3","benchmarkName":"EQ-Bench 3","category":"instruction","score":39.4,"unit":"percent","family":"eqBench3","index":null,"aboveFit":false,"run":null,"source":"EQ-Bench","sourceUrl":"https://eqbench.com/eqbench3.html","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"agentIf","benchmarkName":"AgentIF: instruction following in agentic scenarios","category":"instruction","score":25.81,"unit":"percent","family":"agentIf","index":null,"aboveFit":false,"run":null,"source":"AgentIF authors","sourceUrl":"https://agentif.github.io/","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":35,"benchmarkCount":2,"categoryCount":2,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"nemotron35contentsafety","providerId":"nvidia","slug":"nvidia-nemotron-3-5-content-safety","name":"Nemotron 3.5 Content Safety","provider":"NVIDIA","description":"Nemotron 3.5 Content Safety is a model from NVIDIA. It has no scored result.","logoUrl":null,"releaseDate":"2026-06-04T14:04:24.000Z","contextWindowTokens":131072,"modalities":["text","image"],"openWeights":true,"details":{"apiId":"nvidia/nemotron-3.5-content-safety","maxOutputTokens":117964,"knowledgeCutoff":null,"retiresAt":null,"tools":false,"structuredOutput":false,"cacheReadPerMillion":null,"weightsUrl":"https://huggingface.co/nvidia/Nemotron-3.5-Content-Safety"},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":0.19999999999999998,"outputPerMillion":0.19999999999999998,"batch":null,"freeTier":true,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":178,"timeToFirstToken":0.3425,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[{"host":"vultr","name":"Vultr","kind":"cloud","url":"https://api.vultrinference.com/","inputPerMillion":0.05,"outputPerMillion":0.15,"cacheReadPerMillion":null,"contextTokens":131072,"maxOutputTokens":8192,"deprecated":false},{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":0.2,"outputPerMillion":0.2,"cacheReadPerMillion":null,"contextTokens":131072,"maxOutputTokens":117964,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"nemotron35lightning","providerId":"nvidia","slug":"nvidia-nemotron-3-5-lightning","name":"Nemotron 3.5 Lightning","provider":"NVIDIA","description":"Nemotron 3.5 Lightning is a model from NVIDIA. It has no scored result.","logoUrl":null,"releaseDate":"2026-08-11T12:52:31.000Z","contextWindowTokens":262144,"modalities":["text"],"openWeights":true,"details":{"apiId":"nvidia/nemotron-3.5-lightning","maxOutputTokens":235929,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.04,"weightsUrl":"https://huggingface.co/nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16"},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":0.07,"outputPerMillion":0.19999999999999998,"batch":null,"freeTier":true,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":191,"timeToFirstToken":0.234,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[{"host":"coreweave","name":"CoreWeave","kind":"host","url":"https://docs.wandb.ai/inference","inputPerMillion":0.07,"outputPerMillion":0.2,"cacheReadPerMillion":0.04,"contextTokens":262144,"maxOutputTokens":262144,"deprecated":false},{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":0.07,"outputPerMillion":0.2,"cacheReadPerMillion":0.04,"contextTokens":262144,"maxOutputTokens":235929,"deprecated":false},{"host":"nebius","name":"Nebius","kind":"host","url":"https://docs.tokenfactory.nebius.com/","inputPerMillion":0.06,"outputPerMillion":0.24,"cacheReadPerMillion":0.06,"contextTokens":1048576,"maxOutputTokens":1048576,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"nemotron35supervl","providerId":"nvidia","slug":"nvidia-nemotron-3-5-super-vl","name":"Nemotron 3.5 Super VL","provider":"NVIDIA","description":"Nemotron 3.5 Super VL is a model from NVIDIA. It has 2 scored results in 1 category. It has no index.","logoUrl":null,"releaseDate":"2026-10-15T15:04:22.000Z","contextWindowTokens":1048576,"modalities":["text","image","video"],"openWeights":true,"details":{"apiId":"nvidia/nemotron-3.5-super-vl","maxOutputTokens":943718,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.02,"weightsUrl":"https://huggingface.co/nvidia/NVIDIA-Nemotron-3.5-Super-VL-120B-A12B-BF16"},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":38.64900752817218,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"reasoning":2},"benchmarks":[{"benchmarkId":"arc-prize:v1-public-eval:overall","benchmarkName":"ARC-AGI-1 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":39.75,"unit":"percent","family":"arc-prize:v1-public-eval:overall","index":null,"aboveFit":false,"run":"high effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.027345701499999975,"latency":null},{"benchmarkId":"arc-prize:v1-semi-private:overall","benchmarkName":"ARC-AGI-1 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":22.666666666666668,"unit":"percent","family":"arc-prize:v1-semi-private:overall","index":37.40804386094379,"aboveFit":false,"run":"high effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.03148112700000001,"latency":null},{"benchmarkId":"arc-prize:v2-public-eval:overall","benchmarkName":"ARC-AGI-2 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":1.9444444444444442,"unit":"percent","family":"arc-prize:v2-public-eval:overall","index":null,"aboveFit":false,"run":"high effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.044093047499999996,"latency":null},{"benchmarkId":"arc-prize:v2-semi-private:overall","benchmarkName":"ARC-AGI-2 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":1.5277777777777777,"unit":"percent","family":"arc-prize:v2-semi-private:overall","index":39.88997119540056,"aboveFit":false,"run":"high effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.043333685,"latency":null}],"price":{"inputPerMillion":0.15,"outputPerMillion":0.47,"batch":null,"freeTier":true,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":135,"timeToFirstToken":1.287,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":15,"benchmarkCount":2,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[{"host":"coreweave","name":"CoreWeave","kind":"host","url":"https://docs.wandb.ai/inference","inputPerMillion":0.15,"outputPerMillion":0.47,"cacheReadPerMillion":0.02,"contextTokens":1048576,"maxOutputTokens":1048576,"deprecated":false},{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":0.15,"outputPerMillion":0.47,"cacheReadPerMillion":0.02,"contextTokens":1048576,"maxOutputTokens":943718,"deprecated":false},{"host":"together-ai","name":"Together AI","kind":"host","url":"https://docs.together.ai/docs/serverless-models","inputPerMillion":0.15,"outputPerMillion":0.5,"cacheReadPerMillion":0.02,"contextTokens":262144,"maxOutputTokens":262144,"deprecated":false},{"host":"digitalocean","name":"DigitalOcean","kind":"cloud","url":"https://docs.digitalocean.com/products/gradient-ai-platform/details/models/","inputPerMillion":0.3,"outputPerMillion":0.65,"cacheReadPerMillion":0.06,"contextTokens":1048576,"maxOutputTokens":1048576,"deprecated":false},{"host":"nebius","name":"Nebius","kind":"host","url":"https://docs.tokenfactory.nebius.com/","inputPerMillion":0.3,"outputPerMillion":0.9,"cacheReadPerMillion":0.06,"contextTokens":1048576,"maxOutputTokens":1048576,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:arc-prize-foundation","name":"ARC Prize Foundation","url":"https://arcprize.org","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"nemotroncascade230ba3b","providerId":"nvidia","slug":"nvidia-nemotron-cascade-2-30b-a3b","name":"Nemotron Cascade 2 30B A3B","provider":"NVIDIA","description":"Nemotron Cascade 2 30B A3B is a model from NVIDIA. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":45.729746335277326,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-italian:overall","benchmarkName":"EuroEval Italian","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":56.13624999999999,"unit":"percent","family":"euroeval","index":48.38421012537329,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/italian","measuredAt":null,"stderr":0.537719431953133,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-french:overall","benchmarkName":"EuroEval French","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":54.90625,"unit":"percent","family":"euroeval","index":46.71538314031031,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/french","measuredAt":null,"stderr":0.5344696553594039,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-dutch:overall","benchmarkName":"EuroEval Dutch","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":53.45333333333333,"unit":"percent","family":"euroeval","index":44.74410953024435,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/dutch","measuredAt":null,"stderr":0.5554144265187799,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":45.8475,"unit":"percent","family":"euroeval","index":34.4247437493026,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.6903271416509711,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"nemotron-diffusion-8b","providerId":"pst2154","slug":"nemotron-diffusion-8b","name":"Nemotron Diffusion 8B (optimized vLLM)","provider":"pst2154","description":"Nemotron Diffusion 8B (optimized vLLM) is a standard model from pst2154. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":67.02783709706964,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"reasoning":1},"benchmarks":[{"benchmarkId":"jevBench15","benchmarkName":"jevBench15","category":"reasoning","score":25.684702471457477,"unit":"percent","family":"jevBench15","index":67.02783709706964,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":15,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"nemotronlightning3p530ba3b","providerId":"nvidia","slug":"nvidia-nemotron-lightning-3p5-30b-a3b","name":"Nemotron Lightning 3p5 30B A3B","provider":"NVIDIA","description":"Nemotron Lightning 3p5 30B A3B is a model from NVIDIA. It has 5 scored results in 3 categories. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":27.493235320896044,"coding":43.059105729519935,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":47.156620590676994},"capabilityCounts":{"coding":3,"agentic":1,"math":1},"benchmarks":[{"benchmarkId":"vals-ai:corp_fin_v2:overall","benchmarkName":"CorpFin v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":54.235,"unit":"percent","family":"vals-ai:corp_fin_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","measuredAt":"2026-08-12","stderr":0.98,"costPerTest":0.004794,"latency":38.792},{"benchmarkId":"vals-ai:swebench:overall","benchmarkName":"SWE-bench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":52.8,"unit":"percent","family":"swe-bench-verified","index":40.170967636450584,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/swebench","measuredAt":"2026-09-01","stderr":2.235,"costPerTest":0.031537,"latency":925.464},{"benchmarkId":"vals-ai:cyber:overall","benchmarkName":"CyberBench v1.1","category":null,"domain":"Other","eligible":false,"ranked":true,"score":29.464,"unit":"percent","family":"vals-ai:cyber:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/cyber","measuredAt":"2026-11-05","stderr":5.958,"costPerTest":0.093074,"latency":3031.784},{"benchmarkId":"vals-ai:fabv2:overall","benchmarkName":"Finance Agent (v2)","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":18.5,"unit":"percent","family":"vals-ai:fabv2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/fabv2","measuredAt":"2026-10-29","stderr":0.882,"costPerTest":0.028554,"latency":539.61},{"benchmarkId":"vals-ai:emb:overall","benchmarkName":"EMB","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":16.987,"unit":"percent","family":"vals-ai:emb:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/emb","measuredAt":"2026-10-20","stderr":1.452,"costPerTest":0.182153,"latency":2359.322},{"benchmarkId":"vals-ai:terminal-bench-2-1:overall","benchmarkName":"Terminal-Bench 2.1","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":10.861,"unit":"percent","family":"terminal-bench-2-1","index":27.493235320896044,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2-1","measuredAt":"2026-10-01","stderr":2.622,"costPerTest":0.014296,"latency":1028.327},{"benchmarkId":"vals-ai:medscribe:overall","benchmarkName":"MedScribe","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":4.267,"unit":"percent","family":"vals-ai:medscribe:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medscribe","measuredAt":"2026-10-29","stderr":0.492,"costPerTest":0.006241,"latency":105.515},{"benchmarkId":"vals-ai:code-migration:overall","benchmarkName":"Code Migration","category":"coding","domain":null,"eligible":true,"ranked":true,"score":2.951,"unit":"percent","family":"vals-ai:code-migration:overall","index":46.75682696114774,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/code-migration","measuredAt":"2026-10-29","stderr":0.951,"costPerTest":0.267017,"latency":2604.747},{"benchmarkId":"vals-ai:legal_research:overall","benchmarkName":"Legal Research Bench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":2.885,"unit":"percent","family":"vals-ai:legal_research:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_research","measuredAt":"2026-11-09","stderr":1.163,"costPerTest":0.004365,"latency":285.48},{"benchmarkId":"vals-ai:tax_agent_bench:overall","benchmarkName":"Tax Agent Bench","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":2.591,"unit":"percent","family":"vals-ai:tax_agent_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench","measuredAt":"2026-11-09","stderr":1.146,"costPerTest":0.030533,"latency":1180.079},{"benchmarkId":"vals-ai:vibe-code:overall","benchmarkName":"Vibe Code Bench v1.1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":0,"unit":"percent","family":"vals-ai:vibe-code:overall","index":42.24952259096148,"aboveFit":false,"run":"OpenHands","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","measuredAt":"2026-11-09","stderr":0,"costPerTest":0.128799,"latency":959.662},{"benchmarkId":"vals-ai:programbench:overall","benchmarkName":"ProgramBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":0,"unit":"percent","family":"vals-ai:programbench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/programbench","measuredAt":"2026-11-09","stderr":0,"costPerTest":0.216901,"latency":1465.949},{"benchmarkId":"vals-ai:proof_bench:overall","benchmarkName":"ProofBench v1.1","category":"math","domain":null,"eligible":true,"ranked":true,"score":0,"unit":"percent","family":"vals-ai:proof_bench:overall","index":47.156620590676994,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/proof_bench","measuredAt":"2026-11-09","stderr":0,"costPerTest":0.015926,"latency":919.248}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":50,"benchmarkCount":5,"categoryCount":3,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"nemotronorchestrator8b","providerId":"nvidia","slug":"nvidia-nemotron-orchestrator-8b","name":"Nemotron Orchestrator 8B","provider":"NVIDIA","description":"Nemotron Orchestrator 8B is a model from NVIDIA. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":58.66229616913809,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"agentic":1},"benchmarks":[{"benchmarkId":"tau2-bench:telecom:overall","benchmarkName":"τ²-bench Telecom","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":88.6,"unit":"percent","family":"tau2-bench","index":61.582017861593144,"aboveFit":false,"run":"NVIDIA","source":"Sierra Research","sourceUrl":"https://github.com/sierra-research/tau2-bench","measuredAt":"2026-03-02","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"tau2-bench:retail:overall","benchmarkName":"τ²-bench Retail","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":84.2,"unit":"percent","family":"tau2-bench","index":58.66229616913809,"aboveFit":false,"run":"NVIDIA","source":"Sierra Research","sourceUrl":"https://github.com/sierra-research/tau2-bench","measuredAt":"2026-04-30","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"tau2-bench:airline:overall","benchmarkName":"τ²-bench Airline","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":56,"unit":"percent","family":"tau2-bench","index":39.9495344129488,"aboveFit":false,"run":"NVIDIA","source":"Sierra Research","sourceUrl":"https://github.com/sierra-research/tau2-bench","measuredAt":"2026-03-02","stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":20,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:sierra-research","name":"Sierra Research","url":"https://github.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"nemotronsupervl35","providerId":"nvidia","slug":"nvidia-nemotron-super-vl-3-5","name":"Nemotron Super VL-3.5","provider":"NVIDIA","description":"Nemotron Super VL-3.5 is a model from NVIDIA. It has 3 scored results in 3 categories. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":53.949330021774294,"coding":55.61521135680609,"reasoning":54.50233885912749,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"reasoning":1,"agentic":1,"coding":1},"benchmarks":[{"benchmarkId":"livebench:livebench-reasoning:overall","benchmarkName":"LiveBench Reasoning","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":78.8,"unit":"percent","family":"livebench","index":63.699920231768054,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Reasoning","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.013351074257425743,"latency":null},{"benchmarkId":"livebench:livebench-research:overall","benchmarkName":"LiveBench Research","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":70.9,"unit":"percent","family":"livebench-2-agentic","index":70.44084621223061,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Research","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.16677272727272727,"latency":null},{"benchmarkId":"livebench:livebench-coding-2:overall","benchmarkName":"LiveBench Coding 2.0","category":"coding","domain":null,"eligible":true,"ranked":true,"score":65.5,"unit":"percent","family":"livebench:livebench-coding-2:overall","index":55.61521135680609,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Coding","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.33072413793103445,"latency":null},{"benchmarkId":"livebench:livebench-data-analysis:overall","benchmarkName":"LiveBench Data Analysis","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":64.6,"unit":"percent","family":"livebench","index":45.30475748648695,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Data%20Analysis","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.019416926666666667,"latency":null},{"benchmarkId":"livebench:livebench-design:overall","benchmarkName":"LiveBench Design","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":60.7,"unit":"percent","family":"livebench-2-agentic","index":62.981269044041944,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Design","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.055756,"latency":null},{"benchmarkId":"livebench:livebench-browser:overall","benchmarkName":"LiveBench Browser Use","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":36,"unit":"percent","family":"livebench-2-agentic","index":44.91739099950665,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Browser","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.40927199999999997,"latency":null},{"benchmarkId":"livebench:livebench-long-horizon:overall","benchmarkName":"LiveBench Long Agentic","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":15.8,"unit":"percent","family":"livebench-2-agentic","index":30.14450288211342,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Long%20Agentic","measuredAt":"2026-10-14","stderr":null,"costPerTest":1.619232142857143,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":55.00000000000001,"benchmarkCount":3,"categoryCount":3,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:livebench","name":"LiveBench","url":"https://livebench.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"nemotronsupervl35nvfp4","providerId":"nvidia","slug":"nvidia-nemotron-super-vl-3-5-nvfp4","name":"Nemotron Super VL-3.5 Nvfp4","provider":"NVIDIA","description":"Nemotron Super VL-3.5 Nvfp4 is a model from NVIDIA. It has 3 scored results in 3 categories. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":51.17027186107656,"coding":52.573046037547144,"reasoning":50.616036870687836,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"reasoning":1,"agentic":1,"coding":1},"benchmarks":[{"benchmarkId":"livebench:livebench-reasoning:overall","benchmarkName":"LiveBench Reasoning","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":71.7,"unit":"percent","family":"livebench","index":54.502338859127505,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Reasoning","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.01204011386138614,"latency":null},{"benchmarkId":"livebench:livebench-research:overall","benchmarkName":"LiveBench Research","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":70.9,"unit":"percent","family":"livebench-2-agentic","index":70.44084621223061,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Research","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.10016363636363636,"latency":null},{"benchmarkId":"livebench:livebench-data-analysis:overall","benchmarkName":"LiveBench Data Analysis","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":65.7,"unit":"percent","family":"livebench","index":46.72973488224817,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Data%20Analysis","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.016842526666666666,"latency":null},{"benchmarkId":"livebench:livebench-coding-2:overall","benchmarkName":"LiveBench Coding 2.0","category":"coding","domain":null,"eligible":true,"ranked":true,"score":61.3,"unit":"percent","family":"livebench:livebench-coding-2:overall","index":52.573046037547144,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Coding","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.09972068965517242,"latency":null},{"benchmarkId":"livebench:livebench-design:overall","benchmarkName":"LiveBench Design","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":57.1,"unit":"percent","family":"livebench-2-agentic","index":60.348477102328296,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Design","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.038164,"latency":null},{"benchmarkId":"livebench:livebench-browser:overall","benchmarkName":"LiveBench Browser Use","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":32,"unit":"percent","family":"livebench-2-agentic","index":41.992066619824826,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Browser","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.20000800000000002,"latency":null},{"benchmarkId":"livebench:livebench-long-horizon:overall","benchmarkName":"LiveBench Long Agentic","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":11.5,"unit":"percent","family":"livebench-2-agentic","index":26.999779173955456,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Long%20Agentic","measuredAt":"2026-10-14","stderr":null,"costPerTest":1.1711379310344827,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":55.00000000000001,"benchmarkCount":3,"categoryCount":3,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:livebench","name":"LiveBench","url":"https://livebench.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"nemotron-4-15b","providerId":"nvidia","slug":"nemotron-4-15b","name":"Nemotron-4 15B","provider":"NVIDIA","description":"Nemotron-4 15B is a standard model from NVIDIA. It has no scored result.","logoUrl":null,"releaseDate":"2024-02-26","contextWindowTokens":32000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"decision-index-0-3-neohorse-jev-4b","providerId":"tokenrhythm","slug":"decision-index-0-3-neohorse-jev-4b","name":"NeoHorse-Jev-4B","provider":"TokenRhythm","description":"NeoHorse-Jev-4B is a model from TokenRhythm. It has no scored result.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"nesso04bagentic","providerId":"unknown","slug":"unknown-nesso-0-4b-agentic","name":"Nesso 0.4B Agentic","provider":"Unknown","description":"Nesso 0.4B Agentic is a model. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":5,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-french:overall","benchmarkName":"EuroEval French","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":15.567500000000003,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/french","measuredAt":null,"stderr":0.9214670707627051,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-portuguese:overall","benchmarkName":"EuroEval Portuguese","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":13.708749999999998,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/portuguese","measuredAt":null,"stderr":0.6635322147416808,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-italian:overall","benchmarkName":"EuroEval Italian","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":13.07,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/italian","measuredAt":null,"stderr":0.7635157169305684,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-spanish:overall","benchmarkName":"EuroEval Spanish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":12.357499999999998,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/spanish","measuredAt":null,"stderr":0.5975405320980326,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"nesso4b","providerId":"unknown","slug":"unknown-nesso-4b","name":"Nesso 4B","provider":"Unknown","description":"Nesso 4B is a model. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":33.70047359154633,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-french:overall","benchmarkName":"EuroEval French","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":52.0775,"unit":"percent","family":"euroeval","index":42.87742026714204,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/french","measuredAt":null,"stderr":0.4181525588825208,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-portuguese:overall","benchmarkName":"EuroEval Portuguese","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":49.628750000000004,"unit":"percent","family":"euroeval","index":39.555029958464786,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/portuguese","measuredAt":null,"stderr":0.4404330255555321,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-italian:overall","benchmarkName":"EuroEval Italian","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":49.0575,"unit":"percent","family":"euroeval","index":38.77997514934507,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/italian","measuredAt":null,"stderr":0.33258222667484805,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-dutch:overall","benchmarkName":"EuroEval Dutch","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":45.82111111111111,"unit":"percent","family":"euroeval","index":34.38894009899072,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/dutch","measuredAt":null,"stderr":0.30778379456613836,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":44.806250000000006,"unit":"percent","family":"euroeval","index":33.01200708410192,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":0.28954706007832304,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-spanish:overall","benchmarkName":"EuroEval Spanish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":43.738749999999996,"unit":"percent","family":"euroeval","index":31.563655208854136,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/spanish","measuredAt":null,"stderr":0.3477809116383474,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":43.49571428571429,"unit":"percent","family":"euroeval","index":31.233911665493125,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.2748951562664856,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":40.91625,"unit":"percent","family":"euroeval","index":27.734172147601843,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.46889497758026794,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"nexn25mini","providerId":"nex-agi","slug":"nex-agi-nex-n2-5-mini","name":"Nex N2.5 Mini","provider":"Nex AGI","description":"Nex N2.5 Mini is a model from Nex AGI. It has no scored result.","logoUrl":null,"releaseDate":"2026-09-08T17:54:21.000Z","contextWindowTokens":262144,"modalities":["text","image"],"openWeights":true,"details":{"apiId":"nex-agi/nex-n2.5-mini","maxOutputTokens":235929,"knowledgeCutoff":null,"retiresAt":null,"tools":false,"structuredOutput":true,"cacheReadPerMillion":0.0025,"weightsUrl":"https://huggingface.co/nex-agi/Nex-N2.5-mini"},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":0.024999999999999998,"outputPerMillion":0.09999999999999999,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":27,"timeToFirstToken":26.0355,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"nexn25pro","providerId":"nex-agi","slug":"nex-agi-nex-n2-5-pro","name":"Nex N2.5 Pro","provider":"Nex AGI","description":"Nex N2.5 Pro is a model from Nex AGI. It has no scored result.","logoUrl":null,"releaseDate":"2026-09-08T17:54:10.000Z","contextWindowTokens":262144,"modalities":["text","image"],"openWeights":true,"details":{"apiId":"nex-agi/nex-n2.5-pro","maxOutputTokens":235929,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.015,"weightsUrl":"https://huggingface.co/nex-agi/Nex-N2.5-Pro"},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":0.075,"outputPerMillion":0.25,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":9,"timeToFirstToken":1.723,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"nexn26","providerId":"nex-agi","slug":"nex-agi-nex-n2-6","name":"Nex N2.6","provider":"Nex AGI","description":"Nex N2.6 is a model from Nex AGI. It has no scored result.","logoUrl":null,"releaseDate":"2026-11-10T03:46:19.000Z","contextWindowTokens":262144,"modalities":["text","image"],"openWeights":true,"details":{"apiId":"nex-agi/nex-n2.6","maxOutputTokens":235929,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.024,"weightsUrl":"https://huggingface.co/nex-agi/Nex-N2.6"},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":0.12,"outputPerMillion":0.39999999999999997,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":84,"timeToFirstToken":1.585,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"next-gpt-7b","providerId":"next-gpt-authors","slug":"next-gpt-7b","name":"NExT-GPT 7B","provider":"NExT-GPT authors","description":"NExT-GPT 7B is a standard model from NExT-GPT authors. It has no scored result.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"ninfer-qwen3-8-27b","providerId":"igor-l-ninfer-contributors","slug":"ninfer-qwen3-8-27b","name":"NInfer Qwen3.8-27B NVFP4","provider":"Igor L. / NInfer contributors","description":"NInfer Qwen3.8-27B NVFP4 is a standard model from Igor L. / NInfer contributors. It has 2 scored results in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":65.0049686271586,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"reasoning":2},"benchmarks":[{"benchmarkId":"jevBench14","benchmarkName":"jevBench14","category":"reasoning","score":26.29915101653767,"unit":"percent","family":"jevBench14","index":67.18600551891384,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"jevBench15","benchmarkName":"jevBench15","category":"reasoning","score":18.742649798238652,"unit":"percent","family":"jevBench15","index":62.823931735403356,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":15,"benchmarkCount":2,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"ninfer-qwen3-8-27b-t1-5","providerId":"igor-l-ninfer-contributors","slug":"ninfer-qwen3-8-27b-t1-5","name":"NInfer Qwen3.8-27B NVFP4 (T=1.5)","provider":"Igor L. / NInfer contributors","description":"NInfer Qwen3.8-27B NVFP4 (T=1.5) is a standard model from Igor L. / NInfer contributors. It has 2 scored results in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":65.21034898022221,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"reasoning":2},"benchmarks":[{"benchmarkId":"jevBench14","benchmarkName":"jevBench14","category":"reasoning","score":26.916169238404322,"unit":"percent","family":"jevBench14","index":67.75467114379876,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"jevBench15","benchmarkName":"jevBench15","category":"reasoning","score":18.48189600644971,"unit":"percent","family":"jevBench15","index":62.66602681664565,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":15,"benchmarkCount":2,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"ninfer-qwen3-8-flash-next","providerId":"igor-l-ninfer-contributors","slug":"ninfer-qwen3-8-flash-next","name":"NInfer Qwen3.8-Flash-Next mixed","provider":"Igor L. / NInfer contributors","description":"NInfer Qwen3.8-Flash-Next mixed is a standard model from Igor L. / NInfer contributors. It has 2 scored results in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":77.24298217243181,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"reasoning":2},"benchmarks":[{"benchmarkId":"jevBench15","benchmarkName":"jevBench15","category":"reasoning","score":47.474808502448944,"unit":"percent","family":"jevBench15","index":80.2232920852393,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"jevBench14","benchmarkName":"jevBench14","category":"reasoning","score":33.977533774383616,"unit":"percent","family":"jevBench14","index":74.26267225962434,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":15,"benchmarkCount":2,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"norbloom7bscratch","providerId":"unknown","slug":"unknown-norbloom-7b-scratch","name":"Norbloom 7B Scratch","provider":"Unknown","description":"Norbloom 7B Scratch is a model. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":5,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":19.93625,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":1.1969570794310045,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"normistral11btranslate","providerId":"unknown","slug":"unknown-normistral-11b-translate","name":"Normistral 11B Translate","provider":"Unknown","description":"Normistral 11B Translate is a model. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":5,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":6.2524999999999995,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":0.6528255892656168,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"normistral7bscratch","providerId":"unknown","slug":"unknown-normistral-7b-scratch","name":"Normistral 7B Scratch","provider":"Unknown","description":"Normistral 7B Scratch is a model. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":5,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":18.26375,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":0.8727963321989844,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"norolmo13b","providerId":"unknown","slug":"unknown-norolmo-13b","name":"NorOLMo 13B","provider":"Unknown","description":"NorOLMo 13B is a model. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":11.961721983896638,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":29.29125,"unit":"percent","family":"euroeval","index":11.961721983896638,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":0.9561127352462157,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"north-micro-vision-instruct","providerId":"cohere","slug":"north-micro-vision-instruct","name":"North Micro Vision Instruct","provider":"Cohere","description":"North Micro Vision Instruct is a standard model from Cohere in the North Micro Vision family. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":"2026-08-10","contextWindowTokens":128000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":19.55851795211546,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"multimodal":1},"benchmarks":[{"benchmarkId":"realWorldQa","benchmarkName":"RealWorldQA","category":"multimodal","score":62.2,"unit":"percent","family":"realWorldQa","index":19.55851795211546,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":10,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"north-small-translate-1-0","providerId":"cohere","slug":"north-small-translate-1-0","name":"North Small Translate","provider":"Cohere","description":"North Small Translate is a standard model from Cohere. It has no scored result.","logoUrl":null,"releaseDate":"2026-09-10","contextWindowTokens":16000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[{"benchmarkId":"wmt26","benchmarkName":"WMT26 General Machine Translation (all languages)","category":"multilingual","score":83.6,"unit":"percent","family":"wmt26","index":null,"aboveFit":false,"run":null,"source":"WMT organizers","sourceUrl":"https://www2.statmt.org/wmt26/translation-task.html","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"longContextTranslation","benchmarkName":"Long-context translation evaluation (Cohere)","category":"multilingual","score":48.9,"unit":"percent","family":"longContextTranslation","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"norwaimagistral24breasoning","providerId":"unknown","slug":"unknown-norwai-magistral-24b-reasoning","name":"NorwAI Magistral 24B Reasoning","provider":"Unknown","description":"NorwAI Magistral 24B Reasoning is a model. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":5,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":1.15875,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":0.45375,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"norwaimixtral8x7b","providerId":"unknown","slug":"unknown-norwai-mixtral-8x7b","name":"NorwAI Mixtral 8x7B","provider":"Unknown","description":"NorwAI Mixtral 8x7B is a model. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":21.742337047777074,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":36.50000000000001,"unit":"percent","family":"euroeval","index":21.742337047777074,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":0.9021744565215755,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"norwaimixtral8x7bonethirdepoch","providerId":"unknown","slug":"unknown-norwai-mixtral-8x7b-one-third-epoch","name":"NorwAI Mixtral 8x7B One Third Epoch","provider":"Unknown","description":"NorwAI Mixtral 8x7B One Third Epoch is a model. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":19.95309673350727,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":35.18124999999999,"unit":"percent","family":"euroeval","index":19.95309673350727,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":0.8117592777418685,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"nova3streaming","providerId":"amazon","slug":"amazon-nova-3-streaming","name":"Nova 3 Streaming","provider":"Amazon","description":"Nova 3 Streaming is a model from Amazon. It has no scored result.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"novalitev1","providerId":"amazon","slug":"amazon-nova-lite-1-0","name":"Nova Lite 1.0","provider":"Amazon","description":"Nova Lite 1.0 is a model from Amazon. It has no scored result.","logoUrl":null,"releaseDate":"2024-12-05T22:22:43.000Z","contextWindowTokens":300000,"modalities":["text","image"],"openWeights":null,"details":{"apiId":"amazon/nova-lite-v1","maxOutputTokens":5120,"knowledgeCutoff":"2024-10-31","retiresAt":null,"tools":true,"structuredOutput":false,"cacheReadPerMillion":null,"weightsUrl":null},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":0.06,"outputPerMillion":0.24,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":125,"timeToFirstToken":0.42,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":0.06,"outputPerMillion":0.24,"cacheReadPerMillion":null,"contextTokens":300000,"maxOutputTokens":5120,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"novamicrov1","providerId":"amazon","slug":"amazon-nova-micro-1-0","name":"Nova Micro 1.0","provider":"Amazon","description":"Nova Micro 1.0 is a model from Amazon. It has no scored result.","logoUrl":null,"releaseDate":"2024-12-05T22:20:37.000Z","contextWindowTokens":128000,"modalities":["text"],"openWeights":null,"details":{"apiId":"amazon/nova-micro-v1","maxOutputTokens":5120,"knowledgeCutoff":"2024-10-31","retiresAt":null,"tools":true,"structuredOutput":false,"cacheReadPerMillion":null,"weightsUrl":null},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":0.035,"outputPerMillion":0.14,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":158,"timeToFirstToken":0.362,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":0.035,"outputPerMillion":0.14,"cacheReadPerMillion":null,"contextTokens":128000,"maxOutputTokens":5120,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"novapremierv1","providerId":"amazon","slug":"amazon-nova-premier-1-0","name":"Nova Premier 1.0","provider":"Amazon","description":"Nova Premier 1.0 is a model from Amazon. It has no scored result.","logoUrl":null,"releaseDate":"2025-10-31T22:38:52.000Z","contextWindowTokens":1000000,"modalities":["text","image"],"openWeights":null,"details":{"apiId":"amazon/nova-premier-v1","maxOutputTokens":32000,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":false,"cacheReadPerMillion":0.625,"weightsUrl":null},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":2.5,"outputPerMillion":12.5,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[{"host":"amazon-bedrock","name":"Amazon Bedrock","kind":"maker","url":"https://docs.aws.amazon.com/bedrock/latest/userguide/models-supported.html","inputPerMillion":2.5,"outputPerMillion":12.5,"cacheReadPerMillion":0.625,"contextTokens":1000000,"maxOutputTokens":10000,"deprecated":true},{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":2.5,"outputPerMillion":12.5,"cacheReadPerMillion":0.625,"contextTokens":1000000,"maxOutputTokens":32000,"deprecated":true}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"novaprov1","providerId":"amazon","slug":"amazon-nova-pro-1-0","name":"Nova Pro 1.0","provider":"Amazon","description":"Nova Pro 1.0 is a model from Amazon. It has no scored result.","logoUrl":null,"releaseDate":"2024-12-05T22:05:03.000Z","contextWindowTokens":300000,"modalities":["text","image"],"openWeights":null,"details":{"apiId":"amazon/nova-pro-v1","maxOutputTokens":5120,"knowledgeCutoff":"2024-10-31","retiresAt":null,"tools":true,"structuredOutput":false,"cacheReadPerMillion":null,"weightsUrl":null},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":0.7999999999999999,"outputPerMillion":3.1999999999999997,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":87,"timeToFirstToken":1.061,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":0.8,"outputPerMillion":3.2,"cacheReadPerMillion":null,"contextTokens":300000,"maxOutputTokens":5120,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"nvidianemotron3nano30ba3bbasebf16","providerId":"nvidia","slug":"nvidia-nvidia-nemotron-3-nano-30b-a3b-base-bf16","name":"NVIDIA Nemotron 3 Nano 30B A3B Base BF16","provider":"NVIDIA","description":"NVIDIA Nemotron 3 Nano 30B A3B Base BF16 is a model from NVIDIA. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":35.90955881578904,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-french:overall","benchmarkName":"EuroEval French","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":53.90749999999999,"unit":"percent","family":"euroeval","index":45.3603091961382,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/french","measuredAt":null,"stderr":0.6758120115831029,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-dutch:overall","benchmarkName":"EuroEval Dutch","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":50.02666666666667,"unit":"percent","family":"euroeval","index":40.094911317114914,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/dutch","measuredAt":null,"stderr":0.6824113020643673,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-portuguese:overall","benchmarkName":"EuroEval Portuguese","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":49.54,"unit":"percent","family":"euroeval","index":39.434616629257995,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/portuguese","measuredAt":null,"stderr":0.6844648913567445,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":46.9625,"unit":"percent","family":"euroeval","index":35.937542195111746,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":1.0233576781849052,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-italian:overall","benchmarkName":"EuroEval Italian","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":46.92125,"unit":"percent","family":"euroeval","index":35.88157543646634,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/italian","measuredAt":null,"stderr":0.7609081744862517,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":45.4,"unit":"percent","family":"euroeval","index":33.81758921611911,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.6226769070812285,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":43.43625,"unit":"percent","family":"euroeval","index":31.15323231212117,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.8620977322786553,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-spanish:overall","benchmarkName":"EuroEval Spanish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":38.981249999999996,"unit":"percent","family":"euroeval","index":25.10882237841736,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/spanish","measuredAt":null,"stderr":0.6121427529588176,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"nvidianemotron3nano30ba3bbf16","providerId":"nvidia","slug":"nvidia-nvidia-nemotron-3-nano-30b-a3b-bf16","name":"NVIDIA Nemotron 3 Nano 30B A3B BF16","provider":"NVIDIA","description":"NVIDIA Nemotron 3 Nano 30B A3B BF16 is a model from NVIDIA. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":48.95198064321374,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":58.22624999999999,"unit":"percent","family":"euroeval","index":51.21985923007385,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":0.8226643072967249,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-italian:overall","benchmarkName":"EuroEval Italian","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":58.1875,"unit":"percent","family":"euroeval","index":51.16728439619484,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/italian","measuredAt":null,"stderr":0.6380475589640634,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-french:overall","benchmarkName":"EuroEval French","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":58.01875,"unit":"percent","family":"euroeval","index":50.93832947446363,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/french","measuredAt":null,"stderr":0.4843713709541471,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-portuguese:overall","benchmarkName":"EuroEval Portuguese","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":57.64500000000001,"unit":"percent","family":"euroeval","index":50.43123672188861,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/portuguese","measuredAt":null,"stderr":0.5851682450372713,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-dutch:overall","benchmarkName":"EuroEval Dutch","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":55.464444444444446,"unit":"percent","family":"euroeval","index":47.47272456453888,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/dutch","measuredAt":null,"stderr":0.605816457557001,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":53.29285714285714,"unit":"percent","family":"euroeval","index":44.52638026428762,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.6510023668410531,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-spanish:overall","benchmarkName":"EuroEval Spanish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":50.78625000000001,"unit":"percent","family":"euroeval","index":41.12549112530254,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/spanish","measuredAt":null,"stderr":0.6168202229985654,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":46.715,"unit":"percent","family":"euroeval","index":35.60174164323932,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.6771205117997829,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":{"elo":1348.0898421732,"eloUpper":1353.4895935795957,"eloLower":1342.6900907668041,"votes":15663,"rank":214,"rankUpper":null,"rankLower":null,"arenaName":"NVIDIA-Nemotron-3-Nano-30B-A3B-bf16","organization":"nvidia","updatedAt":"2026-11-09"},"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"nvidianemotron3nano30ba3bfp8","providerId":"nvidia","slug":"nvidia-nvidia-nemotron-3-nano-30b-a3b-fp8","name":"NVIDIA Nemotron 3 Nano 30B A3B FP8","provider":"NVIDIA","description":"NVIDIA Nemotron 3 Nano 30B A3B FP8 is a model from NVIDIA. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":47.022540771933244,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-french:overall","benchmarkName":"EuroEval French","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":57.749999999999986,"unit":"percent","family":"euroeval","index":50.57369756207688,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/french","measuredAt":null,"stderr":0.4658812214502748,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":57.30875,"unit":"percent","family":"euroeval","index":49.97502284080939,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":0.7589003146000138,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-italian:overall","benchmarkName":"EuroEval Italian","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":57.076249999999995,"unit":"percent","family":"euroeval","index":49.65957383753527,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/italian","measuredAt":null,"stderr":0.5810053248465112,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-portuguese:overall","benchmarkName":"EuroEval Portuguese","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":56.1075,"unit":"percent","family":"euroeval","index":48.34520299055985,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/portuguese","measuredAt":null,"stderr":0.679168839464827,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-dutch:overall","benchmarkName":"EuroEval Dutch","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":54.15777777777778,"unit":"percent","family":"euroeval","index":45.69987855330664,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/dutch","measuredAt":null,"stderr":0.5389725021044129,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":52.49571428571429,"unit":"percent","family":"euroeval","index":43.444840824490704,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.48069677318981663,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-spanish:overall","benchmarkName":"EuroEval Spanish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":48.737500000000004,"unit":"percent","family":"euroeval","index":38.345808779247385,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/spanish","measuredAt":null,"stderr":0.6685782489731475,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":17.244999999999997,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":1.1125301962194103,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"nvidianemotron3nano4bbf16","providerId":"nvidia","slug":"nvidia-nvidia-nemotron-3-nano-4b-bf16","name":"NVIDIA Nemotron 3 Nano 4B BF16","provider":"NVIDIA","description":"NVIDIA Nemotron 3 Nano 4B BF16 is a model from NVIDIA. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":15.473494758697242,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-french:overall","benchmarkName":"EuroEval French","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":39.44,"unit":"percent","family":"euroeval","index":25.731240573049604,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/french","measuredAt":null,"stderr":0.38254084749213385,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-italian:overall","benchmarkName":"EuroEval Italian","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":35.90875,"unit":"percent","family":"euroeval","index":20.940146840526246,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/italian","measuredAt":null,"stderr":0.46041591251823605,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-spanish:overall","benchmarkName":"EuroEval Spanish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":32.33875,"unit":"percent","family":"euroeval","index":16.096478274123875,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/spanish","measuredAt":null,"stderr":0.41868656235900387,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-portuguese:overall","benchmarkName":"EuroEval Portuguese","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":32.222500000000004,"unit":"percent","family":"euroeval","index":15.938753772486827,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/portuguese","measuredAt":null,"stderr":0.42623533699119787,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-dutch:overall","benchmarkName":"EuroEval Dutch","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":31.536666666666665,"unit":"percent","family":"euroeval","index":15.008235744907651,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/dutch","measuredAt":null,"stderr":0.4392867395064104,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":31.375714285714285,"unit":"percent","family":"euroeval","index":14.78986039804304,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.3569628117599813,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":31.165000000000003,"unit":"percent","family":"euroeval","index":14.503969596304614,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":0.3976611308639556,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":30.921249999999997,"unit":"percent","family":"euroeval","index":14.173256931581754,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.5515305408587996,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"nvidianemotron3nano4bfp8","providerId":"nvidia","slug":"nvidia-nvidia-nemotron-3-nano-4b-fp8","name":"NVIDIA Nemotron 3 Nano 4B FP8","provider":"NVIDIA","description":"NVIDIA Nemotron 3 Nano 4B FP8 is a model from NVIDIA. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":14.633710718618964,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-french:overall","benchmarkName":"EuroEval French","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":39.2775,"unit":"percent","family":"euroeval","index":25.510765463234378,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/french","measuredAt":null,"stderr":0.6531175238806566,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-italian:overall","benchmarkName":"EuroEval Italian","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":34.92375,"unit":"percent","family":"euroeval","index":19.60372848256929,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/italian","measuredAt":null,"stderr":0.7750745931844237,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-dutch:overall","benchmarkName":"EuroEval Dutch","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":32.67222222222222,"unit":"percent","family":"euroeval","index":16.5489233499071,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/dutch","measuredAt":null,"stderr":0.7747632055611302,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-spanish:overall","benchmarkName":"EuroEval Spanish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":31.547500000000003,"unit":"percent","family":"euroeval","index":15.022934085562007,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/spanish","measuredAt":null,"stderr":0.7244685552182372,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-portuguese:overall","benchmarkName":"EuroEval Portuguese","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":30.97375,"unit":"percent","family":"euroeval","index":14.244487351675907,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/portuguese","measuredAt":null,"stderr":0.8147171671813477,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":29.851250000000004,"unit":"percent","family":"euroeval","index":12.721513131567605,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":1.066310080370621,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":28.349999999999998,"unit":"percent","family":"euroeval","index":10.684662309351474,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.979920277369542,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":27.457142857142856,"unit":"percent","family":"euroeval","index":9.473260607069975,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.7542329527001427,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"nvidianemotron35supervl120ba12bnvfp4","providerId":"nvidia","slug":"nvidia-nvidia-nemotron-3-5-super-vl-120b-a12b-nvfp4","name":"NVIDIA-Nemotron-3.5-Super-VL-120B-A12B (NVFP4)","provider":"NVIDIA","description":"NVIDIA-Nemotron-3.5-Super-VL-120B-A12B (NVFP4) is a model from NVIDIA. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":51.76837577267971,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"agentic":1},"benchmarks":[{"benchmarkId":"lmarena:agent-steerability:overall","benchmarkName":"Agent Arena steerability","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":-17.017192619394194,"unit":"index","family":"agent-arena","index":52.54000116961423,"aboveFit":false,"run":null,"source":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset","measuredAt":"2026-11-11","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lmarena:agent-task-outcome:overall","benchmarkName":"Agent Arena task outcome","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":-18.894268680419447,"unit":"index","family":"agent-arena","index":50.99675037574519,"aboveFit":false,"run":null,"source":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset","measuredAt":"2026-11-11","stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":{"elo":1407.6908946406938,"eloUpper":1417.2282792550263,"eloLower":1398.1535100263613,"votes":4061,"rank":146,"rankUpper":null,"rankLower":null,"arenaName":"NVIDIA-Nemotron-3.5-Super-VL-120B-A12B-nvfp4","organization":"nvidia","updatedAt":"2026-11-09"},"quality":{"coveragePercent":20,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:lmarena","name":"LMArena","url":"https://huggingface.co","license":"CC BY 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"o1-preview","providerId":"openai","slug":"o1-preview","name":"o1-preview","provider":"OpenAI","description":"o1-preview is a reasoning model from OpenAI in the o1 family. It has 4 scored results in 3 categories. It has no index.","logoUrl":null,"releaseDate":"2024-09-12","contextWindowTokens":200000,"modalities":["text"],"openWeights":false,"details":null,"reasoning":true,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":48.87414705429815,"reasoning":null,"multimodal":null,"knowledge":35.36502821486554,"multilingual":null,"instruction":null,"math":32.83875006312516},"capabilityCounts":{"math":2,"knowledge":1,"coding":1},"benchmarks":[{"benchmarkId":"vals-ai:medqa:overall","benchmarkName":"MedQA","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":93.008,"unit":"percent","family":"vals-ai:medqa:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medqa","measuredAt":"2026-04-16","stderr":0.456,"costPerTest":0.108667,"latency":16.443},{"benchmarkId":"epoch-ai:epoch-math-level-5:overall","benchmarkName":"MATH level 5","category":"math","domain":null,"eligible":true,"ranked":true,"score":81.64652567975831,"unit":"percent","family":"epoch-ai:epoch-math-level-5:overall","index":38.658150049100676,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.0642608258161583,"costPerTest":null,"latency":null},{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":76.5,"unit":"percent","family":"gpqa-diamond","index":47.372810369213084,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"swebench-experiments:swe-bench-verified:overall","benchmarkName":"SWE-bench Verified","category":"coding","domain":null,"eligible":true,"ranked":true,"score":64.60000000000001,"unit":"percent","family":"swe-bench-verified","index":48.87414705429815,"aboveFit":false,"run":"W&B Programmer O1 crosscheck5","source":"SWE-bench team","sourceUrl":"https://github.com/SWE-bench/experiments/tree/main/evaluation/verified","measuredAt":"2026-09-01","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":50.31565656565656,"unit":"percent","family":"gpqa-diamond","index":23.357246060518,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.8257821072909395,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":31.11111111111111,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":27.01935007714964,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":6.979205927323109,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":15,"outputPerMillion":60,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":{"elo":1352.7795921934762,"eloUpper":1357.8083602227457,"eloLower":1347.750824164207,"votes":31122,"rank":210,"rankUpper":null,"rankLower":null,"arenaName":"o1-preview","organization":"openai","updatedAt":"2026-11-09"},"quality":{"coveragePercent":45,"benchmarkCount":4,"categoryCount":3,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:swe-bench-team","name":"SWE-bench team","url":"https://github.com","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"o1-pro","providerId":"openai","slug":"o1-pro","name":"o1-pro","provider":"OpenAI","description":"o1-pro is a reasoning model from OpenAI in the o1 family. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":"2025-03-19","contextWindowTokens":200000,"modalities":["text","image","file"],"openWeights":false,"details":{"apiId":"openai/o1-pro","maxOutputTokens":100000,"knowledgeCutoff":"2023-10-31","retiresAt":null,"tools":false,"structuredOutput":true,"cacheReadPerMillion":null,"weightsUrl":null},"reasoning":true,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":49.66574205651255,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"knowledge":1},"benchmarks":[{"benchmarkId":"gpqa","benchmarkName":"Graduate-Level Google-Proof Q&A","category":"knowledge","score":79,"unit":"percent","family":"gpqa-diamond","index":49.66574205651255,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":150,"outputPerMillion":600,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":14.5,"timeToFirstToken":8.732,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":15,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"o3minihigh","providerId":"openai","slug":"openai-o3-mini-high","name":"o3 Mini High","provider":"OpenAI","description":"o3 Mini High is a model from OpenAI. It has no scored result.","logoUrl":null,"releaseDate":"2025-02-12T15:03:31.000Z","contextWindowTokens":200000,"modalities":["text","file"],"openWeights":null,"details":{"apiId":"openai/o3-mini-high","maxOutputTokens":100000,"knowledgeCutoff":"2023-10-31","retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.55,"weightsUrl":null},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":1.1,"outputPerMillion":4.4,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":100.5,"timeToFirstToken":2.225,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":{"elo":1336.5860412131042,"eloUpper":1341.8347561518863,"eloLower":1331.3373262743219,"votes":18589,"rank":224,"rankUpper":null,"rankLower":null,"arenaName":"o3-mini-high","organization":"openai","updatedAt":"2026-11-09"},"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"o3-pro","providerId":"openai","slug":"o3-pro","name":"o3-pro","provider":"OpenAI","description":"o3-pro is a reasoning model from OpenAI in the o3 family. It has 3 scored results in 2 categories. It has no index.","logoUrl":null,"releaseDate":"2025-06-10","contextWindowTokens":200000,"modalities":["text","file","image"],"openWeights":false,"details":{"apiId":"openai/o3-pro","maxOutputTokens":100000,"knowledgeCutoff":"2024-06-30","retiresAt":"2026-12-11","tools":true,"structuredOutput":true,"cacheReadPerMillion":null,"weightsUrl":null},"reasoning":true,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":47.8417059493242,"multimodal":null,"knowledge":54.71019176857136,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"knowledge":1,"reasoning":2},"benchmarks":[{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":84.5,"unit":"percent","family":"gpqa-diamond","index":54.71019176857136,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"arc-prize:v1-public-eval:overall","benchmarkName":"ARC-AGI-1 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":63.339999999999996,"unit":"percent","family":"arc-prize:v1-public-eval:overall","index":null,"aboveFit":false,"run":"high effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":3.9228,"latency":null},{"benchmarkId":"arc-prize:v1-semi-private:overall","benchmarkName":"ARC-AGI-1 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":59.330000000000005,"unit":"percent","family":"arc-prize:v1-semi-private:overall","index":54.26350184966174,"aboveFit":false,"run":"high effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":4.16,"latency":null},{"benchmarkId":"arc-prize:v2-semi-private:overall","benchmarkName":"ARC-AGI-2 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":4.859999999999999,"unit":"percent","family":"arc-prize:v2-semi-private:overall","index":41.419910048986665,"aboveFit":false,"run":"high effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":7.5516,"latency":null},{"benchmarkId":"arc-prize:v2-public-eval:overall","benchmarkName":"ARC-AGI-2 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":3.8899999999999997,"unit":"percent","family":"arc-prize:v2-public-eval:overall","index":null,"aboveFit":false,"run":"high effort","source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":9.1492,"latency":null}],"price":{"inputPerMillion":20,"outputPerMillion":80,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":15,"timeToFirstToken":5.729,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":30,"benchmarkCount":3,"categoryCount":2,"confidence":"unranked","state":"partial"},"hosts":[{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":20,"outputPerMillion":80,"cacheReadPerMillion":null,"contextTokens":200000,"maxOutputTokens":100000,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:arc-prize-foundation","name":"ARC Prize Foundation","url":"https://arcprize.org","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"o4-mini-high","providerId":"openai","slug":"o4-mini-high","name":"o4-mini (high)","provider":"OpenAI","description":"o4-mini (high) is a reasoning model from OpenAI in the o4-mini family. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":"2025-04-16","contextWindowTokens":200000,"modalities":["image","text","file"],"openWeights":false,"details":{"apiId":"openai/o4-mini-high","maxOutputTokens":100000,"knowledgeCutoff":"2024-06-30","retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.275,"weightsUrl":null},"reasoning":true,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":49.30863493410469},"capabilityCounts":{"math":1},"benchmarks":[{"benchmarkId":"aime2025","benchmarkName":"American Invitational Mathematics Examination 2025","category":"math","score":92.7,"unit":"percent","family":"aime2025","index":49.30863493410469,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":1.1,"outputPerMillion":4.4,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":136,"timeToFirstToken":3.346,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":{"elo":1353.170256392411,"eloUpper":1357.1303130512374,"eloLower":1349.210199733585,"votes":44801,"rank":209,"rankUpper":null,"rankLower":null,"arenaName":"o4-mini-2025-04-16","organization":"openai","updatedAt":"2026-11-09"},"quality":{"coveragePercent":10,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"occiglot7beu5","providerId":"unknown","slug":"unknown-occiglot-7b-eu5","name":"Occiglot 7B Eu5","provider":"Unknown","description":"Occiglot 7B Eu5 is a model. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":14.821962543153504,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-french:overall","benchmarkName":"EuroEval French","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":37.446250000000006,"unit":"percent","family":"euroeval","index":23.02618057185501,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/french","measuredAt":null,"stderr":0.41500188252585074,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-dutch:overall","benchmarkName":"EuroEval Dutch","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":35.217777777777776,"unit":"percent","family":"euroeval","index":20.002656523149504,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/dutch","measuredAt":null,"stderr":0.40312513757295004,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-portuguese:overall","benchmarkName":"EuroEval Portuguese","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":32.43375,"unit":"percent","family":"euroeval","index":16.225371415246634,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/portuguese","measuredAt":null,"stderr":0.47865207875867416,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-italian:overall","benchmarkName":"EuroEval Italian","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":31.78625,"unit":"percent","family":"euroeval","index":15.346862900752079,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/italian","measuredAt":null,"stderr":0.4131207450612956,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":31.012499999999996,"unit":"percent","family":"euroeval","index":14.297062185554921,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.717586014008077,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":30.9875,"unit":"percent","family":"euroeval","index":14.263142937891047,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":0.38400032552069535,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":29.417142857142856,"unit":"percent","family":"euroeval","index":12.13252962391833,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.46005101757904004,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-spanish:overall","benchmarkName":"EuroEval Spanish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":28.839999999999996,"unit":"percent","family":"euroeval","index":11.349479563563563,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/spanish","measuredAt":null,"stderr":0.43023975873924064,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"occiglot7beu5instruct","providerId":"unknown","slug":"unknown-occiglot-7b-eu5-instruct","name":"Occiglot 7B Eu5 Instruct","provider":"Unknown","description":"Occiglot 7B Eu5 Instruct is a model. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":16.007076828495514,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-french:overall","benchmarkName":"EuroEval French","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":37.706250000000004,"unit":"percent","family":"euroeval","index":23.378940747559383,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/french","measuredAt":null,"stderr":0.33192148921092773,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-portuguese:overall","benchmarkName":"EuroEval Portuguese","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":36.63125,"unit":"percent","family":"euroeval","index":21.920413098012446,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/portuguese","measuredAt":null,"stderr":0.3501361342392413,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-dutch:overall","benchmarkName":"EuroEval Dutch","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":34.803333333333335,"unit":"percent","family":"euroeval","index":19.440350772988264,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/dutch","measuredAt":null,"stderr":0.3972311576306579,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":32.59571428571429,"unit":"percent","family":"euroeval","index":16.4451196840405,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.33260566834052135,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":31.950000000000003,"unit":"percent","family":"euroeval","index":15.569033972950514,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":0.33954703282461474,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-spanish:overall","benchmarkName":"EuroEval Spanish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":30.26125,"unit":"percent","family":"euroeval","index":13.277788793255269,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/spanish","measuredAt":null,"stderr":0.3761960094153047,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":29.44625,"unit":"percent","family":"euroeval","index":12.172021319412714,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.4817562272560677,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-italian:overall","benchmarkName":"EuroEval Italian","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":27.9775,"unit":"percent","family":"euroeval","index":10.179265519159635,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/italian","measuredAt":null,"stderr":0.4558217167928707,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"oellm9b128ktheta32mv3","providerId":"unknown","slug":"unknown-oellm-9b-128k-theta32m-v3","name":"Oellm 9B 128K Theta32m V3","provider":"Unknown","description":"Oellm 9B 128K Theta32m V3 is a model. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":13.438299518807561,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-dutch:overall","benchmarkName":"EuroEval Dutch","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":35.08555555555555,"unit":"percent","family":"euroeval","index":19.823261391060523,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/dutch","measuredAt":null,"stderr":0.6665277633071702,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-french:overall","benchmarkName":"EuroEval French","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":33.70625,"unit":"percent","family":"euroeval","index":17.951861121338226,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/french","measuredAt":null,"stderr":0.7643921195956954,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-portuguese:overall","benchmarkName":"EuroEval Portuguese","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":32.223749999999995,"unit":"percent","family":"euroeval","index":15.940449734870015,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/portuguese","measuredAt":null,"stderr":0.6885287757240071,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":32.146249999999995,"unit":"percent","family":"euroeval","index":15.83530006711198,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":0.9787611749042766,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":28.61285714285714,"unit":"percent","family":"euroeval","index":11.04129897050315,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.5431841120576307,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-italian:overall","benchmarkName":"EuroEval Italian","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":25.935,"unit":"percent","family":"euroeval","index":7.408062985020457,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/italian","measuredAt":null,"stderr":0.8159388763871961,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":25.8225,"unit":"percent","family":"euroeval","index":7.255426370532994,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.690887110894392,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-spanish:overall","benchmarkName":"EuroEval Spanish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":24.896250000000006,"unit":"percent","family":"euroeval","index":5.998718244586172,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/spanish","measuredAt":null,"stderr":0.6036969438385456,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"ola-omni","providerId":"ola-omni","slug":"ola-omni","name":"Ola","provider":"Ola-Omni","description":"Ola is a standard model from Ola-Omni. It has no scored result.","logoUrl":null,"releaseDate":"2025-02-06","contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"olmo3112532b","providerId":"allen-institute-for-ai","slug":"allen-institute-for-ai-olmo-3-32b","name":"Olmo 3 32B","provider":"Allen Institute for AI","description":"Olmo 3 32B is a model from Allen Institute for AI. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":22.148520038552057,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":44.02,"unit":"percent","family":"euroeval","index":31.945246745072822,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":0.3576332793519082,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-italian:overall","benchmarkName":"EuroEval Italian","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":41.55,"unit":"percent","family":"euroeval","index":28.594025075881255,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/italian","measuredAt":null,"stderr":0.532769884659409,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":38.477500000000006,"unit":"percent","family":"euroeval","index":24.42534953799015,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.578948400118698,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-spanish:overall","benchmarkName":"EuroEval Spanish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":35.12125,"unit":"percent","family":"euroeval","index":19.871690539113963,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/spanish","measuredAt":null,"stderr":0.27709147857702154,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":34.097142857142856,"unit":"percent","family":"euroeval","index":18.48221278659707,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.4133585684638084,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-portuguese:overall","benchmarkName":"EuroEval Portuguese","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":33.6175,"unit":"percent","family":"euroeval","index":17.83144779213145,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/portuguese","measuredAt":null,"stderr":0.3952708723141638,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"olmo310257b","providerId":"allen-institute-for-ai","slug":"allen-institute-for-ai-olmo-3-7b","name":"Olmo 3 7B","provider":"Allen Institute for AI","description":"Olmo 3 7B is a model from Allen Institute for AI. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":17.461001151573768,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":35.73500000000001,"unit":"percent","family":"euroeval","index":20.70440806926228,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":0.3254996159751959,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-italian:overall","benchmarkName":"EuroEval Italian","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":35.223749999999995,"unit":"percent","family":"euroeval","index":20.010759454535872,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/italian","measuredAt":null,"stderr":0.37317891151564286,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":33.441428571428574,"unit":"percent","family":"euroeval","index":17.592559376441542,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.35822122910234094,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":33.2475,"unit":"percent","family":"euroeval","index":17.329442926705994,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.6214951729498791,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-spanish:overall","benchmarkName":"EuroEval Spanish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":30.90625,"unit":"percent","family":"euroeval","index":14.152905382983427,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/spanish","measuredAt":null,"stderr":0.39750786155747914,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-portuguese:overall","benchmarkName":"EuroEval Portuguese","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":30.884999999999998,"unit":"percent","family":"euroeval","index":14.12407402246913,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/portuguese","measuredAt":null,"stderr":0.4301979922082389,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"olmo37binstruct","providerId":"allen-institute-for-ai","slug":"allen-institute-for-ai-olmo-3-7b-instruct","name":"Olmo 3 7B Instruct","provider":"Allen Institute for AI","description":"Olmo 3 7B Instruct is a model from Allen Institute for AI. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":12.351793332031285,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-italian:overall","benchmarkName":"EuroEval Italian","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":31.051249999999996,"unit":"percent","family":"euroeval","index":14.349637019433942,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/italian","measuredAt":null,"stderr":0.4415314258351267,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-portuguese:overall","benchmarkName":"EuroEval Portuguese","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":30.601249999999997,"unit":"percent","family":"euroeval","index":13.739090561484062,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/portuguese","measuredAt":null,"stderr":0.3882090287461125,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":30.10125,"unit":"percent","family":"euroeval","index":13.060705608206419,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":0.32615899113775787,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-spanish:overall","benchmarkName":"EuroEval Spanish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":29.05625,"unit":"percent","family":"euroeval","index":11.642881055856144,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/spanish","measuredAt":null,"stderr":0.3502164509271373,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":28.87875,"unit":"percent","family":"euroeval","index":11.402054397442583,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.44686232499507045,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":27.574285714285715,"unit":"percent","family":"euroeval","index":9.632196510409308,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.3588587352102978,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"olmo37binstructdpo","providerId":"allen-institute-for-ai","slug":"allen-institute-for-ai-olmo-3-7b-instruct-dpo","name":"Olmo 3 7B Instruct DPO","provider":"Allen Institute for AI","description":"Olmo 3 7B Instruct DPO is a model from Allen Institute for AI. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":11.765838328637713,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-portuguese:overall","benchmarkName":"EuroEval Portuguese","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":30.265,"unit":"percent","family":"euroeval","index":13.282876680404854,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/portuguese","measuredAt":null,"stderr":0.3739652390263031,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":29.61875,"unit":"percent","family":"euroeval","index":12.406064128293494,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":0.3769553189702992,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-italian:overall","benchmarkName":"EuroEval Italian","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":29.53625,"unit":"percent","family":"euroeval","index":12.294130611002686,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/italian","measuredAt":null,"stderr":0.6074022555111234,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-spanish:overall","benchmarkName":"EuroEval Spanish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":28.757499999999997,"unit":"percent","family":"euroeval","index":11.237546046272747,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/spanish","measuredAt":null,"stderr":0.36110723822709506,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":27.74875,"unit":"percent","family":"euroeval","index":9.86890440303511,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.576980285625081,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":26.25571428571428,"unit":"percent","family":"euroeval","index":7.843198476479969,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.38929554378153786,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"olmo37binstructsft","providerId":"allen-institute-for-ai","slug":"allen-institute-for-ai-olmo-3-7b-instruct-sft","name":"Olmo 3 7B Instruct SFT","provider":"Allen Institute for AI","description":"Olmo 3 7B Instruct SFT is a model from Allen Institute for AI. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":8.958172603259875,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-portuguese:overall","benchmarkName":"EuroEval Portuguese","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":28.919999999999998,"unit":"percent","family":"euroeval","index":11.458021156087987,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/portuguese","measuredAt":null,"stderr":0.39057449673525796,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":27.3375,"unit":"percent","family":"euroeval","index":9.310932778964244,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":0.37044947361279923,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-spanish:overall","benchmarkName":"EuroEval Spanish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":27.20125,"unit":"percent","family":"euroeval","index":9.126072879196094,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/spanish","measuredAt":null,"stderr":0.41361402599525077,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-italian:overall","benchmarkName":"EuroEval Italian","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":26.95375,"unit":"percent","family":"euroeval","index":8.790272327323663,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/italian","measuredAt":null,"stderr":0.6794057605731644,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":26.53875,"unit":"percent","family":"euroeval","index":8.227212816103219,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.7471424730263967,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":24.041428571428575,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.41524027959233234,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"olmo3132binstruct","providerId":"allen-institute-for-ai","slug":"allen-institute-for-ai-olmo-3-1-32b-instruct","name":"Olmo 3.1 32B Instruct","provider":"Allen Institute for AI","description":"Olmo 3.1 32B Instruct is a model from Allen Institute for AI. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":27.826602097485925,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-italian:overall","benchmarkName":"EuroEval Italian","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":45.043749999999996,"unit":"percent","family":"euroeval","index":33.33423993690879,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/italian","measuredAt":null,"stderr":0.38932834792755583,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":44.45875,"unit":"percent","family":"euroeval","index":32.54052954157395,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":0.321237840236794,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-spanish:overall","benchmarkName":"EuroEval Spanish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":42.2175,"unit":"percent","family":"euroeval","index":29.499668988506915,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/spanish","measuredAt":null,"stderr":0.41710497779336075,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":39.75125,"unit":"percent","family":"euroeval","index":26.153535206464934,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.4566419549274902,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":39.01,"unit":"percent","family":"euroeval","index":25.147829513230832,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.3923009049186606,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-portuguese:overall","benchmarkName":"EuroEval Portuguese","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":37.9225,"unit":"percent","family":"euroeval","index":23.672342239851957,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/portuguese","measuredAt":null,"stderr":0.2975446395080913,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":{"elo":1311.1511134805364,"eloUpper":1317.2533430207864,"eloLower":1305.0488839402863,"votes":11738,"rank":247,"rankUpper":null,"rankLower":null,"arenaName":"Olmo 3.1 32B Instruct","organization":"allenai","updatedAt":"2026-11-09"},"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"olmo3132binstructdpo","providerId":"allen-institute-for-ai","slug":"allen-institute-for-ai-olmo-3-1-32b-instruct-dpo","name":"Olmo 3.1 32B Instruct DPO","provider":"Allen Institute for AI","description":"Olmo 3.1 32B Instruct DPO is a model from Allen Institute for AI. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":26.927136333541906,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":41.275,"unit":"percent","family":"euroeval","index":28.220913351578552,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.506970166775127,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":40.32142857142857,"unit":"percent","family":"euroeval","index":26.927136333541906,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.3326639537467938,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-portuguese:overall","benchmarkName":"EuroEval Portuguese","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":37.58624999999999,"unit":"percent","family":"euroeval","index":23.216128358772732,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/portuguese","measuredAt":null,"stderr":0.6260678377779839,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"olmo3132binstructsft","providerId":"allen-institute-for-ai","slug":"allen-institute-for-ai-olmo-3-1-32b-instruct-sft","name":"Olmo 3.1 32B Instruct SFT","provider":"Allen Institute for AI","description":"Olmo 3.1 32B Instruct SFT is a model from Allen Institute for AI. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":28.154043977612613,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":41.60875,"unit":"percent","family":"euroeval","index":28.673735307891384,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.5264103081247554,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":41.22571428571428,"unit":"percent","family":"euroeval","index":28.154043977612613,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.501853706628877,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-portuguese:overall","benchmarkName":"EuroEval Portuguese","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":36.66374999999999,"unit":"percent","family":"euroeval","index":21.96450811997548,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/portuguese","measuredAt":null,"stderr":0.6298685378711973,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"olmo3132bthink","providerId":"allen-institute-for-ai","slug":"allen-institute-for-ai-olmo-3-1-32b-think","name":"Olmo 3.1 32B Think","provider":"Allen Institute for AI","description":"Olmo 3.1 32B Think is a model from Allen Institute for AI. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":37.92472554753434,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-italian:overall","benchmarkName":"EuroEval Italian","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":52.27250000000001,"unit":"percent","family":"euroeval","index":43.14199039892033,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/italian","measuredAt":null,"stderr":0.8295169904830161,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":49.32000000000001,"unit":"percent","family":"euroeval","index":39.13612724981584,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":0.9538409196506511,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":48.42714285714286,"unit":"percent","family":"euroeval","index":37.92472554753434,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.5639510761817292,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-spanish:overall","benchmarkName":"EuroEval Spanish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":46.12875,"unit":"percent","family":"euroeval","index":34.80633528552127,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/spanish","measuredAt":null,"stderr":0.6842799591541462,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-portuguese:overall","benchmarkName":"EuroEval Portuguese","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":41.3525,"unit":"percent","family":"euroeval","index":28.326063019336587,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/portuguese","measuredAt":null,"stderr":0.5565827656332883,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":{"elo":1271.0144757783082,"eloUpper":1278.2831890190553,"eloLower":1263.7457625375614,"votes":8244,"rank":286,"rankUpper":null,"rankLower":null,"arenaName":"Olmo 3.1 32B Think","organization":"allenai","updatedAt":"2026-11-09"},"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"decision-index-0-3-onejev-0-8b","providerId":"omnijev","slug":"decision-index-0-3-onejev-0-8b","name":"OneJev 0.8B","provider":"OmniJev","description":"OneJev 0.8B is a model from OmniJev. It has no scored result.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"open-jev-json-canvas-joshuasp","providerId":"joshuasp","slug":"open-jev-json-canvas-joshuasp","name":"Open Jev JSON Canvas","provider":"JoshuaSP","description":"Open Jev JSON Canvas is a standard model from JoshuaSP. It has 2 scored results in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":47.21085681031145,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"reasoning":2},"benchmarks":[{"benchmarkId":"jevBench14","benchmarkName":"jevBench14","category":"reasoning","score":0,"unit":"percent","family":"jevBench14","index":42.94778593843212,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"jevBench15","benchmarkName":"jevBench15","category":"reasoning","score":0,"unit":"percent","family":"jevBench15","index":51.47392768219078,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":15,"benchmarkCount":2,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"open-alternative-jev","providerId":"ikermoel","slug":"open-alternative-jev","name":"open-alternative-jev Qwen3.5-4B","provider":"IkerMoel","description":"open-alternative-jev Qwen3.5-4B is a standard model from IkerMoel. It has 2 scored results in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":72.69118929087193,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"reasoning":2},"benchmarks":[{"benchmarkId":"jevBench15","benchmarkName":"jevBench15","category":"reasoning","score":33.5609546866917,"unit":"percent","family":"jevBench15","index":71.79746655338238,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"jevBench14","benchmarkName":"jevBench14","category":"reasoning","score":33.24214483148707,"unit":"percent","family":"jevBench14","index":73.5849120283615,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":15,"benchmarkCount":2,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"open-jev-zefan-2b","providerId":"zefan-cai-zefan-cai","slug":"open-jev-zefan-2b","name":"Open-Jev 2B","provider":"Zefan Cai (@Zefan_Cai)","description":"Open-Jev 2B is a standard model from Zefan Cai (@Zefan_Cai). It has 2 scored results in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":54.55705826493498,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"reasoning":2},"benchmarks":[{"benchmarkId":"jevBench14","benchmarkName":"jevBench14","category":"reasoning","score":9.980246077108298,"unit":"percent","family":"jevBench14","index":52.14593061359963,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"jevBench15","benchmarkName":"jevBench15","category":"reasoning","score":9.072856494116836,"unit":"percent","family":"jevBench15","index":56.96818591627033,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":15,"benchmarkCount":2,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"open-jev-zefan-9b","providerId":"zefan-cai-zefan-cai","slug":"open-jev-zefan-9b","name":"Open-Jev 9B","provider":"Zefan Cai (@Zefan_Cai)","description":"Open-Jev 9B is a standard model from Zefan Cai (@Zefan_Cai). It has 2 scored results in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":59.744544262166315,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"reasoning":2},"benchmarks":[{"benchmarkId":"jevBench15","benchmarkName":"jevBench15","category":"reasoning","score":24.35608345131106,"unit":"percent","family":"jevBench15","index":66.22326402836435,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"jevBench14","benchmarkName":"jevBench14","category":"reasoning","score":11.195362486014277,"unit":"percent","family":"jevBench14","index":53.265824495968275,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":15,"benchmarkCount":2,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"open-jev-deberta-v3-large","providerId":"kotoba-labs","slug":"open-jev-deberta-v3-large","name":"open-jev-deberta-v3-large","provider":"Kotoba Labs","description":"open-jev-deberta-v3-large is a standard model from Kotoba Labs. It has 2 scored results in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":53.03988360991201,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"reasoning":2},"benchmarks":[{"benchmarkId":"jevBench14","benchmarkName":"jevBench14","category":"reasoning","score":12.649316553397922,"unit":"percent","family":"jevBench14","index":54.60583953763325,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"jevBench15","benchmarkName":"jevBench15","category":"reasoning","score":0,"unit":"percent","family":"jevBench15","index":51.47392768219078,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":15,"benchmarkCount":2,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"opendecision","providerId":"deepan-wadhwa","slug":"opendecision","name":"OpenDecision ModernBERT-large","provider":"Deepan Wadhwa","description":"OpenDecision ModernBERT-large is a standard model from Deepan Wadhwa. It has 2 scored results in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":57.18772492285384,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"reasoning":2},"benchmarks":[{"benchmarkId":"jevBench14","benchmarkName":"jevBench14","category":"reasoning","score":21.641705888024706,"unit":"percent","family":"jevBench14","index":62.89354080634746,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"jevBench15","benchmarkName":"jevBench15","category":"reasoning","score":0.013179888010630534,"unit":"percent","family":"jevBench15","index":51.48190903936022,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":15,"benchmarkCount":2,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"openjev-thinking","providerId":"razorback16","slug":"openjev-thinking","name":"OpenJev (thinking, BF16)","provider":"razorback16","description":"OpenJev (thinking, BF16) is a standard model from razorback16. It has 2 scored results in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":59.47629144653201,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"reasoning":2},"benchmarks":[{"benchmarkId":"jevBench15","benchmarkName":"jevBench15","category":"reasoning","score":17.939903136855065,"unit":"percent","family":"jevBench15","index":62.33781168522439,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"jevBench14","benchmarkName":"jevBench14","category":"reasoning","score":14.829064005607973,"unit":"percent","family":"jevBench14","index":56.61477120783963,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":15,"benchmarkCount":2,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"openjev-razorback16","providerId":"razorback16-codiv","slug":"openjev-razorback16","name":"OpenJev DiffusionGemma 26B-A4B NVFP4","provider":"razorback16 / Codiv","description":"OpenJev DiffusionGemma 26B-A4B NVFP4 is a standard model from razorback16 / Codiv. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":76.9106912847991,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"reasoning":1},"benchmarks":[{"benchmarkId":"jevBench14","benchmarkName":"jevBench14","category":"reasoning","score":36.85070900932601,"unit":"percent","family":"jevBench14","index":76.9106912847991,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":15,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"openjev-verdict","providerId":"hemant-heman10x","slug":"openjev-verdict","name":"openJev Verdict","provider":"Hemant (heman10x)","description":"openJev Verdict is a standard model from Hemant (heman10x). It has 2 scored results in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":55.54915705797153,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"reasoning":2},"benchmarks":[{"benchmarkId":"jevBench14","benchmarkName":"jevBench14","category":"reasoning","score":18.094581304233536,"unit":"percent","family":"jevBench14","index":59.62438643375228,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"jevBench15","benchmarkName":"jevBench15","category":"reasoning","score":0,"unit":"percent","family":"jevBench15","index":51.47392768219078,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":15,"benchmarkCount":2,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"openjev-verdict-1-4","providerId":"hemant-heman10x","slug":"openjev-verdict-1-4","name":"openJev Verdict 1.4","provider":"Hemant (heman10x)","description":"openJev Verdict 1.4 is a standard model from Hemant (heman10x). It has 2 scored results in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":55.966874479668924,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"reasoning":2},"benchmarks":[{"benchmarkId":"jevBench14","benchmarkName":"jevBench14","category":"reasoning","score":19.001051642863935,"unit":"percent","family":"jevBench14","index":60.45982127714707,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"jevBench15","benchmarkName":"jevBench15","category":"reasoning","score":0,"unit":"percent","family":"jevBench15","index":51.47392768219078,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":15,"benchmarkCount":2,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"openjev-sglang","providerId":"ekzhang","slug":"openjev-sglang","name":"openjev-sglang Qwen3.6-35B-A3B","provider":"ekzhang","description":"openjev-sglang Qwen3.6-35B-A3B is a standard model from ekzhang. It has 2 scored results in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":68.74318587990497,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"reasoning":2},"benchmarks":[{"benchmarkId":"jevBench15","benchmarkName":"jevBench15","category":"reasoning","score":29.02726618573374,"unit":"percent","family":"jevBench15","index":69.05199654696355,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"jevBench14","benchmarkName":"jevBench14","category":"reasoning","score":27.653667300053762,"unit":"percent","family":"jevBench14","index":68.43437521284639,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":15,"benchmarkCount":2,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"opensourcejev-qwen35-4b-q4km","providerId":"sabeel111","slug":"opensourcejev-qwen35-4b-q4km","name":"OpenSourceJev Qwen3.5-4B Q4_K_M","provider":"sabeel111","description":"OpenSourceJev Qwen3.5-4B Q4_K_M is a standard model from sabeel111. It has 2 scored results in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":70.05361137627484,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"reasoning":2},"benchmarks":[{"benchmarkId":"jevBench14","benchmarkName":"jevBench14","category":"reasoning","score":40.86697866022174,"unit":"percent","family":"jevBench14","index":80.6122261986684,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"jevBench15","benchmarkName":"jevBench15","category":"reasoning","score":13.245465302463623,"unit":"percent","family":"jevBench15","index":59.4949965538813,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":15,"benchmarkCount":2,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"orcacyber-nano-1-0","providerId":"orcarouter","slug":"orcacyber-nano-1-0","name":"OrcaCyber-Nano-1.0","provider":"OrcaRouter","description":"OrcaCyber-Nano-1.0 is a reasoning model from OrcaRouter in the OrcaCyber Nano family. It has 6 scored results in 4 categories. It has no index.","logoUrl":null,"releaseDate":"2026-10-16","contextWindowTokens":262000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":true,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":63.54313513164589,"coding":null,"reasoning":null,"multimodal":null,"knowledge":58.177104479768154,"multilingual":null,"instruction":35.809728847606635,"math":49.77362353387319},"capabilityCounts":{"math":2,"knowledge":1,"instruction":2,"agentic":1},"benchmarks":[{"benchmarkId":"math500","benchmarkName":"MATH-500 Problem Set","category":"math","score":98.6,"unit":"percent","family":"math-500","index":48.32314300196345,"aboveFit":false,"run":null,"source":"Dan Hendrycks, Collin Burns, Saurav Kadavath, Akul Arora, Steven Basart, Eric Tang, Dawn Song, Jacob Steinhardt","sourceUrl":"https://arxiv.org/abs/2103.03874","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gsm8k","benchmarkName":"Grade School Math 8K","category":"math","score":97.5,"unit":"percent","family":"gsm8k","index":null,"aboveFit":false,"run":null,"source":"Karl Cobbe and colleagues (OpenAI)","sourceUrl":"https://arxiv.org/abs/2110.14168","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aime2025","benchmarkName":"American Invitational Mathematics Examination 2025","category":"math","score":96.23,"unit":"percent","family":"aime2025","index":51.22410406578293,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gpqaDiamond","benchmarkName":"GPQA Diamond","category":"knowledge","score":88.28,"unit":"percent","family":"gpqa-diamond","index":58.177104479768154,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"ifeval","benchmarkName":"Instruction-Following Eval","category":"instruction","score":82.81,"unit":"percent","family":"ifeval","index":29.954845369802477,"aboveFit":false,"run":null,"source":"Jeffrey Zhou, Tianjian Lu, Swaroop Mishra, Siddhartha Brahma, Sujoy Basu, Yi Luan, Denny Zhou, Le Hou","sourceUrl":"https://arxiv.org/abs/2311.07911","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"terminalBench21","benchmarkName":"Terminal-Bench 2.1","category":"agentic","score":72.5,"unit":"percent","family":"terminal-bench-2-1","index":63.54313513164589,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"ifBench","benchmarkName":"Instruction Following Benchmark","category":"instruction","score":56.33,"unit":"percent","family":"ifbench-family","index":41.6646123254108,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":50,"benchmarkCount":6,"categoryCount":4,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"orion-mistral-24b","providerId":"lighton","slug":"orion-mistral-24b","name":"OriOn-Mistral-24B","provider":"LightOn","description":"OriOn-Mistral-24B is a standard model from LightOn in the OriOn family. It has no scored result.","logoUrl":null,"releaseDate":"2026-02-07","contextWindowTokens":344000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"orion-qwen-32b","providerId":"lighton","slug":"orion-qwen-32b","name":"OriOn-Qwen-32B","provider":"LightOn","description":"OriOn-Qwen-32B is a standard model from LightOn in the OriOn family. It has no scored result.","logoUrl":null,"releaseDate":"2026-02-08","contextWindowTokens":262000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"ornith-1-0-35b","providerId":"deepreinforce-ai","slug":"ornith-1-0-35b","name":"Ornith-1.0-35B","provider":"DeepReinforce AI","description":"Ornith-1.0-35B is a reasoning model from DeepReinforce AI in the Ornith 1.0 family. It has 5 scored results in 2 categories. It has no index.","logoUrl":null,"releaseDate":"2026-06-01","contextWindowTokens":256000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":true,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":62.37437823639064,"coding":53.308367651565355,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"coding":3,"agentic":2},"benchmarks":[{"benchmarkId":"sweVerified","benchmarkName":"Software Engineering Benchmark Verified","category":"coding","score":75.6,"unit":"percent","family":"swe-bench-verified","index":56.98728040991875,"aboveFit":false,"run":null,"source":"Carlos E. Jimenez, John Yang, Alexander Wettig, Shunyu Yao, Kexin Pei, Ofir Press, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2310.06770","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"clawEval","benchmarkName":"Claw-Eval","category":"agentic","score":69.8,"unit":"percent","family":"clawEval","index":66.0599208662611,"aboveFit":false,"run":null,"source":"Bowen Ye, Rang Li, Qibin Yang, Yuanxin Liu, Linli Yao, Hanglong Lv, Zhihui Xie, Chenxin An, Lei Li, Lingpeng Kong, Qi Liu, Zhifang Sui, Tong Yang","sourceUrl":"https://arxiv.org/abs/2604.06132","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"terminalBench21","benchmarkName":"Terminal-Bench 2.1","category":"agentic","score":64.2,"unit":"percent","family":"terminal-bench-2-1","index":58.68883560652018,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"swePro","benchmarkName":"SWE-bench Pro","category":"coding","score":50.4,"unit":"percent","family":"swePro","index":51.76052672578786,"aboveFit":false,"run":null,"source":"Xiang Deng, Jeff Da, Edwin Pan, Yannis Yiming He, Charles Ide, Kanak Garg, Niklas Lauffer, Andrew Park, Nitin Pasari, Chetan Rane, Karmini Sampath, Maya Krishnan, Srivatsa Kundurthy, Sean Hendryx, Zifan Wang, Chen Bo Calvin Zhang, Noah Jacobson, Bing Liu, Brad Kenstler","sourceUrl":"https://arxiv.org/abs/2509.16941","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"nl2Repo","benchmarkName":"NL2Repo","category":"coding","score":34.6,"unit":"percent","family":"nl2Repo","index":51.17729581898944,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":40,"benchmarkCount":5,"categoryCount":2,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"ornith-1-0-397b","providerId":"deepreinforce-ai","slug":"ornith-1-0-397b","name":"Ornith-1.0-397B","provider":"DeepReinforce AI","description":"Ornith-1.0-397B is a reasoning model from DeepReinforce AI in the Ornith 1.0 family. It has 5 scored results in 2 categories. It has no index.","logoUrl":null,"releaseDate":"2026-06-01","contextWindowTokens":256000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":true,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":71.54564450029629,"coding":61.52598314552686,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"coding":3,"agentic":2},"benchmarks":[{"benchmarkId":"sweVerified","benchmarkName":"Software Engineering Benchmark Verified","category":"coding","score":82.4,"unit":"percent","family":"swe-bench-verified","index":62.00267193884787,"aboveFit":false,"run":null,"source":"Carlos E. Jimenez, John Yang, Alexander Wettig, Shunyu Yao, Kexin Pei, Ofir Press, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2310.06770","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"terminalBench21","benchmarkName":"Terminal-Bench 2.1","category":"agentic","score":77.5,"unit":"percent","family":"terminal-bench-2-1","index":66.46741195401077,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"clawEval","benchmarkName":"Claw-Eval","category":"agentic","score":77.1,"unit":"percent","family":"clawEval","index":76.62387704658181,"aboveFit":false,"run":null,"source":"Bowen Ye, Rang Li, Qibin Yang, Yuanxin Liu, Linli Yao, Hanglong Lv, Zhihui Xie, Chenxin An, Lei Li, Lingpeng Kong, Qi Liu, Zhifang Sui, Tong Yang","sourceUrl":"https://arxiv.org/abs/2604.06132","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"swePro","benchmarkName":"SWE-bench Pro","category":"coding","score":62.2,"unit":"percent","family":"swePro","index":61.80952450380607,"aboveFit":false,"run":null,"source":"Xiang Deng, Jeff Da, Edwin Pan, Yannis Yiming He, Charles Ide, Kanak Garg, Niklas Lauffer, Andrew Park, Nitin Pasari, Chetan Rane, Karmini Sampath, Maya Krishnan, Srivatsa Kundurthy, Sean Hendryx, Zifan Wang, Chen Bo Calvin Zhang, Noah Jacobson, Bing Liu, Brad Kenstler","sourceUrl":"https://arxiv.org/abs/2509.16941","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"nl2Repo","benchmarkName":"NL2Repo","category":"coding","score":48.2,"unit":"percent","family":"nl2Repo","index":60.76575299392664,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":40,"benchmarkCount":5,"categoryCount":2,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"ornith-1-0-9b","providerId":"deepreinforce-ai","slug":"ornith-1-0-9b","name":"Ornith-1.0-9B","provider":"DeepReinforce AI","description":"Ornith-1.0-9B is a reasoning model from DeepReinforce AI in the Ornith 1.0 family. It has 5 scored results in 2 categories. It has no index.","logoUrl":null,"releaseDate":"2026-06-01","contextWindowTokens":256000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":true,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":51.356311236533,"coding":47.91597411885873,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"coding":3,"agentic":2},"benchmarks":[{"benchmarkId":"sweVerified","benchmarkName":"Software Engineering Benchmark Verified","category":"coding","score":69.4,"unit":"percent","family":"swe-bench-verified","index":52.41442342765987,"aboveFit":false,"run":null,"source":"Carlos E. Jimenez, John Yang, Alexander Wettig, Shunyu Yao, Kexin Pei, Ofir Press, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2310.06770","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"clawEval","benchmarkName":"Claw-Eval","category":"agentic","score":63.1,"unit":"percent","family":"clawEval","index":56.36423505692565,"aboveFit":false,"run":null,"source":"Bowen Ye, Rang Li, Qibin Yang, Yuanxin Liu, Linli Yao, Hanglong Lv, Zhihui Xie, Chenxin An, Lei Li, Lingpeng Kong, Qi Liu, Zhifang Sui, Tong Yang","sourceUrl":"https://arxiv.org/abs/2604.06132","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"terminalBench21","benchmarkName":"Terminal-Bench 2.1","category":"agentic","score":43.1,"unit":"percent","family":"terminal-bench-2-1","index":46.348387416140355,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"swePro","benchmarkName":"SWE-bench Pro","category":"coding","score":42.9,"unit":"percent","family":"swePro","index":45.37345186687799,"aboveFit":false,"run":null,"source":"Xiang Deng, Jeff Da, Edwin Pan, Yannis Yiming He, Charles Ide, Kanak Garg, Niklas Lauffer, Andrew Park, Nitin Pasari, Chetan Rane, Karmini Sampath, Maya Krishnan, Srivatsa Kundurthy, Sean Hendryx, Zifan Wang, Chen Bo Calvin Zhang, Noah Jacobson, Bing Liu, Brad Kenstler","sourceUrl":"https://arxiv.org/abs/2509.16941","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"nl2Repo","benchmarkName":"NL2Repo","category":"coding","score":27.2,"unit":"percent","family":"nl2Repo","index":45.96004706203832,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":40,"benchmarkCount":5,"categoryCount":2,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"ornith-1-5-122b","providerId":"ornith-ai","slug":"ornith-1-5-122b","name":"Ornith-1.5-122B","provider":"Ornith AI","description":"Ornith-1.5-122B is a reasoning model from Ornith AI in the Ornith 1.5 family. It has no scored result.","logoUrl":null,"releaseDate":null,"contextWindowTokens":262144,"modalities":["text"],"openWeights":true,"details":null,"reasoning":true,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"ornith-1-5-35b-a3b","providerId":"ornith-ai","slug":"ornith-1-5-35b-a3b","name":"Ornith-1.5-35B-A3B","provider":"Ornith AI","description":"Ornith-1.5-35B-A3B is a reasoning model from Ornith AI in the Ornith 1.5 family. It has 14 scored results in 3 categories. It has no index.","logoUrl":null,"releaseDate":"2026-08-18","contextWindowTokens":262000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":true,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":53.60358985451774,"coding":59.48200016313962,"reasoning":null,"multimodal":null,"knowledge":54.545920963385285,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"knowledge":2,"coding":3,"agentic":9},"benchmarks":[{"benchmarkId":"gpqa","benchmarkName":"Graduate-Level Google-Proof Q&A","category":"knowledge","score":89.2,"unit":"percent","family":"gpqa-diamond","index":59.02090334069436,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gpqaDiamond","benchmarkName":"GPQA Diamond","category":"knowledge","score":89.2,"unit":"percent","family":"gpqa-diamond","index":59.02090334069436,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"sweVerified","benchmarkName":"Software Engineering Benchmark Verified","category":"coding","score":79,"unit":"percent","family":"swe-bench-verified","index":59.49497617438331,"aboveFit":false,"run":null,"source":"Carlos E. Jimenez, John Yang, Alexander Wettig, Shunyu Yao, Kexin Pei, Ofir Press, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2310.06770","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"clawEval","benchmarkName":"Claw-Eval","category":"agentic","score":72.5,"unit":"percent","family":"clawEval","index":69.96713753569479,"aboveFit":false,"run":null,"source":"Bowen Ye, Rang Li, Qibin Yang, Yuanxin Liu, Linli Yao, Hanglong Lv, Zhihui Xie, Chenxin An, Lei Li, Lingpeng Kong, Qi Liu, Zhifang Sui, Tong Yang","sourceUrl":"https://arxiv.org/abs/2604.06132","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mcpAtlas","benchmarkName":"MCP Atlas","category":"agentic","score":70.2,"unit":"percent","family":"mcpAtlas","index":58.334654155848085,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"terminalBench21","benchmarkName":"Terminal-Bench 2.1","category":"agentic","score":67.8,"unit":"percent","family":"terminal-bench-2-1","index":60.794314918622895,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"wideResearch","benchmarkName":"WideResearch","category":"agentic","score":67.8,"unit":"percent","family":"wideResearch","index":51.046543637231295,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"browseComp","benchmarkName":"BrowseComp","category":"agentic","score":67.6,"unit":"percent","family":"browseComp","index":53.31433293615159,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"swePro","benchmarkName":"SWE-bench Pro","category":"coding","score":59.6,"unit":"percent","family":"swePro","index":59.59533855271731,"aboveFit":false,"run":null,"source":"Xiang Deng, Jeff Da, Edwin Pan, Yannis Yiming He, Charles Ide, Kanak Garg, Niklas Lauffer, Andrew Park, Nitin Pasari, Chetan Rane, Karmini Sampath, Maya Krishnan, Srivatsa Kundurthy, Sean Hendryx, Zifan Wang, Chen Bo Calvin Zhang, Noah Jacobson, Bing Liu, Brad Kenstler","sourceUrl":"https://arxiv.org/abs/2509.16941","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"toolathlonVerified","benchmarkName":"Toolathlon-Verified","category":"agentic","score":48.7,"unit":"percent","family":"toolathlonVerified","index":49.09503565763753,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"nl2Repo","benchmarkName":"NL2Repo","category":"coding","score":46.2,"unit":"percent","family":"nl2Repo","index":59.35568576231823,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hleWithTools","benchmarkName":"Humanity's Last Exam with tools","category":"agentic","score":33.4,"unit":"percent","family":"hleWithTools","index":45.896816321825874,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hle","benchmarkName":"Humanity's Last Exam","category":"knowledge","score":25.6,"unit":"percent","family":"hle","index":50.07093858607622,"aboveFit":false,"run":null,"source":"Center for AI Safety, Scale AI, and thousands of expert contributors","sourceUrl":"https://lastexam.ai/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hleNoTools","benchmarkName":"Humanity's Last Exam without tools","category":"knowledge","score":25.6,"unit":"percent","family":"hle","index":50.07093858607622,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"deepSwe","benchmarkName":"DeepSWE","category":"agentic","score":22,"unit":"percent","family":"deepSwe","index":37.07010648073313,"aboveFit":false,"run":null,"source":"Datacurve AI","sourceUrl":"https://deepswe.datacurve.ai/blog","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"frontierBench","benchmarkName":"Terminal-Bench 3.0","category":"agentic","score":5.1,"unit":"percent","family":"frontierBench","index":56.913367046914495,"aboveFit":false,"run":null,"source":"Ryan Marten, Alex Shaw, Andy Konwinski, Harbor, and the Laude Institute","sourceUrl":"https://www.frontierbench.ai/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":55.00000000000001,"benchmarkCount":14,"categoryCount":3,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"ornith-1-5-397b","providerId":"ornith-ai","slug":"ornith-1-5-397b","name":"Ornith-1.5-397B","provider":"Ornith AI","description":"Ornith-1.5-397B is a reasoning model from Ornith AI in the Ornith 1.5 family. It has 14 scored results in 3 categories. It has no index.","logoUrl":null,"releaseDate":"2026-08-18","contextWindowTokens":262000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":true,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":67.43147919780424,"coding":65.88990184021151,"reasoning":null,"multimodal":null,"knowledge":65.72811285602394,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"knowledge":2,"agentic":9,"coding":3},"benchmarks":[{"benchmarkId":"gpqa","benchmarkName":"Graduate-Level Google-Proof Q&A","category":"knowledge","score":92.8,"unit":"percent","family":"gpqa-diamond","index":62.322724970405574,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gpqaDiamond","benchmarkName":"GPQA Diamond","category":"knowledge","score":92.8,"unit":"percent","family":"gpqa-diamond","index":62.322724970405574,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"browseComp","benchmarkName":"BrowseComp","category":"agentic","score":86.6,"unit":"percent","family":"browseComp","index":68.32470925013872,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"terminalBench21","benchmarkName":"Terminal-Bench 2.1","category":"agentic","score":86.1,"unit":"percent","family":"terminal-bench-2-1","index":71.49716808847838,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"sweVerified","benchmarkName":"Software Engineering Benchmark Verified","category":"coding","score":86,"unit":"percent","family":"swe-bench-verified","index":64.65787921886916,"aboveFit":false,"run":null,"source":"Carlos E. Jimenez, John Yang, Alexander Wettig, Shunyu Yao, Kexin Pei, Ofir Press, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2310.06770","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"clawEval","benchmarkName":"Claw-Eval","category":"agentic","score":81.4,"unit":"percent","family":"clawEval","index":82.84648137197621,"aboveFit":false,"run":null,"source":"Bowen Ye, Rang Li, Qibin Yang, Yuanxin Liu, Linli Yao, Hanglong Lv, Zhihui Xie, Chenxin An, Lei Li, Lingpeng Kong, Qi Liu, Zhifang Sui, Tong Yang","sourceUrl":"https://arxiv.org/abs/2604.06132","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"wideResearch","benchmarkName":"WideResearch","category":"agentic","score":80.8,"unit":"percent","family":"wideResearch","index":63.08654324314704,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mcpAtlas","benchmarkName":"MCP Atlas","category":"agentic","score":80,"unit":"percent","family":"mcpAtlas","index":64.93393099735155,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"toolathlonVerified","benchmarkName":"Toolathlon-Verified","category":"agentic","score":71.2,"unit":"percent","family":"toolathlonVerified","index":66.50348564839575,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"swePro","benchmarkName":"SWE-bench Pro","category":"coding","score":65.1,"unit":"percent","family":"swePro","index":64.27919344925121,"aboveFit":false,"run":null,"source":"Xiang Deng, Jeff Da, Edwin Pan, Yannis Yiming He, Charles Ide, Kanak Garg, Niklas Lauffer, Andrew Park, Nitin Pasari, Chetan Rane, Karmini Sampath, Maya Krishnan, Srivatsa Kundurthy, Sean Hendryx, Zifan Wang, Chen Bo Calvin Zhang, Noah Jacobson, Bing Liu, Brad Kenstler","sourceUrl":"https://arxiv.org/abs/2509.16941","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"nl2Repo","benchmarkName":"NL2Repo","category":"coding","score":59.5,"unit":"percent","family":"nl2Repo","index":68.73263285251416,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hleWithTools","benchmarkName":"Humanity's Last Exam with tools","category":"agentic","score":56.1,"unit":"percent","family":"hleWithTools","index":65.23920963974147,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"deepSwe","benchmarkName":"DeepSWE","category":"agentic","score":56,"unit":"percent","family":"deepSwe","index":61.78715733652761,"aboveFit":false,"run":null,"source":"Datacurve AI","sourceUrl":"https://deepswe.datacurve.ai/blog","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hle","benchmarkName":"Humanity's Last Exam","category":"knowledge","score":44.6,"unit":"percent","family":"hle","index":69.13350074164231,"aboveFit":false,"run":null,"source":"Center for AI Safety, Scale AI, and thousands of expert contributors","sourceUrl":"https://lastexam.ai/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hleNoTools","benchmarkName":"Humanity's Last Exam without tools","category":"knowledge","score":44.6,"unit":"percent","family":"hle","index":69.13350074164231,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"frontierBench","benchmarkName":"Terminal-Bench 3.0","category":"agentic","score":13.5,"unit":"percent","family":"frontierBench","index":62.664627204481334,"aboveFit":false,"run":null,"source":"Ryan Marten, Alex Shaw, Andy Konwinski, Harbor, and the Laude Institute","sourceUrl":"https://www.frontierbench.ai/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":55.00000000000001,"benchmarkCount":14,"categoryCount":3,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"ornith-1-5-9b","providerId":"ornith-ai","slug":"ornith-1-5-9b","name":"Ornith-1.5-9B","provider":"Ornith AI","description":"Ornith-1.5-9B is a reasoning model from Ornith AI in the Ornith 1.5 family. It has 12 scored results in 3 categories. It has no index.","logoUrl":null,"releaseDate":"2026-08-18","contextWindowTokens":262000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":true,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":47.364250889742465,"coding":50.73885738852107,"reasoning":null,"multimodal":null,"knowledge":50.55298880691714,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"knowledge":2,"coding":3,"agentic":7},"benchmarks":[{"benchmarkId":"gpqa","benchmarkName":"Graduate-Level Google-Proof Q&A","category":"knowledge","score":86.4,"unit":"percent","family":"gpqa-diamond","index":56.452819850918964,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gpqaDiamond","benchmarkName":"GPQA Diamond","category":"knowledge","score":86.4,"unit":"percent","family":"gpqa-diamond","index":56.452819850918964,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"sweVerified","benchmarkName":"Software Engineering Benchmark Verified","category":"coding","score":70.6,"unit":"percent","family":"swe-bench-verified","index":53.299492521000296,"aboveFit":false,"run":null,"source":"Carlos E. Jimenez, John Yang, Alexander Wettig, Shunyu Yao, Kexin Pei, Ofir Press, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2310.06770","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"clawEval","benchmarkName":"Claw-Eval","category":"agentic","score":66.5,"unit":"percent","family":"clawEval","index":61.28443382584214,"aboveFit":false,"run":null,"source":"Bowen Ye, Rang Li, Qibin Yang, Yuanxin Liu, Linli Yao, Hanglong Lv, Zhihui Xie, Chenxin An, Lei Li, Lingpeng Kong, Qi Liu, Zhifang Sui, Tong Yang","sourceUrl":"https://arxiv.org/abs/2604.06132","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"wideResearch","benchmarkName":"WideResearch","category":"agentic","score":59.5,"unit":"percent","family":"wideResearch","index":43.35946696576201,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"browseComp","benchmarkName":"BrowseComp","category":"agentic","score":56.4,"unit":"percent","family":"browseComp","index":44.466111108959176,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mcpAtlas","benchmarkName":"MCP Atlas","category":"agentic","score":54.2,"unit":"percent","family":"mcpAtlas","index":47.56032461869957,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"swePro","benchmarkName":"SWE-bench Pro","category":"coding","score":47.5,"unit":"percent","family":"swePro","index":49.29085778034272,"aboveFit":false,"run":null,"source":"Xiang Deng, Jeff Da, Edwin Pan, Yannis Yiming He, Charles Ide, Kanak Garg, Niklas Lauffer, Andrew Park, Nitin Pasari, Chetan Rane, Karmini Sampath, Maya Krishnan, Srivatsa Kundurthy, Sean Hendryx, Zifan Wang, Chen Bo Calvin Zhang, Noah Jacobson, Bing Liu, Brad Kenstler","sourceUrl":"https://arxiv.org/abs/2509.16941","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"terminalBench21","benchmarkName":"Terminal-Bench 2.1","category":"agentic","score":46.2,"unit":"percent","family":"terminal-bench-2-1","index":48.16143904600659,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"toolathlonVerified","benchmarkName":"Toolathlon-Verified","category":"agentic","score":41.2,"unit":"percent","family":"toolathlonVerified","index":43.29221899405145,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"nl2Repo","benchmarkName":"NL2Repo","category":"coding","score":32.4,"unit":"percent","family":"nl2Repo","index":49.62622186422019,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hleWithTools","benchmarkName":"Humanity's Last Exam with tools","category":"agentic","score":30.5,"unit":"percent","family":"hleWithTools","index":43.425761668876305,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hle","benchmarkName":"Humanity's Last Exam","category":"knowledge","score":20.2,"unit":"percent","family":"hle","index":44.65315776291532,"aboveFit":false,"run":null,"source":"Center for AI Safety, Scale AI, and thousands of expert contributors","sourceUrl":"https://lastexam.ai/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"hleNoTools","benchmarkName":"Humanity's Last Exam without tools","category":"knowledge","score":20.2,"unit":"percent","family":"hle","index":44.65315776291532,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":55.00000000000001,"benchmarkCount":12,"categoryCount":3,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"oxalphamax","providerId":"unknown","slug":"unknown-ox-alpha-max","name":"Ox Alpha Max","provider":"Unknown","description":"Ox Alpha Max is a model. It has 6 scored results in 6 categories. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":63.43498785340586,"coding":61.35973790483071,"reasoning":60.30037756738045,"multimodal":null,"knowledge":49.34634324011275,"multilingual":null,"instruction":48.498636889714575,"math":47.83371380532466},"capabilityCounts":{"math":1,"reasoning":1,"coding":1,"knowledge":1,"instruction":1,"agentic":1},"benchmarks":[{"benchmarkId":"livebench:livebench-math:overall","benchmarkName":"LiveBench Mathematics","category":"math","domain":null,"eligible":true,"ranked":true,"score":77.53275,"unit":"percent","family":"livebench:livebench-math:overall","index":47.83371380532466,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Mathematics","measuredAt":"2026-06-25","stderr":null,"costPerTest":0,"latency":null},{"benchmarkId":"livebench:livebench-reasoning:overall","benchmarkName":"LiveBench Reasoning","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":76.5865,"unit":"percent","family":"livebench","index":60.832477081297654,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Reasoning","measuredAt":"2026-10-14","stderr":null,"costPerTest":0,"latency":null},{"benchmarkId":"livebench:livebench-data-analysis:overall","benchmarkName":"LiveBench Data Analysis","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":75.765,"unit":"percent","family":"livebench","index":59.76827805346326,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Data%20Analysis","measuredAt":"2026-10-14","stderr":null,"costPerTest":0,"latency":null},{"benchmarkId":"livebench:livebench-coding:overall","benchmarkName":"LiveBench Coding","category":"coding","domain":null,"eligible":true,"ranked":true,"score":75.75,"unit":"percent","family":"livebench:livebench-coding:overall","index":61.35973790483071,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Coding","measuredAt":"2026-06-25","stderr":null,"costPerTest":0,"latency":null},{"benchmarkId":"livebench:livebench-language:overall","benchmarkName":"LiveBench Language","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":66.13633333333333,"unit":"percent","family":"livebench:livebench-language:overall","index":49.34634324011275,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Language","measuredAt":"2026-06-25","stderr":null,"costPerTest":0,"latency":null},{"benchmarkId":"livebench:livebench-instruction-following:overall","benchmarkName":"LiveBench Instruction Following","category":"instruction","domain":null,"eligible":true,"ranked":true,"score":60.25,"unit":"percent","family":"livebench:livebench-instruction-following:overall","index":48.498636889714575,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=IF","measuredAt":"2026-06-25","stderr":null,"costPerTest":0,"latency":null},{"benchmarkId":"livebench:livebench-agentic-coding:overall","benchmarkName":"LiveBench Agentic Coding","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":52.626,"unit":"percent","family":"livebench:livebench-agentic-coding:overall","index":63.43498785340586,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Agentic%20Coding","measuredAt":"2026-06-25","stderr":null,"costPerTest":0,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":85,"benchmarkCount":6,"categoryCount":6,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:livebench","name":"LiveBench","url":"https://livebench.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"palmyrax5","providerId":"writer","slug":"writer-palmyra-x5","name":"Palmyra X5","provider":"Writer","description":"Palmyra X5 is a model from Writer. It has no scored result.","logoUrl":null,"releaseDate":"2026-01-21T13:57:03.000Z","contextWindowTokens":1040000,"modalities":["text"],"openWeights":null,"details":{"apiId":"writer/palmyra-x5","maxOutputTokens":8192,"knowledgeCutoff":null,"retiresAt":null,"tools":false,"structuredOutput":false,"cacheReadPerMillion":null,"weightsUrl":null},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":0.6,"outputPerMillion":6,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":59.5,"timeToFirstToken":1.444,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[{"host":"amazon-bedrock","name":"Amazon Bedrock","kind":"cloud","url":"https://docs.aws.amazon.com/bedrock/latest/userguide/models-supported.html","inputPerMillion":0.6,"outputPerMillion":6,"cacheReadPerMillion":null,"contextTokens":1040000,"maxOutputTokens":8192,"deprecated":true},{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":0.6,"outputPerMillion":6,"cacheReadPerMillion":null,"contextTokens":1040000,"maxOutputTokens":8192,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"pandagpt-7b","providerId":"pandagpt-authors","slug":"pandagpt-7b","name":"PandaGPT 7B","provider":"PandaGPT authors","description":"PandaGPT 7B is a standard model from PandaGPT authors. It has no scored result.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"parakeet-ctc-1-1b","providerId":"nvidia","slug":"parakeet-ctc-1-1b","name":"Parakeet CTC 1.1B","provider":"NVIDIA","description":"Parakeet CTC 1.1B is a standard model from NVIDIA. It has no scored result.","logoUrl":null,"releaseDate":"2023-12-28","contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"parakeet-redux","providerId":"moondream","slug":"parakeet-redux","name":"Parakeet Redux","provider":"Moondream","description":"Parakeet Redux is a standard model from Moondream. It has no scored result.","logoUrl":null,"releaseDate":"2026-09-22","contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"parakeet-tdt-0-6b-v3","providerId":"nvidia","slug":"parakeet-tdt-0-6b-v3","name":"Parakeet TDT 0.6B v3","provider":"NVIDIA","description":"Parakeet TDT 0.6B v3 is a standard model from NVIDIA. It has no scored result.","logoUrl":null,"releaseDate":"2025-08-14","contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"particle-1","providerId":"particle-ai","slug":"particle-1","name":"Particle-1","provider":"Particle AI","description":"Particle-1 is a reasoning model from Particle AI. It has 4 scored results in 2 categories. It has no index.","logoUrl":null,"releaseDate":"2026-10-28","contextWindowTokens":1048576,"modalities":["text","image"],"openWeights":false,"details":{"apiId":"particle-ai/particle-1","maxOutputTokens":131072,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.003,"weightsUrl":null},"reasoning":true,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":70.86084762899378,"coding":null,"reasoning":null,"multimodal":null,"knowledge":47.216331965434236,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"knowledge":2,"agentic":2},"benchmarks":[{"benchmarkId":"gpqaDiamond","benchmarkName":"GPQA Diamond","category":"knowledge","score":90.4,"unit":"percent","family":"gpqa-diamond","index":60.1215105505981,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmlu","benchmarkName":"Massive Multitask Language Understanding","category":"knowledge","score":86.08,"unit":"percent","family":"mmlu","index":34.31115338027037,"aboveFit":false,"run":null,"source":"Dan Hendrycks, Collin Burns, Steven Basart, Andy Zou, Mantas Mazeika, Dawn Song, Jacob Steinhardt","sourceUrl":"https://arxiv.org/abs/2009.03300","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"terminalBench21","benchmarkName":"Terminal-Bench 2.1","category":"agentic","score":79.93,"unit":"percent","family":"terminal-bench-2-1","index":67.88861048968012,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"deepSwe","benchmarkName":"DeepSWE","category":"agentic","score":72.57,"unit":"percent","family":"deepSwe","index":73.83308476830746,"aboveFit":false,"run":null,"source":"Datacurve AI","sourceUrl":"https://deepswe.datacurve.ai/blog","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":0.049999999999999996,"outputPerMillion":0.5,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":120,"timeToFirstToken":0.76,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":35,"benchmarkCount":4,"categoryCount":2,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"perceptronmk1","providerId":"perceptron","slug":"perceptron-perceptron-mk1","name":"Perceptron Mk1","provider":"Perceptron","description":"Perceptron Mk1 is a model from Perceptron. It has no scored result.","logoUrl":null,"releaseDate":"2026-05-12T14:43:49.000Z","contextWindowTokens":32768,"modalities":["text","image","video"],"openWeights":null,"details":{"apiId":"perceptron/perceptron-mk1","maxOutputTokens":8192,"knowledgeCutoff":null,"retiresAt":null,"tools":false,"structuredOutput":true,"cacheReadPerMillion":null,"weightsUrl":null},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":0.15,"outputPerMillion":1.5,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":31,"timeToFirstToken":0.644,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":0.15,"outputPerMillion":1.5,"cacheReadPerMillion":null,"contextTokens":32768,"maxOutputTokens":8192,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"perceptronmk15","providerId":"perceptron","slug":"perceptron-perceptron-mk1-5","name":"Perceptron Mk1.5","provider":"Perceptron","description":"Perceptron Mk1.5 is a model from Perceptron. It has no scored result.","logoUrl":null,"releaseDate":"2026-09-25T16:11:01.000Z","contextWindowTokens":36864,"modalities":["text","image","video","audio"],"openWeights":null,"details":{"apiId":"perceptron/perceptron-mk1.5","maxOutputTokens":8192,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":null,"weightsUrl":null},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":0.15,"outputPerMillion":1.5,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":106,"timeToFirstToken":0.934,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"pplx-decider-flash-v1-2","providerId":"perplexity","slug":"pplx-decider-flash-v1-2","name":"Perplexity Decider Flash v1.2 4B","provider":"Perplexity","description":"Perplexity Decider Flash v1.2 4B is a model from Perplexity in the Perplexity Decider Flash v1.2 family. It has no scored result.","logoUrl":null,"releaseDate":"2026-10-16","contextWindowTokens":262000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":0.01,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"pplx-decider-v1-27b","providerId":"perplexity","slug":"pplx-decider-v1-27b","name":"Perplexity Decider v1 27B","provider":"Perplexity","description":"Perplexity Decider v1 27B is a standard model from Perplexity in the Perplexity Decider family. It has no scored result.","logoUrl":null,"releaseDate":"2026-10-01","contextWindowTokens":262000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":0.02,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"decision-index-0-3-perplexity-decider-v1-1-27b","providerId":"perplexity","slug":"decision-index-0-3-perplexity-decider-v1-1-27b","name":"Perplexity Decider v1.1 (27B)","provider":"Perplexity","description":"Perplexity Decider v1.1 (27B) is a model from Perplexity. It has no scored result.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":0.02,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"pplx-decider-v1-2","providerId":"perplexity","slug":"pplx-decider-v1-2","name":"Perplexity Decider v1.2 27B","provider":"Perplexity","description":"Perplexity Decider v1.2 27B is a model from Perplexity in the Perplexity Decider v1.2 family. It has no scored result.","logoUrl":null,"releaseDate":"2026-10-16","contextWindowTokens":262000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":0.02,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"pharia-1-llm-7b-control","providerId":"aleph-alpha","slug":"pharia-1-llm-7b-control","name":"Pharia-1-LLM-7B-control","provider":"Aleph Alpha","description":"Pharia-1-LLM-7B-control is a standard model from Aleph Alpha in the Pharia-1-LLM-7B family. It has no scored result.","logoUrl":null,"releaseDate":"2024-08-26","contextWindowTokens":8000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"pharia-1-llm-7b-control-aligned","providerId":"aleph-alpha","slug":"pharia-1-llm-7b-control-aligned","name":"Pharia-1-LLM-7B-control-aligned","provider":"Aleph Alpha","description":"Pharia-1-LLM-7B-control-aligned is a standard model from Aleph Alpha in the Pharia-1-LLM-7B family. It has no scored result.","logoUrl":null,"releaseDate":"2024-08-26","contextWindowTokens":8000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"phi3medium14b","providerId":"microsoft","slug":"microsoft-phi-3-medium-14b","name":"phi-3-medium 14B","provider":"Microsoft","description":"phi-3-medium 14B is a model from Microsoft. It has 2 scored results in 2 categories. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":5,"multilingual":null,"instruction":null,"math":7.52798098569383},"capabilityCounts":{"knowledge":1,"math":1},"benchmarks":[{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":27.58838383838384,"unit":"percent","family":"gpqa-diamond","index":5,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.6117685500292727,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-math-level-5:overall","benchmarkName":"MATH level 5","category":"math","domain":null,"eligible":true,"ranked":true,"score":17.560422960725074,"unit":"percent","family":"epoch-ai:epoch-math-level-5:overall","index":7.52798098569383,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0.7295657661098519,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":25,"benchmarkCount":2,"categoryCount":2,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"phi-3-5-mini","providerId":"microsoft","slug":"phi-3-5-mini","name":"Phi-3.5-mini","provider":"Microsoft","description":"Phi-3.5-mini is a standard model from Microsoft. It has 6 scored results in 3 categories. It has no index.","logoUrl":null,"releaseDate":"2024-08-01","contextWindowTokens":128000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":5,"reasoning":null,"multimodal":null,"knowledge":7.252245307043367,"multilingual":10.273098771432402,"instruction":null,"math":null},"capabilityCounts":{"knowledge":4,"coding":1,"multilingual":1},"benchmarks":[{"benchmarkId":"gsm8k","benchmarkName":"Grade School Math 8K","category":"math","score":86.2,"unit":"percent","family":"gsm8k","index":null,"aboveFit":false,"run":null,"source":"Karl Cobbe and colleagues (OpenAI)","sourceUrl":"https://arxiv.org/abs/2110.14168","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"bbh","benchmarkName":"BIG-Bench Hard","category":"reasoning","score":69,"unit":"percent","family":"bbh","index":null,"aboveFit":false,"run":null,"source":"Mirac Suzgun, Nathan Scales, Nathanael Schärli, Sebastian Gehrmann, Yi Tay, Hyung Won Chung, Aakanksha Chowdhery, Quoc V. Le, Ed H. Chi, Denny Zhou, Jason Wei","sourceUrl":"https://arxiv.org/abs/2210.09261","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmlu","benchmarkName":"Massive Multitask Language Understanding","category":"knowledge","score":69,"unit":"percent","family":"mmlu","index":13.917831172762497,"aboveFit":false,"run":null,"source":"Dan Hendrycks, Collin Burns, Steven Basart, Andy Zou, Mantas Mazeika, Dawn Song, Jacob Steinhardt","sourceUrl":"https://arxiv.org/abs/2009.03300","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"humaneval","benchmarkName":"Evaluating Large Language Models Trained on Code","category":"coding","score":62.8,"unit":"percent","family":"humaneval","index":5,"aboveFit":false,"run":null,"source":"Mark Chen, Jerry Tworek, Heewoo Jun, Qiming Yuan, Henrique Ponde de Oliveira Pinto, Jared Kaplan, Harri Edwards, Yuri Burda, Nicholas Joseph, Greg Brockman, Alex Ray, Raul Puri, Gretchen Krueger, Michael Petrov, Heidy Khlaaf, Girish Sastry, Pamela Mishkin, Brooke Chan, Scott Gray, Nick Ryder, Mikhail Pavlov, Alethea Power, Lukasz Kaiser, Mohammad Bavarian, Clemens Winter, Philippe Tillet, Felipe Petroski Such, Dave Cummings, Matthias Plappert, Fotios Chantzis, Elizabeth Barnes, Ariel Herbert-Voss, William Hebgen Guss, Alex Nichol, Alex Paino, Nikolas Tezak, Jie Tang, Igor Babuschkin, Suchir Balaji, Shantanu Jain, William Saunders, Christopher Hesse, Andrew N. Carr, Jan Leike, Josh Achiam, Vedant Misra, Evan Morikawa, Alec Radford, Matthew Knight, Miles Brundage, Mira Murati, Katie Mayer, Peter Welinder, Bob McGrew, Dario Amodei, Sam McCandlish, Ilya Sutskever, Wojciech Zaremba","sourceUrl":"https://arxiv.org/abs/2107.03374","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmmlu","benchmarkName":"MMMLU","category":"knowledge","score":55.4,"unit":"percent","family":"mmmlu","index":5,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mgsm","benchmarkName":"Multilingual Grade School Math","category":"multilingual","score":47.9,"unit":"percent","family":"mgsm","index":10.273098771432402,"aboveFit":false,"run":null,"source":"Freda Shi, Mirac Suzgun, Markus Freitag, Xuezhi Wang, Suraj Srivats, Soroush Vosoughi, Hyung Won Chung, Yi Tay, Sebastian Ruder, Denny Zhou, Dipanjan Das, Jason Wei","sourceUrl":"https://arxiv.org/abs/2210.03057","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmluPro","benchmarkName":"Massive Multitask Language Understanding Professional","category":"knowledge","score":47.4,"unit":"percent","family":"mmlu-pro","index":5,"aboveFit":false,"run":null,"source":"Yubo Wang, Xueguang Ma, Ge Zhang, Yuansheng Ni, Abhranil Chandra, Shiguang Guo, Weiming Ren, Aaran Arulraj, Xuan He, Ziyan Jiang, Tianle Li, Max Ku, Kai Wang, Alex Zhuang, Rongqi Fan, Xiang Yue, Wenhu Chen","sourceUrl":"https://arxiv.org/abs/2406.01574","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gpqa","benchmarkName":"Graduate-Level Google-Proof Q&A","category":"knowledge","score":30.4,"unit":"percent","family":"gpqa-diamond","index":5.091150055410992,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":40,"benchmarkCount":6,"categoryCount":3,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"phi-4-multimodal-instruct","providerId":"microsoft","slug":"phi-4-multimodal-instruct","name":"Phi-4 Multimodal Instruct","provider":"Microsoft","description":"Phi-4 Multimodal Instruct is a standard model from Microsoft. It has 3 scored results in 2 categories. It has no index.","logoUrl":null,"releaseDate":"2025-02-26","contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":5,"knowledge":17.751574018142605,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"knowledge":2,"multimodal":1},"benchmarks":[{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":31.5,"unit":"percent","family":"gpqa-diamond","index":6.100039997822762,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaMmmuPro","benchmarkName":"Artificial Analysis MMMU-Pro","category":"multimodal","score":14.5,"unit":"percent","family":"mmmu-pro","index":5,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmmu-pro","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":5,"unit":"percent","family":"hle","index":29.403108038462445,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":17,"timeToFirstToken":0.79,"source":"BenchLM","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":25,"benchmarkCount":3,"categoryCount":2,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"phi-4-reasoning-plus","providerId":"microsoft","slug":"phi-4-reasoning-plus","name":"Phi-4-reasoning-plus","provider":"Microsoft","description":"Phi-4-reasoning-plus is a reasoning model from Microsoft in the Phi-4-reasoning family. It has 6 scored results in 4 categories. It has no index.","logoUrl":null,"releaseDate":"2025-04-30","contextWindowTokens":32000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":true,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":33.40302576354422,"reasoning":null,"multimodal":null,"knowledge":40.31696475030644,"multilingual":null,"instruction":34.16389403328399,"math":40.98761633983534},"capabilityCounts":{"instruction":1,"math":2,"knowledge":2,"coding":1},"benchmarks":[{"benchmarkId":"ifeval","benchmarkName":"Instruction-Following Eval","category":"instruction","score":84.9,"unit":"percent","family":"ifeval","index":34.16389403328399,"aboveFit":false,"run":null,"source":"Jeffrey Zhou, Tianjian Lu, Swaroop Mishra, Siddhartha Brahma, Sujoy Basu, Yi Luan, Denny Zhou, Le Hou","sourceUrl":"https://arxiv.org/abs/2311.07911","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aime2024","benchmarkName":"American Invitational Mathematics Examination 2024","category":"math","score":81.3,"unit":"percent","family":"aime2024","index":40.643197245755864,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aime2025","benchmarkName":"American Invitational Mathematics Examination 2025","category":"math","score":78,"unit":"percent","family":"aime2025","index":41.332035433914825,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmluPro","benchmarkName":"Massive Multitask Language Understanding Professional","category":"knowledge","score":76,"unit":"percent","family":"mmlu-pro","index":40.23163146079016,"aboveFit":false,"run":null,"source":"Yubo Wang, Xueguang Ma, Ge Zhang, Yuansheng Ni, Abhranil Chandra, Shiguang Guo, Weiming Ren, Aaran Arulraj, Xuan He, Ziyan Jiang, Tianle Li, Max Ku, Kai Wang, Alex Zhuang, Rongqi Fan, Xiang Yue, Wenhu Chen","sourceUrl":"https://arxiv.org/abs/2406.01574","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gpqa","benchmarkName":"Graduate-Level Google-Proof Q&A","category":"knowledge","score":68.9,"unit":"percent","family":"gpqa-diamond","index":40.402298039822725,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"liveCodeBench","benchmarkName":"LiveCodeBench: Holistic and Contamination Free Evaluation of Large Language Models for Code","category":"coding","score":53.1,"unit":"percent","family":"livecodebench","index":33.40302576354422,"aboveFit":false,"run":null,"source":"Naman Jain, King Han, Alex Gu, Wen-Ding Li, Fanjia Yan, Tianjun Zhang, Sida Wang, Armando Solar-Lezama, Koushik Sen, Ion Stoica","sourceUrl":"https://arxiv.org/abs/2403.07974","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":50,"benchmarkCount":6,"categoryCount":4,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"phonic-1","providerId":"phonic","slug":"phonic-1","name":"Phonic-1","provider":"Phonic","description":"Phonic-1 is a reasoning model from Phonic. It has no scored result.","logoUrl":null,"releaseDate":"2026-10-29","contextWindowTokens":null,"modalities":["text"],"openWeights":false,"details":null,"reasoning":true,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"phonon-1","providerId":"fermion-research","slug":"phonon-1","name":"Phonon-1","provider":"Fermion Research","description":"Phonon-1 is a standard model from Fermion Research. It has no scored result.","logoUrl":null,"releaseDate":"2026-08-28","contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"phonon-2","providerId":"fermion-research","slug":"phonon-2","name":"Phonon-2","provider":"Fermion Research","description":"Phonon-2 is a standard model from Fermion Research. It has no scored result.","logoUrl":null,"releaseDate":"2026-09-01","contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"pleias12bpreview","providerId":"pleias","slug":"pleias-pleias-1-2b-preview","name":"Pleias 1.2B Preview","provider":"PleIAs","description":"Pleias 1.2B Preview is a model from PleIAs. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":5,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-french:overall","benchmarkName":"EuroEval French","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":21.59875,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/french","measuredAt":null,"stderr":0.692328498618972,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-dutch:overall","benchmarkName":"EuroEval Dutch","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":20.837777777777774,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/dutch","measuredAt":null,"stderr":0.5844011390800198,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-portuguese:overall","benchmarkName":"EuroEval Portuguese","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":17.705,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/portuguese","measuredAt":null,"stderr":0.740391577140097,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-italian:overall","benchmarkName":"EuroEval Italian","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":16.3625,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/italian","measuredAt":null,"stderr":0.5842944463196617,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":15.783749999999996,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.5977954290557933,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":15.153749999999999,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":0.577966532335567,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-spanish:overall","benchmarkName":"EuroEval Spanish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":14.556249999999999,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/spanish","measuredAt":null,"stderr":0.5986012341283636,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":12.072857142857142,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.5680219886517042,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"pleias350mpreview","providerId":"pleias","slug":"pleias-pleias-350m-preview","name":"Pleias 350M Preview","provider":"PleIAs","description":"Pleias 350M Preview is a model from PleIAs. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":5,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-dutch:overall","benchmarkName":"EuroEval Dutch","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":13.96111111111111,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/dutch","measuredAt":null,"stderr":0.5366609155431434,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-french:overall","benchmarkName":"EuroEval French","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":13.685,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/french","measuredAt":null,"stderr":0.4381441686248945,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-italian:overall","benchmarkName":"EuroEval Italian","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":12.44,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/italian","measuredAt":null,"stderr":0.5261534947142327,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":11.8375,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.466172915879934,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-spanish:overall","benchmarkName":"EuroEval Spanish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":10.3475,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/spanish","measuredAt":null,"stderr":0.3592331241686935,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":9.65375,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":0.689318050684878,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-portuguese:overall","benchmarkName":"EuroEval Portuguese","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":7.982500000000001,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/portuguese","measuredAt":null,"stderr":0.629482674503437,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":7.732857142857142,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.45213100864637884,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"pleias3bpreview","providerId":"pleias","slug":"pleias-pleias-3b-preview","name":"Pleias 3B Preview","provider":"PleIAs","description":"Pleias 3B Preview is a model from PleIAs. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":5,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-french:overall","benchmarkName":"EuroEval French","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":22.938750000000002,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/french","measuredAt":null,"stderr":0.4984084043232016,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-portuguese:overall","benchmarkName":"EuroEval Portuguese","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":15.36375,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/portuguese","measuredAt":null,"stderr":0.8630270129028407,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":15.05875,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":0.806037762763011,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-italian:overall","benchmarkName":"EuroEval Italian","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":14.993750000000002,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/italian","measuredAt":null,"stderr":0.6994551450950947,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-spanish:overall","benchmarkName":"EuroEval Spanish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":14.88875,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/spanish","measuredAt":null,"stderr":0.4766681235408972,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":10.401428571428571,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.5854197865895995,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"pllum12bchat2412","providerId":"unknown","slug":"unknown-pllum-12b-chat-2412","name":"PLLuM 12B Chat 2412","provider":"Unknown","description":"PLLuM 12B Chat 2412 is a model. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":32.39491905695971,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":44.35142857142857,"unit":"percent","family":"euroeval","index":32.39491905695971,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.5513489949983715,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"pllum12binstruct2412","providerId":"unknown","slug":"unknown-pllum-12b-instruct-2412","name":"PLLuM 12B Instruct 2412","provider":"Unknown","description":"PLLuM 12B Instruct 2412 is a model. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":33.64508561371423,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":45.27285714285715,"unit":"percent","family":"euroeval","index":33.64508561371423,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.5334447162603848,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"pllum12bncchat2412","providerId":"unknown","slug":"unknown-pllum-12b-nc-chat-2412","name":"PLLuM 12B Nc Chat 2412","provider":"Unknown","description":"PLLuM 12B Nc Chat 2412 is a model. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":31.21840572370391,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":43.48428571428571,"unit":"percent","family":"euroeval","index":31.21840572370391,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.6871473121328704,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"pllum12bncchat2507","providerId":"unknown","slug":"unknown-pllum-12b-nc-chat-2507","name":"PLLuM 12B Nc Chat 2507","provider":"Unknown","description":"PLLuM 12B Nc Chat 2507 is a model. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":33.10237765109211,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":44.87285714285714,"unit":"percent","family":"euroeval","index":33.10237765109211,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.6130585749555507,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"pllum12bncinstruct2412","providerId":"unknown","slug":"unknown-pllum-12b-nc-instruct-2412","name":"PLLuM 12B Nc Instruct 2412","provider":"Unknown","description":"PLLuM 12B Nc Instruct 2412 is a model. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":31.309503131715488,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":43.55142857142857,"unit":"percent","family":"euroeval","index":31.309503131715488,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.6591382747998499,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"pllum12bncinstruct2507","providerId":"unknown","slug":"unknown-pllum-12b-nc-instruct-2507","name":"PLLuM 12B Nc Instruct 2507","provider":"Unknown","description":"PLLuM 12B Nc Instruct 2507 is a model. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":34.50954186846232,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":45.91000000000001,"unit":"percent","family":"euroeval","index":34.50954186846232,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.6287921690326103,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"pllum8x7binstruct2412","providerId":"unknown","slug":"unknown-pllum-8x7b-instruct-2412","name":"PLLuM 8x7B Instruct 2412","provider":"Unknown","description":"PLLuM 8x7B Instruct 2412 is a model. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":32.745740989940444,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":44.61000000000001,"unit":"percent","family":"euroeval","index":32.745740989940444,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.642860317452479,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"plumb-4b","providerId":"crh225","slug":"plumb-4b","name":"Plumb-4B","provider":"crh225","description":"Plumb-4B is a standard model from crh225. It has 2 scored results in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":94.90450171885607,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"reasoning":2},"benchmarks":[{"benchmarkId":"jevBench15","benchmarkName":"jevBench15","category":"reasoning","score":71.56069240662383,"unit":"percent","family":"jevBench15","index":94.80900343771214,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"jevBench14","benchmarkName":"jevBench14","category":"reasoning","score":65.84344821613185,"unit":"percent","family":"jevBench14","index":95,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":15,"benchmarkCount":2,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"pokee-isaac-28b","providerId":"pokee-ai","slug":"pokee-isaac-28b","name":"Pokee-Isaac 28B","provider":"Pokee AI","description":"Pokee-Isaac 28B is a reasoning model from Pokee AI in the Pokee-Isaac family. It has 5 scored results in 2 categories. It has no index.","logoUrl":null,"releaseDate":"2026-08-03","contextWindowTokens":10000000,"modalities":["text"],"openWeights":false,"details":null,"reasoning":true,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":55.40290304336545,"coding":null,"reasoning":47.66844563946254,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"agentic":4,"reasoning":1},"benchmarks":[{"benchmarkId":"pinchBench","benchmarkName":"PinchBench","category":"agentic","score":95.67,"unit":"percent","family":"pinchBench","index":55.74313640194039,"aboveFit":false,"run":null,"source":"Kilo Code","sourceUrl":"https://pinchbench.com/about","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mcpAtlasClaimCoverage","benchmarkName":"MCP-Atlas mean claim coverage","category":"agentic","score":74.59,"unit":"percent","family":"mcpAtlasClaimCoverage","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"bfclV4","benchmarkName":"Berkeley Function Calling Leaderboard v4","category":"agentic","score":70.94,"unit":"percent","family":"bfclV4","index":51.0307207912175,"aboveFit":false,"run":null,"source":"Shishir G. Patil et al.","sourceUrl":"https://gorilla.cs.berkeley.edu/leaderboard.html","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"tau3Bench","benchmarkName":"τ³-Bench Tool-Agent-User Evaluation","category":"agentic","score":66.2,"unit":"percent","family":"tau3Bench","index":55.62254954575806,"aboveFit":false,"run":null,"source":"Sierra Research","sourceUrl":"https://github.com/sierra-research/tau2-bench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"terminalBench21","benchmarkName":"Terminal-Bench 2.1","category":"agentic","score":65.1,"unit":"percent","family":"terminal-bench-2-1","index":59.21520543454586,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mrcrv2","benchmarkName":"MRCRv2","category":"reasoning","score":60.7,"unit":"percent","family":"mrcrv2","index":47.66844563946254,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":0.15,"outputPerMillion":1,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":35,"benchmarkCount":5,"categoryCount":2,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"principledmachinescortex01","providerId":"unknown","slug":"unknown-principledmachines-cortex-0-1","name":"PrincipledMachines Cortex 0.1","provider":"Unknown","description":"PrincipledMachines Cortex 0.1 is a model. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":51.360255554033735,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"agentic":1},"benchmarks":[{"benchmarkId":"tau2-bench:banking-knowledge:overall","benchmarkName":"τ²-bench Banking","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":73.19587628865979,"unit":"percent","family":"tau2-bench","index":51.360255554033735,"aboveFit":false,"run":"medium effort · Principled Machines","source":"Sierra Research","sourceUrl":"https://github.com/sierra-research/tau2-bench","measuredAt":"2026-10-20","stderr":null,"costPerTest":0.5955274784149485,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":20,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:sierra-research","name":"Sierra Research","url":"https://github.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"quasar-438b","providerId":"multiverse-computing","slug":"quasar-438b","name":"Quasar 438B","provider":"Multiverse Computing","description":"Quasar 438B is a reasoning model from Multiverse Computing in the Quasar family. It has 7 scored results in 4 categories. It has no index.","logoUrl":null,"releaseDate":"2026-09-02","contextWindowTokens":1000000,"modalities":["text"],"openWeights":false,"details":null,"reasoning":true,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":60.04304022810831,"coding":58.54734258505993,"reasoning":57.346206375549514,"multimodal":null,"knowledge":43.74717959367421,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"reasoning":2,"knowledge":2,"agentic":2,"coding":1},"benchmarks":[{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":76.3,"unit":"percent","family":"lcr","index":59.645993561356576,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":73.2,"unit":"percent","family":"gpqa-diamond","index":44.346140541977796,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"terminalBench21","benchmarkName":"Terminal-Bench 2.1","category":"agentic","score":69.3,"unit":"percent","family":"terminal-bench-2-1","index":61.67159796533236,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaSciCode","benchmarkName":"Artificial Analysis SciCode","category":"coding","score":48.1,"unit":"percent","family":"scicode-family","index":58.54734258505993,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/scicode","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gdpvalAaNormalized","benchmarkName":"GDPval-AA normalized","category":"agentic","score":34.1,"unit":"percent","family":"gdpvalAaNormalized","index":58.41448249088426,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":18.7,"unit":"percent","family":"hle","index":43.14821864537063,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":9.4,"unit":"percent","family":"critpt","index":55.04641918974245,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":0.6,"outputPerMillion":1.8,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":180,"timeToFirstToken":12.39,"source":"BenchLM","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":70,"benchmarkCount":7,"categoryCount":4,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"decision-index-0-3-quyet-1-0-large","providerId":"chinhnc","slug":"decision-index-0-3-quyet-1-0-large","name":"Quyet-1.0-Large","provider":"chinhnc","description":"Quyet-1.0-Large is a model from chinhnc. It has no scored result.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"qwen25","providerId":"alibaba","slug":"alibaba-qwen-2-5","name":"Qwen 2.5","provider":"Alibaba","description":"Qwen 2.5 is a model from Alibaba. It has 2 scored results in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":24.63612778099832,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"coding":2},"benchmarks":[{"benchmarkId":"swebench-experiments:swe-bench-verified:overall","benchmarkName":"SWE-bench Verified","category":"coding","domain":null,"eligible":true,"ranked":true,"score":32.800000000000004,"unit":"percent","family":"swe-bench-verified","index":25.419816080776748,"aboveFit":false,"run":"SWE-Fixer","source":"SWE-bench team","sourceUrl":"https://github.com/SWE-bench/experiments/tree/main/evaluation/verified","measuredAt":"2026-09-01","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"swebench-experiments:swe-bench-lite:overall","benchmarkName":"SWE-bench Lite","category":"coding","domain":null,"eligible":true,"ranked":true,"score":24,"unit":"percent","family":"swe-bench-lite","index":23.852439481219893,"aboveFit":false,"run":"SWE-Fixer","source":"SWE-bench team","sourceUrl":"https://github.com/SWE-bench/experiments/tree/main/evaluation/lite","measuredAt":"2025-09-11","stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":20,"benchmarkCount":2,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:swe-bench-team","name":"SWE-bench team","url":"https://github.com","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"qwen35flashhosted35ba3b","providerId":"alibaba","slug":"alibaba-qwen-3-5-flash-hosted-35b-a3b","name":"Qwen 3.5 Flash (hosted 35B-A3B)","provider":"Alibaba","description":"Qwen 3.5 Flash (hosted 35B-A3B) is a model from Alibaba. It has 8 scored results in 3 categories. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":47.84126250329661,"multimodal":null,"knowledge":46.78069645677262,"multilingual":null,"instruction":null,"math":44.28562820380012},"capabilityCounts":{"math":4,"knowledge":2,"reasoning":2},"benchmarks":[{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":84.44444444444444,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":58.04836584958089,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":5.463890236888291,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":82.32323232323232,"unit":"percent","family":"gpqa-diamond","index":52.713719935791424,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.717875263904491,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-simpleqa-verified:overall","benchmarkName":"SimpleQA Verified","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":20.32032032032032,"unit":"percent","family":"epoch-ai:epoch-simpleqa-verified:overall","index":40.8476729777538,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.2737194890497592,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tiers-1-3-v2-private:overall","benchmarkName":"FrontierMath-Tiers-1-3-v2-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":18.24561403508772,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tiers-1-3-v2-private:overall","index":40.4983064763871,"aboveFit":false,"run":"none effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.2917941277334046,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-mystery-game-puzzles:overall","benchmarkName":"Mystery Game Puzzles","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":16,"unit":"percent","family":"epoch-ai:epoch-mystery-game-puzzles:overall","index":51.071636549352945,"aboveFit":false,"run":"none effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":3.6845294917747067,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-chess-puzzles:overall","benchmarkName":"Chess Puzzles","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":14.000000000000002,"unit":"percent","family":"epoch-ai:epoch-chess-puzzles:overall","index":44.61088845724028,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":3.4873508801977704,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-2025-02-28-public:overall","benchmarkName":"FrontierMath-2025-02-28-Public","category":"math","domain":null,"eligible":false,"ranked":true,"score":10,"unit":"percent","family":"epoch-ai:epoch-frontiermath-2025-02-28-public:overall","index":null,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":10,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-2025-02-28-private:overall","benchmarkName":"FrontierMath-2025-02-28-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":6.206896551724138,"unit":"percent","family":"epoch-ai:epoch-frontiermath-2025-02-28-private:overall","index":35.393482762066874,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.4192986356621422,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-private:overall","benchmarkName":"FrontierMath-Tier-4-2025-07-01-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":0,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-private:overall","index":43.20235772716562,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":40,"benchmarkCount":8,"categoryCount":3,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"qwen35plushosted397ba17b","providerId":"alibaba","slug":"alibaba-qwen-3-5-plus-hosted-397b-a17b","name":"Qwen 3.5 Plus (hosted 397B-A17B)","provider":"Alibaba","description":"Qwen 3.5 Plus (hosted 397B-A17B) is a model from Alibaba. It has 7 scored results in 3 categories. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":46.330931963773956,"multimodal":null,"knowledge":50.0341185211484,"multilingual":null,"instruction":null,"math":51.04751631213776},"capabilityCounts":{"math":3,"knowledge":2,"reasoning":2},"benchmarks":[{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":86.66666666666667,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":59.341241506765535,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":5.124707431905383,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":84.84848484848484,"unit":"percent","family":"gpqa-diamond","index":55.02981254922523,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.554565042660364,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-2025-02-28-public:overall","benchmarkName":"FrontierMath-2025-02-28-Public","category":"math","domain":null,"eligible":false,"ranked":true,"score":50,"unit":"percent","family":"epoch-ai:epoch-frontiermath-2025-02-28-public:overall","index":null,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":16.666666666666664,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-simpleqa-verified:overall","benchmarkName":"SimpleQA Verified","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":25.35070140280561,"unit":"percent","family":"epoch-ai:epoch-simpleqa-verified:overall","index":45.038424493071574,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.3777178222916078,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-2025-02-28-private:overall","benchmarkName":"FrontierMath-2025-02-28-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":21.03448275862069,"unit":"percent","family":"epoch-ai:epoch-frontiermath-2025-02-28-private:overall","index":48.482944118708296,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.397373715108675,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-mystery-game-puzzles:overall","benchmarkName":"Mystery Game Puzzles","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":17,"unit":"percent","family":"epoch-ai:epoch-mystery-game-puzzles:overall","index":51.84605144605811,"aboveFit":false,"run":"none effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":3.7752516806863685,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-chess-puzzles:overall","benchmarkName":"Chess Puzzles","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":9,"unit":"percent","family":"epoch-ai:epoch-chess-puzzles:overall","index":40.81581248148979,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.8762349126466145,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-private:overall","benchmarkName":"FrontierMath-Tier-4-2025-07-01-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":2.083333333333333,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-private:overall","index":45.31836331093945,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.083333333333333,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":40,"benchmarkCount":7,"categoryCount":3,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"qwenplus","providerId":"alibaba","slug":"alibaba-qwen-plus","name":"Qwen Plus","provider":"Alibaba","description":"Qwen Plus is a model from Alibaba. It has 4 scored results in 2 categories. It has no index.","logoUrl":null,"releaseDate":"2025-02-01T11:37:20.000Z","contextWindowTokens":1000000,"modalities":["text"],"openWeights":null,"details":{"apiId":"qwen/qwen-plus","maxOutputTokens":32768,"knowledgeCutoff":"2025-03-31","retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.052000000000000005,"weightsUrl":null},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":21.330665023763434,"multilingual":null,"instruction":null,"math":27.134741159481493},"capabilityCounts":{"math":3,"knowledge":1},"benchmarks":[{"benchmarkId":"epoch-ai:epoch-math-level-5:overall","benchmarkName":"MATH level 5","category":"math","domain":null,"eligible":true,"ranked":true,"score":65.27567975830816,"unit":"percent","family":"epoch-ai:epoch-math-level-5:overall","index":30.705923597134344,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.1334654774046506,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":48.10606060606061,"unit":"percent","family":"gpqa-diamond","index":21.330665023763434,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.7144936812277987,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":17.77777777777778,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":19.262096134041823,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":4.360797453817488,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-2025-02-28-private:overall","benchmarkName":"FrontierMath-2025-02-28-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":1.7241379310344827,"unit":"percent","family":"epoch-ai:epoch-frontiermath-2025-02-28-private:overall","index":31.436203747268305,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0.7657032895812118,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-2025-02-28-public:overall","benchmarkName":"FrontierMath-2025-02-28-Public","category":"math","domain":null,"eligible":false,"ranked":true,"score":0,"unit":"percent","family":"epoch-ai:epoch-frontiermath-2025-02-28-public:overall","index":null,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":0.26,"outputPerMillion":0.78,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":48,"timeToFirstToken":0.431,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":{"elo":1326.4187330596249,"eloUpper":1334.6997238631993,"eloLower":1318.1377422560502,"votes":5819,"rank":235,"rankUpper":null,"rankLower":null,"arenaName":"Qwen-Plus-0125","organization":"alibaba","updatedAt":"2026-11-09"},"quality":{"coveragePercent":25,"benchmarkCount":4,"categoryCount":2,"confidence":"unranked","state":"partial"},"hosts":[{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":0.26,"outputPerMillion":0.78,"cacheReadPerMillion":0.052,"contextTokens":1000000,"maxOutputTokens":32768,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"qwenturbo","providerId":"alibaba","slug":"alibaba-qwen-turbo","name":"Qwen Turbo","provider":"Alibaba","description":"Qwen Turbo is a model from Alibaba. It has 3 scored results in 2 categories. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":15.540433490178934,"multilingual":null,"instruction":null,"math":19.393488964156692},"capabilityCounts":{"math":2,"knowledge":1},"benchmarks":[{"benchmarkId":"epoch-ai:epoch-math-level-5:overall","benchmarkName":"MATH level 5","category":"math","domain":null,"eligible":true,"ranked":true,"score":56.23111782477341,"unit":"percent","family":"epoch-ai:epoch-math-level-5:overall","index":26.31247899449089,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.1364630947550554,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":41.792929292929294,"unit":"percent","family":"gpqa-diamond","index":15.540433490178934,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.617727508870251,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":6.111111111111111,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":12.474498933822488,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.2340279838327226,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":25,"benchmarkCount":3,"categoryCount":2,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"decision-index-0-3-qwen-2-5-1b-rlcd","providerId":"harshatheg","slug":"decision-index-0-3-qwen-2-5-1b-rlcd","name":"Qwen-2.5-1B-RLCD","provider":"harshatheg","description":"Qwen-2.5-1B-RLCD is a model from harshatheg. It has no scored result.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"qwen-agentworld-35b-a3b","providerId":"alibaba","slug":"qwen-agentworld-35b-a3b","name":"Qwen-AgentWorld-35B-A3B","provider":"Alibaba","description":"Qwen-AgentWorld-35B-A3B is a reasoning model from Alibaba in the Qwen-AgentWorld family. It has no scored result.","logoUrl":null,"releaseDate":"2026-06-22","contextWindowTokens":262000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":true,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"qwen-audio-7b","providerId":"alibaba","slug":"qwen-audio-7b","name":"Qwen-Audio 7B","provider":"Alibaba","description":"Qwen-Audio 7B is a standard model from Alibaba. It has no scored result.","logoUrl":null,"releaseDate":"2023-11-30","contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"qwen-audio-chat-7b","providerId":"alibaba","slug":"qwen-audio-chat-7b","name":"Qwen-Audio-Chat 7B","provider":"Alibaba","description":"Qwen-Audio-Chat 7B is a standard model from Alibaba. It has no scored result.","logoUrl":null,"releaseDate":"2023-11-30","contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"qwen1532b","providerId":"alibaba","slug":"alibaba-qwen1-5-32b","name":"Qwen1.5 32B","provider":"Alibaba","description":"Qwen1.5 32B is a model from Alibaba. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":5.40752830640605,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"knowledge":1},"benchmarks":[{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":30.7449494949495,"unit":"percent","family":"gpqa-diamond","index":5.40752830640605,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.0200517973424446,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":15,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"qwen1572b","providerId":"alibaba","slug":"alibaba-qwen1-5-72b","name":"Qwen1.5 72B","provider":"Alibaba","description":"Qwen1.5 72B is a model from Alibaba. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":5,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"knowledge":1},"benchmarks":[{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":28.819444444444443,"unit":"percent","family":"gpqa-diamond","index":5,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.993691228122493,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":15,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"qwen272b","providerId":"alibaba","slug":"alibaba-qwen2-72b","name":"Qwen2 72B","provider":"Alibaba","description":"Qwen2 72B is a model from Alibaba. It has 2 scored results in 2 categories. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":14.613996444805409,"multilingual":null,"instruction":null,"math":17.97502357945352},"capabilityCounts":{"knowledge":1,"math":1},"benchmarks":[{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":40.782828282828284,"unit":"percent","family":"gpqa-diamond","index":14.613996444805409,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.574288051254005,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-math-level-5:overall","benchmarkName":"MATH level 5","category":"math","domain":null,"eligible":true,"ranked":true,"score":39.06722054380665,"unit":"percent","family":"epoch-ai:epoch-math-level-5:overall","index":17.97502357945352,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.0317258861464411,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":25,"benchmarkCount":2,"categoryCount":2,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"qwen2-audio-7b-instruct","providerId":"alibaba","slug":"qwen2-audio-7b-instruct","name":"Qwen2-Audio 7B Instruct","provider":"Alibaba","description":"Qwen2-Audio 7B Instruct is a standard model from Alibaba. It has no scored result.","logoUrl":null,"releaseDate":"2024-08-09","contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"qwen257b72b","providerId":"alibaba","slug":"alibaba-qwen2-5-7b-72b","name":"Qwen2.5 (7B + 72B)","provider":"Alibaba","description":"Qwen2.5 (7B + 72B) is a model from Alibaba. It has 2 scored results in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":24.90855845965384,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"coding":2},"benchmarks":[{"benchmarkId":"swe-bench:swe-bench-verified:overall","benchmarkName":"SWE-bench Verified","category":"coding","domain":null,"eligible":true,"ranked":true,"score":32.8,"unit":"percent","family":"swe-bench-verified","index":25.41981608077674,"aboveFit":false,"run":"SWE-Fixer","source":"SWE-bench team","sourceUrl":"https://www.swebench.com/","measuredAt":"2026-02-26","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"swe-bench:swe-bench-lite:overall","benchmarkName":"SWE-bench Lite","category":"coding","domain":null,"eligible":true,"ranked":true,"score":24.67,"unit":"percent","family":"swe-bench-lite","index":24.397300838530935,"aboveFit":false,"run":"SWE-Fixer","source":"SWE-bench team","sourceUrl":"https://www.swebench.com/","measuredAt":"2025-09-11","stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":20,"benchmarkCount":2,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:swe-bench-team","name":"SWE-bench team","url":"https://www.swebench.com","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"qwen2532b","providerId":"alibaba","slug":"alibaba-qwen2-5-32b","name":"Qwen2.5 32B","provider":"Alibaba","description":"Qwen2.5 32B is a model from Alibaba. It has 3 scored results in 2 categories. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":19.477790933016394,"multilingual":null,"instruction":null,"math":19.718128740524346},"capabilityCounts":{"math":2,"knowledge":1},"benchmarks":[{"benchmarkId":"epoch-ai:epoch-math-level-5:overall","benchmarkName":"MATH level 5","category":"math","domain":null,"eligible":true,"ranked":true,"score":56.07061933534743,"unit":"percent","family":"epoch-ai:epoch-math-level-5:overall","index":26.234515990059847,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.076720798872177,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":46.08585858585859,"unit":"percent","family":"gpqa-diamond","index":19.477790933016394,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.595783004579065,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":7.361111111111112,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":13.201741490988844,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.1420595022864277,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":25,"benchmarkCount":3,"categoryCount":2,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"qwen2572binstruct","providerId":"alibaba","slug":"alibaba-qwen2-5-72b-instruct","name":"Qwen2.5 72B Instruct","provider":"Alibaba","description":"Qwen2.5 72B Instruct is a model from Alibaba. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":"2024-09-19T00:00:00.000Z","contextWindowTokens":32768,"modalities":["text"],"openWeights":true,"details":{"apiId":"qwen/qwen-2.5-72b-instruct","maxOutputTokens":16384,"knowledgeCutoff":"2024-06-30","retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":null,"weightsUrl":"https://huggingface.co/Qwen/Qwen2.5-72B-Instruct"},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":51.60641751327189,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-french:overall","benchmarkName":"EuroEval French","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":61.22375000000001,"unit":"percent","family":"euroeval","index":55.28677702497334,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/french","measuredAt":null,"stderr":0.30902315851728657,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":55.798571428571435,"unit":"percent","family":"euroeval","index":47.92605800157045,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.2287454694544099,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":0.36,"outputPerMillion":0.39999999999999997,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":28,"timeToFirstToken":0.375,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":{"elo":1269.0258520993934,"eloUpper":1273.088825630637,"eloLower":1264.9628785681498,"votes":39406,"rank":287,"rankUpper":null,"rankLower":null,"arenaName":"Qwen2.5-72B-Instruct","organization":"alibaba","updatedAt":"2026-11-09"},"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"qwen2572binstructturbo","providerId":"alibaba","slug":"alibaba-qwen2-5-72b-instruct-turbo","name":"Qwen2.5 72B Instruct Turbo","provider":"Alibaba","description":"Qwen2.5 72B Instruct Turbo is a model from Alibaba. It has no scored result.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[{"benchmarkId":"vals-ai:legal_bench:overall","benchmarkName":"LegalBench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":79.403,"unit":"percent","family":"vals-ai:legal_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","measuredAt":"2026-10-13","stderr":0.66,"costPerTest":null,"latency":0.831},{"benchmarkId":"vals-ai:medqa:overall","benchmarkName":"MedQA","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":77.395,"unit":"percent","family":"vals-ai:medqa:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medqa","measuredAt":"2026-04-16","stderr":0.736,"costPerTest":0.000836,"latency":5.573}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"qwen257b","providerId":"alibaba","slug":"alibaba-qwen2-5-7b","name":"Qwen2.5 7B","provider":"Alibaba","description":"Qwen2.5 7B is a model from Alibaba. It has 2 scored results in 2 categories. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":9.750201956594431,"multilingual":null,"instruction":null,"math":10.37357599089745},"capabilityCounts":{"knowledge":1,"math":1},"benchmarks":[{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":35.47979797979798,"unit":"percent","family":"gpqa-diamond","index":9.750201956594431,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.291485990511179,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":2.5,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":10.37357599089745,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0.9401267913629434,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":25,"benchmarkCount":2,"categoryCount":2,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"qwen257binstruct","providerId":"alibaba","slug":"alibaba-qwen2-5-7b-instruct","name":"Qwen2.5 7B Instruct","provider":"Alibaba","description":"Qwen2.5 7B Instruct is a model from Alibaba. It has no scored result.","logoUrl":null,"releaseDate":"2024-10-16T00:00:00.000Z","contextWindowTokens":32768,"modalities":["text"],"openWeights":true,"details":{"apiId":"qwen/qwen-2.5-7b-instruct","maxOutputTokens":29491,"knowledgeCutoff":"2024-06-30","retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":null,"weightsUrl":"https://huggingface.co/Qwen/Qwen2.5-7B-Instruct"},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":0.09999999999999999,"outputPerMillion":0.19999999999999998,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":86,"timeToFirstToken":0.284,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"qwen257binstructturbo","providerId":"alibaba","slug":"alibaba-qwen2-5-7b-instruct-turbo","name":"Qwen2.5 7B Instruct Turbo","provider":"Alibaba","description":"Qwen2.5 7B Instruct Turbo is a model from Alibaba. It has no scored result.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[{"benchmarkId":"vals-ai:legal_bench:overall","benchmarkName":"LegalBench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":69.559,"unit":"percent","family":"vals-ai:legal_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","measuredAt":"2026-10-13","stderr":0.783,"costPerTest":null,"latency":0.395}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"qwen2-5-coder-32b-instruct","providerId":"alibaba","slug":"qwen2-5-coder-32b-instruct","name":"Qwen2.5 Coder 32B Instruct","provider":"Alibaba","description":"Qwen2.5 Coder 32B Instruct is a standard model from Alibaba in the Qwen2.5 Coder family. It has 3 scored results in 2 categories. It has no index.","logoUrl":null,"releaseDate":"2024-11-12","contextWindowTokens":32768,"modalities":["text"],"openWeights":true,"details":{"apiId":"qwen/qwen-2.5-coder-32b-instruct","maxOutputTokens":29491,"knowledgeCutoff":"2024-06-30","retiresAt":null,"tools":false,"structuredOutput":false,"cacheReadPerMillion":null,"weightsUrl":"https://huggingface.co/Qwen/Qwen2.5-Coder-32B-Instruct"},"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":23.90782304632018,"reasoning":null,"multimodal":null,"knowledge":21.676685101461164,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"knowledge":2,"coding":1},"benchmarks":[{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":41.7,"unit":"percent","family":"gpqa-diamond","index":15.455201282004573,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"swebench-experiments:swe-bench-verified:overall","benchmarkName":"SWE-bench Verified","category":"coding","domain":null,"eligible":true,"ranked":true,"score":38,"unit":"percent","family":"swe-bench-verified","index":29.25511548525194,"aboveFit":false,"run":"Skywork-SWE-32B","source":"SWE-bench team","sourceUrl":"https://github.com/SWE-bench/experiments/tree/main/evaluation/verified","measuredAt":"2026-09-01","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"swe-bench:swe-bench-verified:overall","benchmarkName":"SWE-bench Verified","category":"coding","domain":null,"eligible":true,"ranked":true,"score":23.5,"unit":"percent","family":"swe-bench-verified","index":18.56053060738841,"aboveFit":false,"run":"Skywork-SWE-32B","source":"SWE-bench team","sourceUrl":"https://www.swebench.com/","measuredAt":"2026-02-26","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":3.5,"unit":"percent","family":"hle","index":27.898168920917755,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":0.66,"outputPerMillion":1,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":31,"timeToFirstToken":1.785,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":{"elo":1230.0509479447442,"eloUpper":1238.0952470279321,"eloLower":1222.0066488615566,"votes":5432,"rank":306,"rankUpper":null,"rankLower":null,"arenaName":"Qwen2.5-Coder-32B-Instruct","organization":"alibaba","updatedAt":"2026-11-09"},"quality":{"coveragePercent":35,"benchmarkCount":3,"categoryCount":2,"confidence":"unranked","state":"partial"},"hosts":[{"host":"cloudflare","name":"Cloudflare Workers AI","kind":"cloud","url":"https://developers.cloudflare.com/workers-ai/models/","inputPerMillion":0.66,"outputPerMillion":1,"cacheReadPerMillion":null,"contextTokens":32768,"maxOutputTokens":32768,"deprecated":false},{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":0.66,"outputPerMillion":1,"cacheReadPerMillion":null,"contextTokens":32768,"maxOutputTokens":29491,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:swe-bench-team","name":"SWE-bench team","url":"https://github.com","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"qwen25vl72binstruct","providerId":"alibaba","slug":"alibaba-qwen2-5-vl-72b-instruct","name":"Qwen2.5 VL 72B Instruct","provider":"Alibaba","description":"Qwen2.5 VL 72B Instruct is a model from Alibaba. It has no scored result.","logoUrl":null,"releaseDate":"2025-02-01T11:45:11.000Z","contextWindowTokens":128000,"modalities":["text","image"],"openWeights":true,"details":{"apiId":"qwen/qwen2.5-vl-72b-instruct","maxOutputTokens":115200,"knowledgeCutoff":"2024-06-30","retiresAt":null,"tools":false,"structuredOutput":true,"cacheReadPerMillion":0.39999999999999997,"weightsUrl":"https://huggingface.co/Qwen/Qwen2.5-VL-72B-Instruct"},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":0.7999999999999999,"outputPerMillion":1,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":26,"timeToFirstToken":1.513,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":0.8,"outputPerMillion":1,"cacheReadPerMillion":0.4,"contextTokens":128000,"maxOutputTokens":115200,"deprecated":false},{"host":"ovhcloud","name":"OVHcloud","kind":"cloud","url":"https://www.ovhcloud.com/en/public-cloud/ai-endpoints/catalog//","inputPerMillion":1.01,"outputPerMillion":1.01,"cacheReadPerMillion":null,"contextTokens":32768,"maxOutputTokens":32768,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"qwen2-5-1m","providerId":"alibaba","slug":"qwen2-5-1m","name":"Qwen2.5-1M","provider":"Alibaba","description":"Qwen2.5-1M is a standard model from Alibaba. It has no scored result.","logoUrl":null,"releaseDate":"2025-01-27","contextWindowTokens":1000000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"qwen2-5-72b","providerId":"alibaba","slug":"qwen2-5-72b","name":"Qwen2.5-72B","provider":"Alibaba","description":"Qwen2.5-72B is a standard model from Alibaba. It has 3 scored results in 2 categories. It has no index.","logoUrl":null,"releaseDate":"2024-09-19","contextWindowTokens":128000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":22.286053226804874,"multilingual":null,"instruction":null,"math":21.644498777610735},"capabilityCounts":{"math":2,"knowledge":1},"benchmarks":[{"benchmarkId":"epoch-ai:epoch-math-level-5:overall","benchmarkName":"MATH level 5","category":"math","domain":null,"eligible":true,"ranked":true,"score":63.1703172205438,"unit":"percent","family":"epoch-ai:epoch-math-level-5:overall","index":29.68323242136243,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.071424933114761,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":49.14772727272727,"unit":"percent","family":"gpqa-diamond","index":22.286053226804874,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.6947767404178666,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":8.055555555555555,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":13.605765133859045,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.609821301896721,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":25,"benchmarkCount":3,"categoryCount":2,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"qwen2-5-max","providerId":"alibaba","slug":"qwen2-5-max","name":"Qwen2.5-Max","provider":"Alibaba","description":"Qwen2.5-Max is a standard model from Alibaba. It has 6 scored results in 3 categories. It has no index.","logoUrl":null,"releaseDate":"2025-01-28","contextWindowTokens":null,"modalities":["text"],"openWeights":false,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":21.454752065904778,"reasoning":null,"multimodal":null,"knowledge":35.43266913853501,"multilingual":null,"instruction":null,"math":26.91737950481243},"capabilityCounts":{"knowledge":2,"math":3,"coding":1},"benchmarks":[{"benchmarkId":"mmluPro","benchmarkName":"Massive Multitask Language Understanding Professional","category":"knowledge","score":76.1,"unit":"percent","family":"mmlu-pro","index":40.35761949109784,"aboveFit":false,"run":null,"source":"Yubo Wang, Xueguang Ma, Ge Zhang, Yuansheng Ni, Abhranil Chandra, Shiguang Guo, Weiming Ren, Aaran Arulraj, Xuan He, Ziyan Jiang, Tianle Li, Max Ku, Kai Wang, Alex Zhuang, Rongqi Fan, Xiang Yue, Wenhu Chen","sourceUrl":"https://arxiv.org/abs/2406.01574","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-math-level-5:overall","benchmarkName":"MATH level 5","category":"math","domain":null,"eligible":true,"ranked":true,"score":67.18277945619336,"unit":"percent","family":"epoch-ai:epoch-math-level-5:overall","index":31.632307532138498,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.1143967442882197,"costPerTest":null,"latency":null},{"benchmarkId":"gpqa","benchmarkName":"Graduate-Level Google-Proof Q&A","category":"knowledge","score":60.1,"unit":"percent","family":"gpqa-diamond","index":32.33117850052861,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":56.12373737373737,"unit":"percent","family":"gpqa-diamond","index":28.684259071415745,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.785883147570945,"costPerTest":null,"latency":null},{"benchmarkId":"liveCodeBench","benchmarkName":"LiveCodeBench: Holistic and Contamination Free Evaluation of Large Language Models for Code","category":"coding","score":38.7,"unit":"percent","family":"livecodebench","index":21.454752065904778,"aboveFit":false,"run":null,"source":"Naman Jain, King Han, Alex Gu, Wen-Ding Li, Fanjia Yan, Tianjun Zhang, Sida Wang, Armando Solar-Lezama, Koushik Sen, Ion Stoica","sourceUrl":"https://arxiv.org/abs/2403.07974","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":16.11111111111111,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":18.29243939115335,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":4.102194808229969,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-2025-02-28-private:overall","benchmarkName":"FrontierMath-2025-02-28-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":1.0344827586206897,"unit":"percent","family":"epoch-ai:epoch-frontiermath-2025-02-28-private:overall","index":30.82739159114545,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0.5951886714450794,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-2025-02-28-public:overall","benchmarkName":"FrontierMath-2025-02-28-Public","category":"math","domain":null,"eligible":false,"ranked":true,"score":0,"unit":"percent","family":"epoch-ai:epoch-frontiermath-2025-02-28-public:overall","index":null,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":{"elo":1366.6746862900786,"eloUpper":1370.765644520045,"eloLower":1362.5837280601122,"votes":32484,"rank":193,"rankUpper":null,"rankLower":null,"arenaName":"Qwen2.5-Max","organization":"alibaba","updatedAt":"2026-11-09"},"quality":{"coveragePercent":45,"benchmarkCount":6,"categoryCount":3,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"qwen2-5-omni-7b","providerId":"alibaba","slug":"qwen2-5-omni-7b","name":"Qwen2.5-Omni 7B","provider":"Alibaba","description":"Qwen2.5-Omni 7B is a standard model from Alibaba. It has no scored result.","logoUrl":null,"releaseDate":"2025-03-26","contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"qwen2-5-vl-32b","providerId":"alibaba","slug":"qwen2-5-vl-32b","name":"Qwen2.5-VL-32B","provider":"Alibaba","description":"Qwen2.5-VL-32B is a standard model from Alibaba. It has 4 scored results in 3 categories. It has no index.","logoUrl":null,"releaseDate":"2025-01-26","contextWindowTokens":32000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":31.881392789768714,"reasoning":null,"multimodal":19.57272215123436,"knowledge":25.27976853139816,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"coding":1,"knowledge":2,"multimodal":1},"benchmarks":[{"benchmarkId":"humaneval","benchmarkName":"Evaluating Large Language Models Trained on Code","category":"coding","score":91.5,"unit":"percent","family":"humaneval","index":31.881392789768714,"aboveFit":false,"run":null,"source":"Mark Chen, Jerry Tworek, Heewoo Jun, Qiming Yuan, Henrique Ponde de Oliveira Pinto, Jared Kaplan, Harri Edwards, Yuri Burda, Nicholas Joseph, Greg Brockman, Alex Ray, Raul Puri, Gretchen Krueger, Michael Petrov, Heidy Khlaaf, Girish Sastry, Pamela Mishkin, Brooke Chan, Scott Gray, Nick Ryder, Mikhail Pavlov, Alethea Power, Lukasz Kaiser, Mohammad Bavarian, Clemens Winter, Philippe Tillet, Felipe Petroski Such, Dave Cummings, Matthias Plappert, Fotios Chantzis, Elizabeth Barnes, Ariel Herbert-Voss, William Hebgen Guss, Alex Nichol, Alex Paino, Nikolas Tezak, Jie Tang, Igor Babuschkin, Suchir Balaji, Shantanu Jain, William Saunders, Christopher Hesse, Andrew N. Carr, Jan Leike, Josh Achiam, Vedant Misra, Evan Morikawa, Alec Radford, Matthew Knight, Miles Brundage, Mira Murati, Katie Mayer, Peter Welinder, Bob McGrew, Dario Amodei, Sam McCandlish, Ilya Sutskever, Wojciech Zaremba","sourceUrl":"https://arxiv.org/abs/2107.03374","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmluPro","benchmarkName":"Massive Multitask Language Understanding Professional","category":"knowledge","score":68.8,"unit":"percent","family":"mmlu-pro","index":31.16049327863668,"aboveFit":false,"run":null,"source":"Yubo Wang, Xueguang Ma, Ge Zhang, Yuansheng Ni, Abhranil Chandra, Shiguang Guo, Weiming Ren, Aaran Arulraj, Xuan He, Ziyan Jiang, Tianle Li, Max Ku, Kai Wang, Alex Zhuang, Rongqi Fan, Xiang Yue, Wenhu Chen","sourceUrl":"https://arxiv.org/abs/2406.01574","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmmuPro","benchmarkName":"Massive Multi-discipline Multimodal Understanding Pro","category":"multimodal","score":49.5,"unit":"percent","family":"mmmu-pro","index":19.57272215123436,"aboveFit":false,"run":null,"source":"MMMU-Pro authors","sourceUrl":"https://arxiv.org/abs/2409.02813","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gpqa","benchmarkName":"Graduate-Level Google-Proof Q&A","category":"knowledge","score":46,"unit":"percent","family":"gpqa-diamond","index":19.399043784159645,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":45,"benchmarkCount":4,"categoryCount":3,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"qwen306b","providerId":"alibaba","slug":"alibaba-qwen3-0-6b","name":"Qwen3 0.6B","provider":"Alibaba","description":"Qwen3 0.6B is a model from Alibaba. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":5.914862326750438,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-dutch:overall","benchmarkName":"EuroEval Dutch","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":24.834444444444436,"unit":"percent","family":"euroeval","index":5.914862326750438,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/dutch","measuredAt":null,"stderr":0.36924347081666964,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"qwen306bnothinking","providerId":"alibaba","slug":"alibaba-qwen3-0-6b-no-thinking","name":"Qwen3 0.6B No Thinking","provider":"Alibaba","description":"Qwen3 0.6B No Thinking is a model from Alibaba. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":5,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-dutch:overall","benchmarkName":"EuroEval Dutch","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":24.559999999999995,"unit":"percent","family":"euroeval","index":5.542504363506936,"aboveFit":false,"run":"no-thinking effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/dutch","measuredAt":null,"stderr":0.405078867492355,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-portuguese:overall","benchmarkName":"EuroEval Portuguese","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":24.235,"unit":"percent","family":"euroeval","index":5.101554143876477,"aboveFit":false,"run":"no-thinking effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/portuguese","measuredAt":null,"stderr":0.44632002251747566,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-french:overall","benchmarkName":"EuroEval French","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":23.61375,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":"no-thinking effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/french","measuredAt":null,"stderr":0.9281887604361518,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":23.14125,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":"no-thinking effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":0.3530647072421711,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":21.8775,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":"no-thinking effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.6725824763551307,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-spanish:overall","benchmarkName":"EuroEval Spanish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":20.643750000000004,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":"no-thinking effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/spanish","measuredAt":null,"stderr":0.4470441113581522,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-italian:overall","benchmarkName":"EuroEval Italian","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":20.59,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":"no-thinking effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/italian","measuredAt":null,"stderr":0.482276502641379,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":20.154285714285717,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":"no-thinking effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.5556445053507959,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"qwen317b","providerId":"alibaba","slug":"alibaba-qwen3-1-7b","name":"Qwen3 1.7B","provider":"Alibaba","description":"Qwen3 1.7B is a model from Alibaba. It has 2 scored results in 2 categories. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":5.29172367573436,"multilingual":null,"instruction":null,"math":13.605765133859045},"capabilityCounts":{"knowledge":1,"math":1},"benchmarks":[{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":30.61868686868687,"unit":"percent","family":"gpqa-diamond","index":5.29172367573436,"aboveFit":false,"run":"none effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.2564006554294687,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":8.055555555555555,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":13.605765133859045,"aboveFit":false,"run":"none effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.81353251405273,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":25,"benchmarkCount":2,"categoryCount":2,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"qwen317bnothinking","providerId":"alibaba","slug":"alibaba-qwen3-1-7b-no-thinking","name":"Qwen3 1.7B No Thinking","provider":"Alibaba","description":"Qwen3 1.7B No Thinking is a model from Alibaba. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":19.833719825756887,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-french:overall","benchmarkName":"EuroEval French","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":42.60374999999999,"unit":"percent","family":"euroeval","index":30.023721364913882,"aboveFit":false,"run":"no-thinking effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/french","measuredAt":null,"stderr":0.3331689698936562,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-portuguese:overall","benchmarkName":"EuroEval Portuguese","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":38.5325,"unit":"percent","family":"euroeval","index":24.49997188285068,"aboveFit":false,"run":"no-thinking effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/portuguese","measuredAt":null,"stderr":0.33925239498049237,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-italian:overall","benchmarkName":"EuroEval Italian","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":37.03249999999999,"unit":"percent","family":"euroeval","index":22.464817023017744,"aboveFit":false,"run":"no-thinking effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/italian","measuredAt":null,"stderr":0.40915690755503564,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-dutch:overall","benchmarkName":"EuroEval Dutch","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":36.547777777777775,"unit":"percent","family":"euroeval","index":21.807160498868033,"aboveFit":false,"run":"no-thinking effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/dutch","measuredAt":null,"stderr":0.26152213512529643,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-spanish:overall","benchmarkName":"EuroEval Spanish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":33.63875,"unit":"percent","family":"euroeval","index":17.860279152645745,"aboveFit":false,"run":"no-thinking effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/spanish","measuredAt":null,"stderr":0.4175205833249422,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":33.526250000000005,"unit":"percent","family":"euroeval","index":17.707642538158282,"aboveFit":false,"run":"no-thinking effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":0.3059769190641673,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":32.565,"unit":"percent","family":"euroeval","index":16.403447465482003,"aboveFit":false,"run":"no-thinking effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.5599693071945997,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":31.069999999999997,"unit":"percent","family":"euroeval","index":14.375076455181855,"aboveFit":false,"run":"no-thinking effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.3351910816601328,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"qwen314b","providerId":"alibaba","slug":"alibaba-qwen3-14b","name":"Qwen3 14B","provider":"Alibaba","description":"Qwen3 14B is a model from Alibaba. It has 2 scored results in 2 categories. It has no index.","logoUrl":null,"releaseDate":"2025-04-28T21:41:18.000Z","contextWindowTokens":40960,"modalities":["text"],"openWeights":true,"details":{"apiId":"qwen/qwen3-14b","maxOutputTokens":16384,"knowledgeCutoff":"2025-03-31","retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":null,"weightsUrl":"https://huggingface.co/Qwen/Qwen3-14B"},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":26.19445951197441,"multilingual":null,"instruction":null,"math":23.94877039133613},"capabilityCounts":{"knowledge":1,"math":1},"benchmarks":[{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":53.40909090909091,"unit":"percent","family":"gpqa-diamond","index":26.19445951197441,"aboveFit":false,"run":"none effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.752462335115392,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":25.833333333333336,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":23.94877039133613,"aboveFit":false,"run":"none effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":4.823265376162594,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":0.12,"outputPerMillion":0.24,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":54.5,"timeToFirstToken":0.921,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":25,"benchmarkCount":2,"categoryCount":2,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"qwen314bnothinking","providerId":"alibaba","slug":"alibaba-qwen3-14b-no-thinking","name":"Qwen3 14B No Thinking","provider":"Alibaba","description":"Qwen3 14B No Thinking is a model from Alibaba. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":41.63851974621873,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-french:overall","benchmarkName":"EuroEval French","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":57.09250000000001,"unit":"percent","family":"euroeval","index":49.68162134851681,"aboveFit":false,"run":"no-thinking effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/french","measuredAt":null,"stderr":0.36259481518631786,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-dutch:overall","benchmarkName":"EuroEval Dutch","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":54.382222222222225,"unit":"percent","family":"euroeval","index":46.00439802122238,"aboveFit":false,"run":"no-thinking effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/dutch","measuredAt":null,"stderr":0.2673071013268318,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":53.22375,"unit":"percent","family":"euroeval","index":44.43261777253104,"aboveFit":false,"run":"no-thinking effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":0.26628520893958796,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-portuguese:overall","benchmarkName":"EuroEval Portuguese","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":51.985,"unit":"percent","family":"euroeval","index":42.75191905078567,"aboveFit":false,"run":"no-thinking effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/portuguese","measuredAt":null,"stderr":0.3449637662132068,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-italian:overall","benchmarkName":"EuroEval Italian","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":50.34374999999999,"unit":"percent","family":"euroeval","index":40.5251204416518,"aboveFit":false,"run":"no-thinking effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/italian","measuredAt":null,"stderr":0.3364706525092493,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":50.08428571428571,"unit":"percent","family":"euroeval","index":40.1730871069688,"aboveFit":false,"run":"no-thinking effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.2691862264117463,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-spanish:overall","benchmarkName":"EuroEval Spanish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":48.67875,"unit":"percent","family":"euroeval","index":38.26609854723726,"aboveFit":false,"run":"no-thinking effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/spanish","measuredAt":null,"stderr":0.24784382683456127,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":47.05624999999999,"unit":"percent","family":"euroeval","index":36.064739373851296,"aboveFit":false,"run":"no-thinking effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.45219430281683115,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"qwen3-235b-a22b-reasoning","providerId":"alibaba","slug":"qwen3-235b-a22b-reasoning","name":"Qwen3 235B-A22B (Reasoning)","provider":"Alibaba","description":"Qwen3 235B-A22B (Reasoning) is a reasoning model from Alibaba in the Qwen3 235B-A22B family. It has 9 scored results in 4 categories. It has no index.","logoUrl":null,"releaseDate":"2025-04-29","contextWindowTokens":128000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":true,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":44.05160203121554,"multilingual":46.71732034895929,"instruction":31.143045710211137,"math":44.535446261340034},"capabilityCounts":{"math":3,"knowledge":4,"instruction":1,"multilingual":1},"benchmarks":[{"benchmarkId":"math500","benchmarkName":"MATH-500 Problem Set","category":"math","score":98,"unit":"percent","family":"math-500","index":47.68908807450553,"aboveFit":false,"run":null,"source":"Dan Hendrycks, Collin Burns, Saurav Kadavath, Akul Arora, Steven Basart, Eric Tang, Dawn Song, Jacob Steinhardt","sourceUrl":"https://arxiv.org/abs/2103.03874","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmluRedux","benchmarkName":"MMLU-Redux","category":"knowledge","score":92.7,"unit":"percent","family":"mmluRedux","index":47.41583358298924,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"cEval","benchmarkName":"C-Eval","category":"knowledge","score":89.6,"unit":"percent","family":"cEval","index":45.26292924626172,"aboveFit":false,"run":null,"source":"C-Eval authors","sourceUrl":"https://arxiv.org/abs/2305.08322","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aime2024","benchmarkName":"American Invitational Mathematics Examination 2024","category":"math","score":85.7,"unit":"percent","family":"aime2024","index":42.68602491841169,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmmlu","benchmarkName":"MMMLU","category":"knowledge","score":84.3,"unit":"percent","family":"mmmlu","index":41.10756737096494,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"ifeval","benchmarkName":"Instruction-Following Eval","category":"instruction","score":83.4,"unit":"percent","family":"ifeval","index":31.143045710211137,"aboveFit":false,"run":null,"source":"Jeffrey Zhou, Tianjian Lu, Swaroop Mishra, Siddhartha Brahma, Sujoy Basu, Yi Luan, Denny Zhou, Le Hou","sourceUrl":"https://arxiv.org/abs/2311.07911","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aime2025","benchmarkName":"American Invitational Mathematics Examination 2025","category":"math","score":81.5,"unit":"percent","family":"aime2025","index":43.23122579110289,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"include","benchmarkName":"INCLUDE","category":"multilingual","score":78.7,"unit":"percent","family":"include","index":46.71732034895929,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gpqa","benchmarkName":"Graduate-Level Google-Proof Q&A","category":"knowledge","score":71.1,"unit":"percent","family":"gpqa-diamond","index":42.42007792464624,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"bfclV3","benchmarkName":"Berkeley Function Calling Leaderboard v3","category":"agentic","score":70.8,"unit":"percent","family":"bfclV3","index":null,"aboveFit":false,"run":null,"source":"Shishir G. Patil et al.","sourceUrl":"https://gorilla.cs.berkeley.edu/leaderboard.html","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"liveCodeBenchV5","benchmarkName":"LiveCodeBench v5","category":"coding","score":70.7,"unit":"percent","family":"liveCodeBenchV5","index":null,"aboveFit":false,"run":null,"source":"LiveCodeBench maintainers","sourceUrl":"https://github.com/LiveCodeBench/LiveCodeBench","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":35,"benchmarkCount":9,"categoryCount":4,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"qwen330ba3b","providerId":"alibaba","slug":"alibaba-qwen3-30b-a3b","name":"Qwen3 30B A3B","provider":"Alibaba","description":"Qwen3 30B A3B is a model from Alibaba. It has 2 scored results in 2 categories. It has no index.","logoUrl":null,"releaseDate":"2025-04-28T22:16:44.000Z","contextWindowTokens":40960,"modalities":["text"],"openWeights":true,"details":{"apiId":"qwen/qwen3-30b-a3b","maxOutputTokens":16384,"knowledgeCutoff":"2025-03-31","retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":null,"weightsUrl":"https://huggingface.co/Qwen/Qwen3-30B-A3B"},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":23.415148375853846,"multilingual":null,"instruction":null,"math":23.787160934188048},"capabilityCounts":{"knowledge":1,"math":1},"benchmarks":[{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":50.378787878787875,"unit":"percent","family":"gpqa-diamond","index":23.415148375853846,"aboveFit":false,"run":"none effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.827514183647295,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":25.555555555555554,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":23.787160934188048,"aboveFit":false,"run":"none effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":5.2095537290747576,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":0.12,"outputPerMillion":0.5,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":50,"timeToFirstToken":0.415,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":{"elo":1316.78847404025,"eloUpper":1321.5056030476926,"eloLower":1312.0713450328074,"votes":26137,"rank":244,"rankUpper":null,"rankLower":null,"arenaName":"Qwen3-30B-A3B","organization":"alibaba","updatedAt":"2026-11-09"},"quality":{"coveragePercent":25,"benchmarkCount":2,"categoryCount":2,"confidence":"unranked","state":"partial"},"hosts":[{"host":"cloudflare","name":"Cloudflare Workers AI","kind":"cloud","url":"https://developers.cloudflare.com/workers-ai/models/","inputPerMillion":0.0509,"outputPerMillion":0.335,"cacheReadPerMillion":null,"contextTokens":32768,"maxOutputTokens":32768,"deprecated":true},{"host":"deepinfra","name":"Deep Infra","kind":"host","url":"https://deepinfra.com/models","inputPerMillion":0.12,"outputPerMillion":0.5,"cacheReadPerMillion":null,"contextTokens":40960,"maxOutputTokens":16384,"deprecated":false},{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":0.12,"outputPerMillion":0.5,"cacheReadPerMillion":null,"contextTokens":40960,"maxOutputTokens":16384,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"qwen330ba3binstruct2507","providerId":"alibaba","slug":"alibaba-qwen3-30b-a3b-instruct-2507","name":"Qwen3 30B A3B Instruct 2507","provider":"Alibaba","description":"Qwen3 30B A3B Instruct 2507 is a model from Alibaba. It has no scored result.","logoUrl":null,"releaseDate":"2025-07-29T16:36:05.000Z","contextWindowTokens":262144,"modalities":["text"],"openWeights":true,"details":{"apiId":"qwen/qwen3-30b-a3b-instruct-2507","maxOutputTokens":235929,"knowledgeCutoff":"2025-06-30","retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.09999999999999999,"weightsUrl":"https://huggingface.co/Qwen/Qwen3-30B-A3B-Instruct-2507"},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":0.09999999999999999,"outputPerMillion":0.3,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":44,"timeToFirstToken":0.289,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":{"elo":1383.8377701870368,"eloUpper":1388.7199838946353,"eloLower":1378.9555564794382,"votes":23287,"rank":175,"rankUpper":null,"rankLower":null,"arenaName":"Qwen3-30B-A3B-Instruct-2507","organization":"alibaba","updatedAt":"2026-11-09"},"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[{"host":"siliconflow","name":"SiliconFlow","kind":"host","url":"https://cloud.siliconflow.com/models","inputPerMillion":0.09,"outputPerMillion":0.3,"cacheReadPerMillion":null,"contextTokens":262000,"maxOutputTokens":262000,"deprecated":false},{"host":"nebius","name":"Nebius","kind":"host","url":"https://docs.tokenfactory.nebius.com/","inputPerMillion":0.1,"outputPerMillion":0.3,"cacheReadPerMillion":0.1,"contextTokens":262144,"maxOutputTokens":8192,"deprecated":false},{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":0.1,"outputPerMillion":0.3,"cacheReadPerMillion":0.1,"contextTokens":262144,"maxOutputTokens":235929,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"qwen330ba3bthinking2507","providerId":"alibaba","slug":"alibaba-qwen3-30b-a3b-thinking-2507","name":"Qwen3 30B A3B Thinking 2507","provider":"Alibaba","description":"Qwen3 30B A3B Thinking 2507 is a model from Alibaba. It has no scored result.","logoUrl":null,"releaseDate":"2025-08-28T16:39:52.000Z","contextWindowTokens":81920,"modalities":["text"],"openWeights":true,"details":{"apiId":"qwen/qwen3-30b-a3b-thinking-2507","maxOutputTokens":32768,"knowledgeCutoff":"2025-06-30","retiresAt":"2026-10-09","tools":true,"structuredOutput":false,"cacheReadPerMillion":null,"weightsUrl":"https://huggingface.co/Qwen/Qwen3-30B-A3B-Thinking-2507"},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":0.19999999999999998,"outputPerMillion":2.4,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":121.5,"timeToFirstToken":0.312,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"qwen3-32b","providerId":"alibaba","slug":"qwen3-32b","name":"Qwen3 32B","provider":"Alibaba","description":"Qwen3 32B is a standard model from Alibaba. It has 10 scored results in 4 categories. It has no index.","logoUrl":null,"releaseDate":"2025-04-29","contextWindowTokens":131072,"modalities":["text"],"openWeights":true,"details":{"apiId":"qwen/qwen3-32b","maxOutputTokens":16384,"knowledgeCutoff":"2025-03-31","retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":null,"weightsUrl":"https://huggingface.co/Qwen/Qwen3-32B"},"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":32.59245990698656,"multilingual":34.84530042360028,"instruction":30.74026593380142,"math":21.836615665545633},"capabilityCounts":{"math":4,"knowledge":4,"instruction":1,"multilingual":1},"benchmarks":[{"benchmarkId":"math500","benchmarkName":"MATH-500 Problem Set","category":"math","score":88.6,"unit":"percent","family":"math-500","index":37.75556087766464,"aboveFit":false,"run":null,"source":"Dan Hendrycks, Collin Burns, Saurav Kadavath, Akul Arora, Steven Basart, Eric Tang, Dawn Song, Jacob Steinhardt","sourceUrl":"https://arxiv.org/abs/2103.03874","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmluRedux","benchmarkName":"MMLU-Redux","category":"knowledge","score":85.7,"unit":"percent","family":"mmluRedux","index":35.54136527000765,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"cEval","benchmarkName":"C-Eval","category":"knowledge","score":83.3,"unit":"percent","family":"cEval","index":36.81541823486713,"aboveFit":false,"run":null,"source":"C-Eval authors","sourceUrl":"https://arxiv.org/abs/2305.08322","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"ifeval","benchmarkName":"Instruction-Following Eval","category":"instruction","score":83.2,"unit":"percent","family":"ifeval","index":30.74026593380142,"aboveFit":false,"run":null,"source":"Jeffrey Zhou, Tianjian Lu, Swaroop Mishra, Siddhartha Brahma, Sujoy Basu, Yi Luan, Denny Zhou, Le Hou","sourceUrl":"https://arxiv.org/abs/2311.07911","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmmlu","benchmarkName":"MMMLU","category":"knowledge","score":76.5,"unit":"percent","family":"mmmlu","index":30.953999238502227,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"include","benchmarkName":"INCLUDE","category":"multilingual","score":70.9,"unit":"percent","family":"include","index":34.84530042360028,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"bfclV3","benchmarkName":"Berkeley Function Calling Leaderboard v3","category":"agentic","score":63,"unit":"percent","family":"bfclV3","index":null,"aboveFit":false,"run":null,"source":"Shishir G. Patil et al.","sourceUrl":"https://gorilla.cs.berkeley.edu/leaderboard.html","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gpqa","benchmarkName":"Graduate-Level Google-Proof Q&A","category":"knowledge","score":54.6,"unit":"percent","family":"gpqa-diamond","index":27.286728788469794,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":54.10353535353535,"unit":"percent","family":"gpqa-diamond","index":26.831384980668705,"aboveFit":false,"run":"none effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.81462342016025,"costPerTest":null,"latency":null},{"benchmarkId":"liveCodeBenchV5","benchmarkName":"LiveCodeBench v5","category":"coding","score":31.3,"unit":"percent","family":"liveCodeBenchV5","index":null,"aboveFit":false,"run":null,"source":"LiveCodeBench maintainers","sourceUrl":"https://github.com/LiveCodeBench/LiveCodeBench","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aime2024","benchmarkName":"American Invitational Mathematics Examination 2024","category":"math","score":31,"unit":"percent","family":"aime2024","index":17.28996271516771,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":23.055555555555557,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":22.332675819855336,"aboveFit":false,"run":"none effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":5.148605337836995,"costPerTest":null,"latency":null},{"benchmarkId":"aime2025","benchmarkName":"American Invitational Mathematics Examination 2025","category":"math","score":20.2,"unit":"percent","family":"aime2025","index":9.96826324949484,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":0.08,"outputPerMillion":0.28,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":46,"timeToFirstToken":0.387,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":{"elo":1340.075563631161,"eloUpper":1349.3842114339502,"eloLower":1330.7669158283718,"votes":3926,"rank":219,"rankUpper":null,"rankLower":null,"arenaName":"Qwen3-32B","organization":"alibaba","updatedAt":"2026-11-09"},"quality":{"coveragePercent":35,"benchmarkCount":10,"categoryCount":4,"confidence":"unranked","state":"partial"},"hosts":[{"host":"deepinfra","name":"Deep Infra","kind":"host","url":"https://deepinfra.com/models","inputPerMillion":0.08,"outputPerMillion":0.28,"cacheReadPerMillion":null,"contextTokens":40960,"maxOutputTokens":16384,"deprecated":false},{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":0.08,"outputPerMillion":0.28,"cacheReadPerMillion":null,"contextTokens":131072,"maxOutputTokens":16384,"deprecated":false},{"host":"siliconflow","name":"SiliconFlow","kind":"host","url":"https://cloud.siliconflow.com/models","inputPerMillion":0.14,"outputPerMillion":0.57,"cacheReadPerMillion":null,"contextTokens":131000,"maxOutputTokens":131000,"deprecated":false},{"host":"amazon-bedrock","name":"Amazon Bedrock","kind":"cloud","url":"https://docs.aws.amazon.com/bedrock/latest/userguide/models-supported.html","inputPerMillion":0.15,"outputPerMillion":0.6,"cacheReadPerMillion":null,"contextTokens":32768,"maxOutputTokens":16384,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"qwen3-32b-reasoning","providerId":"alibaba","slug":"qwen3-32b-reasoning","name":"Qwen3 32B (Reasoning)","provider":"Alibaba","description":"Qwen3 32B (Reasoning) is a reasoning model from Alibaba in the Qwen3 32B family. It has 9 scored results in 4 categories. It has no index.","logoUrl":null,"releaseDate":"2025-04-29","contextWindowTokens":128000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":true,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":40.69403976789661,"multilingual":39.10705116603685,"instruction":34.36528392148884,"math":42.032650140850805},"capabilityCounts":{"math":3,"knowledge":4,"instruction":1,"multilingual":1},"benchmarks":[{"benchmarkId":"math500","benchmarkName":"MATH-500 Problem Set","category":"math","score":97.2,"unit":"percent","family":"math-500","index":46.84368150456162,"aboveFit":false,"run":null,"source":"Dan Hendrycks, Collin Burns, Saurav Kadavath, Akul Arora, Steven Basart, Eric Tang, Dawn Song, Jacob Steinhardt","sourceUrl":"https://arxiv.org/abs/2103.03874","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmluRedux","benchmarkName":"MMLU-Redux","category":"knowledge","score":90.9,"unit":"percent","family":"mmluRedux","index":44.36239887393683,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"cEval","benchmarkName":"C-Eval","category":"knowledge","score":87.3,"unit":"percent","family":"cEval","index":42.17891728972084,"aboveFit":false,"run":null,"source":"C-Eval authors","sourceUrl":"https://arxiv.org/abs/2305.08322","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"ifeval","benchmarkName":"Instruction-Following Eval","category":"instruction","score":85,"unit":"percent","family":"ifeval","index":34.36528392148884,"aboveFit":false,"run":null,"source":"Jeffrey Zhou, Tianjian Lu, Swaroop Mishra, Siddhartha Brahma, Sujoy Basu, Yi Luan, Denny Zhou, Le Hou","sourceUrl":"https://arxiv.org/abs/2311.07911","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aime2024","benchmarkName":"American Invitational Mathematics Examination 2024","category":"math","score":81.4,"unit":"percent","family":"aime2024","index":40.68962514740714,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmmlu","benchmarkName":"MMMLU","category":"knowledge","score":80.6,"unit":"percent","family":"mmmlu","index":36.291131205565954,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"include","benchmarkName":"INCLUDE","category":"multilingual","score":73.7,"unit":"percent","family":"include","index":39.10705116603685,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aime2025","benchmarkName":"American Invitational Mathematics Examination 2025","category":"math","score":72.9,"unit":"percent","family":"aime2025","index":38.56464377058366,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"bfclV3","benchmarkName":"Berkeley Function Calling Leaderboard v3","category":"agentic","score":70.3,"unit":"percent","family":"bfclV3","index":null,"aboveFit":false,"run":null,"source":"Shishir G. Patil et al.","sourceUrl":"https://gorilla.cs.berkeley.edu/leaderboard.html","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gpqa","benchmarkName":"Graduate-Level Google-Proof Q&A","category":"knowledge","score":68.4,"unit":"percent","family":"gpqa-diamond","index":39.94371170236283,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"liveCodeBenchV5","benchmarkName":"LiveCodeBench v5","category":"coding","score":65.7,"unit":"percent","family":"liveCodeBenchV5","index":null,"aboveFit":false,"run":null,"source":"LiveCodeBench maintainers","sourceUrl":"https://github.com/LiveCodeBench/LiveCodeBench","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":35,"benchmarkCount":9,"categoryCount":4,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"qwen332bnothinking","providerId":"alibaba","slug":"alibaba-qwen3-32b-no-thinking","name":"Qwen3 32B No Thinking","provider":"Alibaba","description":"Qwen3 32B No Thinking is a model from Alibaba. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":47.183953318752785,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-french:overall","benchmarkName":"EuroEval French","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":57.207499999999996,"unit":"percent","family":"euroeval","index":49.83764988777065,"aboveFit":false,"run":"no-thinking effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/french","measuredAt":null,"stderr":0.3662606653737199,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":56.57249999999999,"unit":"percent","family":"euroeval","index":48.97610099710804,"aboveFit":false,"run":"no-thinking effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":0.2680980464307788,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-dutch:overall","benchmarkName":"EuroEval Dutch","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":56.31111111111111,"unit":"percent","family":"euroeval","index":48.621456418755685,"aboveFit":false,"run":"no-thinking effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/dutch","measuredAt":null,"stderr":0.2821959742824737,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-portuguese:overall","benchmarkName":"EuroEval Portuguese","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":55.51749999999999,"unit":"percent","family":"euroeval","index":47.54470874569221,"aboveFit":false,"run":"no-thinking effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/portuguese","measuredAt":null,"stderr":0.3339910178432947,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":54.98571428571429,"unit":"percent","family":"euroeval","index":46.823197891813365,"aboveFit":false,"run":"no-thinking effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.2473162067525074,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-italian:overall","benchmarkName":"EuroEval Italian","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":53.55,"unit":"percent","family":"euroeval","index":44.87526395454469,"aboveFit":false,"run":"no-thinking effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/italian","measuredAt":null,"stderr":0.25666125535421197,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-spanish:overall","benchmarkName":"EuroEval Spanish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":50.3025,"unit":"percent","family":"euroeval","index":40.46915368300641,"aboveFit":false,"run":"no-thinking effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/spanish","measuredAt":null,"stderr":0.3097075233183721,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":48.68375,"unit":"percent","family":"euroeval","index":38.272882396770044,"aboveFit":false,"run":"no-thinking effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.37310353790871503,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"qwen34b","providerId":"alibaba","slug":"alibaba-qwen3-4b","name":"Qwen3 4B","provider":"Alibaba","description":"Qwen3 4B is a model from Alibaba. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":17.16169831958259,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"knowledge":1},"benchmarks":[{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":43.56060606060606,"unit":"percent","family":"gpqa-diamond","index":17.16169831958259,"aboveFit":false,"run":"none effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.6892896738900394,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":15,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"qwen34binstruct2507","providerId":"alibaba","slug":"alibaba-qwen3-4b-instruct","name":"Qwen3 4B Instruct","provider":"Alibaba","description":"Qwen3 4B Instruct is a model from Alibaba. It has 2 scored results in 2 categories. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":19.246181671673007,"multilingual":null,"instruction":null,"math":39.30166882040368},"capabilityCounts":{"math":1,"knowledge":1},"benchmarks":[{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":52.22222222222223,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":39.30166882040368,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":6.591537955214949,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":45.83333333333333,"unit":"percent","family":"gpqa-diamond","index":19.246181671673007,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.850219323539526,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":25,"benchmarkCount":2,"categoryCount":2,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"qwen34bnothinking","providerId":"alibaba","slug":"alibaba-qwen3-4b-no-thinking","name":"Qwen3 4B No Thinking","provider":"Alibaba","description":"Qwen3 4B No Thinking is a model from Alibaba. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":31.46359342824569,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-french:overall","benchmarkName":"EuroEval French","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":49.38875000000001,"unit":"percent","family":"euroeval","index":39.229405180891526,"aboveFit":false,"run":"no-thinking effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/french","measuredAt":null,"stderr":0.311302706380783,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-dutch:overall","benchmarkName":"EuroEval Dutch","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":46.24666666666666,"unit":"percent","family":"euroeval","index":34.966321070335916,"aboveFit":false,"run":"no-thinking effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/dutch","measuredAt":null,"stderr":0.3196545202960206,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-portuguese:overall","benchmarkName":"EuroEval Portuguese","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":45.730000000000004,"unit":"percent","family":"euroeval","index":34.26532328528236,"aboveFit":false,"run":"no-thinking effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/portuguese","measuredAt":null,"stderr":0.33680947878585604,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":44.25125,"unit":"percent","family":"euroeval","index":32.258999785963724,"aboveFit":false,"run":"no-thinking effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":0.2706387121237463,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-italian:overall","benchmarkName":"EuroEval Italian","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":43.07875,"unit":"percent","family":"euroeval","index":30.66818707052765,"aboveFit":false,"run":"no-thinking effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/italian","measuredAt":null,"stderr":0.42036070820189647,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":41.30714285714286,"unit":"percent","family":"euroeval","index":28.264523812860688,"aboveFit":false,"run":"no-thinking effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.24507391637992795,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-spanish:overall","benchmarkName":"EuroEval Spanish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":39.47125,"unit":"percent","family":"euroeval","index":25.773639632629457,"aboveFit":false,"run":"no-thinking effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/spanish","measuredAt":null,"stderr":0.3618766191673621,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":38.655,"unit":"percent","family":"euroeval","index":24.666176196403708,"aboveFit":false,"run":"no-thinking effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.5801494958198274,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"qwen34bthinking2507","providerId":"alibaba","slug":"alibaba-qwen3-4b-thinking","name":"Qwen3 4B Thinking","provider":"Alibaba","description":"Qwen3 4B Thinking is a model from Alibaba. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":45.753678248906844,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-portuguese:overall","benchmarkName":"EuroEval Portuguese","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":55.616249999999994,"unit":"percent","family":"euroeval","index":47.67868977396455,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/portuguese","measuredAt":null,"stderr":0.3299502803453878,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-italian:overall","benchmarkName":"EuroEval Italian","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":55.43625,"unit":"percent","family":"euroeval","index":47.434471190784606,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/italian","measuredAt":null,"stderr":0.4164957982981341,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-french:overall","benchmarkName":"EuroEval French","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":55.25875,"unit":"percent","family":"euroeval","index":47.193644532371046,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/french","measuredAt":null,"stderr":0.2923396996646196,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":54.57375,"unit":"percent","family":"euroeval","index":46.26425714638067,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":0.2316516781722075,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-dutch:overall","benchmarkName":"EuroEval Dutch","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":53.82111111111111,"unit":"percent","family":"euroeval","index":45.24309935143302,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/dutch","measuredAt":null,"stderr":0.3850412836611898,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-spanish:overall","benchmarkName":"EuroEval Spanish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":51.51,"unit":"percent","family":"euroeval","index":42.10745334517191,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/spanish","measuredAt":null,"stderr":0.3776282894593571,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":51.35,"unit":"percent","family":"euroeval","index":41.89037016012307,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.2932575659723036,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":48.78875,"unit":"percent","family":"euroeval","index":38.41534323695834,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.44975687876896336,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"qwen38b","providerId":"alibaba","slug":"alibaba-qwen3-8b","name":"Qwen3 8B","provider":"Alibaba","description":"Qwen3 8B is a model from Alibaba. It has 2 scored results in 2 categories. It has no index.","logoUrl":null,"releaseDate":"2025-04-28T21:43:52.000Z","contextWindowTokens":131072,"modalities":["text"],"openWeights":true,"details":{"apiId":"qwen/qwen3-8b","maxOutputTokens":8192,"knowledgeCutoff":"2025-03-31","retiresAt":"2026-10-09","tools":true,"structuredOutput":false,"cacheReadPerMillion":null,"weightsUrl":"https://huggingface.co/Qwen/Qwen3-8B"},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":20.75164187040498,"multilingual":null,"instruction":null,"math":21.847847448411095},"capabilityCounts":{"knowledge":1,"math":1},"benchmarks":[{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":47.474747474747474,"unit":"percent","family":"gpqa-diamond","index":20.75164187040498,"aboveFit":false,"run":"none effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.623286175358367,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":22.22222222222222,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":21.847847448411095,"aboveFit":false,"run":"none effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":4.52770806168054,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":0.117,"outputPerMillion":0.45499999999999996,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":52,"timeToFirstToken":0.608,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":25,"benchmarkCount":2,"categoryCount":2,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"qwen38bnothinking","providerId":"alibaba","slug":"alibaba-qwen3-8b-no-thinking","name":"Qwen3 8B No Thinking","provider":"Alibaba","description":"Qwen3 8B No Thinking is a model from Alibaba. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":37.37402233367715,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-french:overall","benchmarkName":"EuroEval French","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":52.53999999999999,"unit":"percent","family":"euroeval","index":43.504926348923846,"aboveFit":false,"run":"no-thinking effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/french","measuredAt":null,"stderr":0.3715529067306566,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-dutch:overall","benchmarkName":"EuroEval Dutch","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":49.71666666666667,"unit":"percent","family":"euroeval","index":39.67431264608277,"aboveFit":false,"run":"no-thinking effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/dutch","measuredAt":null,"stderr":0.3014225531634213,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-portuguese:overall","benchmarkName":"EuroEval Portuguese","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":48.708749999999995,"unit":"percent","family":"euroeval","index":38.30680164443391,"aboveFit":false,"run":"no-thinking effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/portuguese","measuredAt":null,"stderr":0.3014859034847235,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":48.363749999999996,"unit":"percent","family":"euroeval","index":37.83871602667234,"aboveFit":false,"run":"no-thinking effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":0.260117161102454,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-italian:overall","benchmarkName":"EuroEval Italian","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":47.67875,"unit":"percent","family":"euroeval","index":36.90932864068197,"aboveFit":false,"run":"no-thinking effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/italian","measuredAt":null,"stderr":0.3231171420707976,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":45.34428571428571,"unit":"percent","family":"euroeval","index":33.74199774989674,"aboveFit":false,"run":"no-thinking effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.22457215557861238,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-spanish:overall","benchmarkName":"EuroEval Spanish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":43.495,"unit":"percent","family":"euroeval","index":31.23294254413129,"aboveFit":false,"run":"no-thinking effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/spanish","measuredAt":null,"stderr":0.32408042983185514,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":40.269999999999996,"unit":"percent","family":"euroeval","index":26.85735959549049,"aboveFit":false,"run":"no-thinking effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.4899266526736426,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"qwen3coder30ba3binstruct","providerId":"alibaba","slug":"alibaba-qwen3-coder-30b-a3b-instruct","name":"Qwen3 Coder 30B A3B Instruct","provider":"Alibaba","description":"Qwen3 Coder 30B A3B Instruct is a model from Alibaba. It has 2 scored results in 1 category. It has no index.","logoUrl":null,"releaseDate":"2025-07-31T14:32:59.000Z","contextWindowTokens":262144,"modalities":["text"],"openWeights":true,"details":{"apiId":"qwen/qwen3-coder-30b-a3b-instruct","maxOutputTokens":235929,"knowledgeCutoff":"2025-06-30","retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":null,"weightsUrl":"https://huggingface.co/Qwen/Qwen3-Coder-30B-A3B-Instruct"},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":41.27807186056801,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"coding":2},"benchmarks":[{"benchmarkId":"swebench-experiments:swe-bench-verified:overall","benchmarkName":"SWE-bench Verified","category":"coding","domain":null,"eligible":true,"ranked":true,"score":52.2,"unit":"percent","family":"swe-bench-verified","index":39.72843308978037,"aboveFit":false,"run":"EntroPO + R2E","source":"SWE-bench team","sourceUrl":"https://github.com/SWE-bench/experiments/tree/main/evaluation/verified","measuredAt":"2026-09-01","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"swebench-experiments:swe-bench-lite:overall","benchmarkName":"SWE-bench Lite","category":"coding","domain":null,"eligible":true,"ranked":true,"score":47.33333333333333,"unit":"percent","family":"swe-bench-lite","index":42.82771063135565,"aboveFit":false,"run":"EntroPO + R2E","source":"SWE-bench team","sourceUrl":"https://github.com/SWE-bench/experiments/tree/main/evaluation/lite","measuredAt":"2025-09-11","stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":0.15,"outputPerMillion":0.6,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":32,"timeToFirstToken":1.018,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":20,"benchmarkCount":2,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[{"host":"siliconflow","name":"SiliconFlow","kind":"host","url":"https://cloud.siliconflow.com/models","inputPerMillion":0.07,"outputPerMillion":0.28,"cacheReadPerMillion":null,"contextTokens":262000,"maxOutputTokens":262000,"deprecated":false},{"host":"amazon-bedrock","name":"Amazon Bedrock","kind":"cloud","url":"https://docs.aws.amazon.com/bedrock/latest/userguide/models-supported.html","inputPerMillion":0.15,"outputPerMillion":0.6,"cacheReadPerMillion":null,"contextTokens":262144,"maxOutputTokens":131072,"deprecated":true},{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":0.15,"outputPerMillion":0.6,"cacheReadPerMillion":null,"contextTokens":262144,"maxOutputTokens":235929,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:swe-bench-team","name":"SWE-bench team","url":"https://github.com","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"qwen3coderflash","providerId":"alibaba","slug":"alibaba-qwen3-coder-flash","name":"Qwen3 Coder Flash","provider":"Alibaba","description":"Qwen3 Coder Flash is a model from Alibaba. It has no scored result.","logoUrl":null,"releaseDate":"2025-09-17T13:25:36.000Z","contextWindowTokens":1000000,"modalities":["text"],"openWeights":null,"details":{"apiId":"qwen/qwen3-coder-flash","maxOutputTokens":65536,"knowledgeCutoff":"2025-06-30","retiresAt":null,"tools":true,"structuredOutput":false,"cacheReadPerMillion":0.039,"weightsUrl":null},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":0.195,"outputPerMillion":0.975,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":60,"timeToFirstToken":0.8525,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":0.195,"outputPerMillion":0.975,"cacheReadPerMillion":0.039,"contextTokens":1000000,"maxOutputTokens":65536,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"qwen3codernext","providerId":"alibaba","slug":"alibaba-qwen3-coder-next","name":"Qwen3 Coder Next","provider":"Alibaba","description":"Qwen3 Coder Next is a model from Alibaba. It has no scored result.","logoUrl":null,"releaseDate":"2026-02-04T00:15:01.000Z","contextWindowTokens":262144,"modalities":["text"],"openWeights":true,"details":{"apiId":"qwen/qwen3-coder-next","maxOutputTokens":235929,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.07,"weightsUrl":"https://huggingface.co/Qwen/Qwen3-Coder-Next"},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":0.12,"outputPerMillion":0.7999999999999999,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":67,"timeToFirstToken":0.791,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":0.12,"outputPerMillion":0.8,"cacheReadPerMillion":0.07,"contextTokens":262144,"maxOutputTokens":235929,"deprecated":false},{"host":"regolo","name":"Regolo","kind":"host","url":"https://docs.regolo.ai/","inputPerMillion":0.3,"outputPerMillion":1.2,"cacheReadPerMillion":null,"contextTokens":262144,"maxOutputTokens":16384,"deprecated":true},{"host":"amazon-bedrock","name":"Amazon Bedrock","kind":"cloud","url":"https://docs.aws.amazon.com/bedrock/latest/userguide/models-supported.html","inputPerMillion":0.5,"outputPerMillion":1.2,"cacheReadPerMillion":null,"contextTokens":262144,"maxOutputTokens":65536,"deprecated":false},{"host":"together-ai","name":"Together AI","kind":"host","url":"https://docs.together.ai/docs/serverless-models","inputPerMillion":0.5,"outputPerMillion":1.2,"cacheReadPerMillion":null,"contextTokens":262144,"maxOutputTokens":262144,"deprecated":true}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"qwen3coderplus","providerId":"alibaba","slug":"alibaba-qwen3-coder-plus","name":"Qwen3 Coder Plus","provider":"Alibaba","description":"Qwen3 Coder Plus is a model from Alibaba. It has no scored result.","logoUrl":null,"releaseDate":"2025-09-23T21:25:07.000Z","contextWindowTokens":1000000,"modalities":["text"],"openWeights":null,"details":{"apiId":"qwen/qwen3-coder-plus","maxOutputTokens":65536,"knowledgeCutoff":"2025-06-30","retiresAt":"2026-10-09","tools":true,"structuredOutput":true,"cacheReadPerMillion":0.13,"weightsUrl":null},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":0.65,"outputPerMillion":3.25,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":38,"timeToFirstToken":1.352,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"qwen3maxthinking","providerId":"alibaba","slug":"alibaba-qwen3-max-thinking","name":"Qwen3 Max Thinking","provider":"Alibaba","description":"Qwen3 Max Thinking is a model from Alibaba. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":"2026-02-09T21:18:21.000Z","contextWindowTokens":262144,"modalities":["text"],"openWeights":null,"details":{"apiId":"qwen/qwen3-max-thinking","maxOutputTokens":65536,"knowledgeCutoff":null,"retiresAt":"2026-10-09","tools":true,"structuredOutput":true,"cacheReadPerMillion":null,"weightsUrl":null},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":55.46783192508039,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"agentic":1},"benchmarks":[{"benchmarkId":"tau2-bench:telecom:overall","benchmarkName":"τ²-bench Telecom","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":98.24561403508771,"unit":"percent","family":"tau2-bench","index":67.98258798325102,"aboveFit":false,"run":"Qwen","source":"Sierra Research","sourceUrl":"https://github.com/sierra-research/tau2-bench","measuredAt":"2026-03-02","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"tau2-bench:retail:overall","benchmarkName":"τ²-bench Retail","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":79.3859649122807,"unit":"percent","family":"tau2-bench","index":55.46783192508039,"aboveFit":false,"run":"Qwen","source":"Sierra Research","sourceUrl":"https://github.com/sierra-research/tau2-bench","measuredAt":"2026-04-30","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"tau2-bench:airline:overall","benchmarkName":"τ²-bench Airline","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":69,"unit":"percent","family":"tau2-bench","index":48.57598486792968,"aboveFit":false,"run":"Qwen","source":"Sierra Research","sourceUrl":"https://github.com/sierra-research/tau2-bench","measuredAt":"2026-03-02","stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":0.78,"outputPerMillion":3.9,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":59,"timeToFirstToken":0.8,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":20,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:sierra-research","name":"Sierra Research","url":"https://github.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"qwen3next80ba3binstruct","providerId":"alibaba","slug":"alibaba-qwen3-next-80b-a3b-instruct","name":"Qwen3 Next 80B A3B Instruct","provider":"Alibaba","description":"Qwen3 Next 80B A3B Instruct is a model from Alibaba. It has no scored result.","logoUrl":null,"releaseDate":"2025-09-11T17:36:53.000Z","contextWindowTokens":262144,"modalities":["text"],"openWeights":true,"details":{"apiId":"qwen/qwen3-next-80b-a3b-instruct","maxOutputTokens":16384,"knowledgeCutoff":"2025-09-30","retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":null,"weightsUrl":"https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Instruct"},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":0.09,"outputPerMillion":1.1,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":103,"timeToFirstToken":0.56,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":{"elo":1417.2539457135258,"eloUpper":1421.9946807869087,"eloLower":1412.5132106401425,"votes":22692,"rank":127,"rankUpper":null,"rankLower":null,"arenaName":"Qwen3-Next-80B-A3B-Instruct","organization":"alibaba","updatedAt":"2026-11-09"},"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[{"host":"deepinfra","name":"Deep Infra","kind":"host","url":"https://deepinfra.com/models","inputPerMillion":0.09,"outputPerMillion":1.1,"cacheReadPerMillion":null,"contextTokens":262144,"maxOutputTokens":32768,"deprecated":false},{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":0.09,"outputPerMillion":1.1,"cacheReadPerMillion":null,"contextTokens":262144,"maxOutputTokens":16384,"deprecated":false},{"host":"amazon-bedrock","name":"Amazon Bedrock","kind":"cloud","url":"https://docs.aws.amazon.com/bedrock/latest/userguide/models-supported.html","inputPerMillion":0.15,"outputPerMillion":1.2,"cacheReadPerMillion":null,"contextTokens":262144,"maxOutputTokens":262000,"deprecated":false},{"host":"io-net","name":"IO.NET","kind":"host","url":"https://io.net/docs/guides/intelligence/io-intelligence","inputPerMillion":0.11875,"outputPerMillion":1.2625,"cacheReadPerMillion":0.059375,"contextTokens":262144,"maxOutputTokens":4096,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"qwen3next80ba3bthinking","providerId":"alibaba","slug":"alibaba-qwen3-next-80b-a3b-thinking","name":"Qwen3 Next 80B A3B Thinking","provider":"Alibaba","description":"Qwen3 Next 80B A3B Thinking is a model from Alibaba. It has no scored result.","logoUrl":null,"releaseDate":"2025-09-11T17:38:04.000Z","contextWindowTokens":262144,"modalities":["text"],"openWeights":true,"details":{"apiId":"qwen/qwen3-next-80b-a3b-thinking","maxOutputTokens":235929,"knowledgeCutoff":"2025-09-30","retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":null,"weightsUrl":"https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Thinking"},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":0.15,"outputPerMillion":1.2,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":111,"timeToFirstToken":0.5105,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":{"elo":1368.327342218191,"eloUpper":1374.173353177021,"eloLower":1362.481331259361,"votes":13450,"rank":192,"rankUpper":null,"rankLower":null,"arenaName":"Qwen3-Next-80B-A3B-Thinking","organization":"alibaba","updatedAt":"2026-11-09"},"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"qwen3vl235ba22binstruct","providerId":"alibaba","slug":"alibaba-qwen3-vl-235b-a22b-instruct","name":"Qwen3 VL 235B A22B Instruct","provider":"Alibaba","description":"Qwen3 VL 235B A22B Instruct is a model from Alibaba. It has no scored result.","logoUrl":null,"releaseDate":"2025-09-23T23:04:47.000Z","contextWindowTokens":262144,"modalities":["text","image"],"openWeights":true,"details":{"apiId":"qwen/qwen3-vl-235b-a22b-instruct","maxOutputTokens":32768,"knowledgeCutoff":"2025-03-31","retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.09999999999999999,"weightsUrl":"https://huggingface.co/Qwen/Qwen3-VL-235B-A22B-Instruct"},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":0.21,"outputPerMillion":1.9,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":33,"timeToFirstToken":0.616,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":{"elo":1420.0128010323347,"eloUpper":1426.5217637926953,"eloLower":1413.5038382719736,"votes":11288,"rank":117,"rankUpper":null,"rankLower":null,"arenaName":"Qwen3-VL-235B-A22B-Instruct","organization":"alibaba","updatedAt":"2026-11-09"},"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[{"host":"deepinfra","name":"Deep Infra","kind":"host","url":"https://deepinfra.com/models","inputPerMillion":0.2,"outputPerMillion":0.88,"cacheReadPerMillion":0.11,"contextTokens":262144,"maxOutputTokens":32768,"deprecated":false},{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":0.21,"outputPerMillion":1.9,"cacheReadPerMillion":0.1,"contextTokens":262144,"maxOutputTokens":32768,"deprecated":false},{"host":"stackit","name":"STACKIT","kind":"cloud","url":"https://docs.stackit.cloud/products/data-and-ai/ai-model-serving/basics/available-shared-models","inputPerMillion":1.76,"outputPerMillion":2.05,"cacheReadPerMillion":null,"contextTokens":218000,"maxOutputTokens":16384,"deprecated":false},{"host":"amazon-bedrock","name":"Amazon Bedrock","kind":"cloud","url":"https://docs.aws.amazon.com/bedrock/latest/userguide/models-supported.html","inputPerMillion":0.53,"outputPerMillion":2.66,"cacheReadPerMillion":null,"contextTokens":262144,"maxOutputTokens":262000,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"qwen3vl235ba22bthinking","providerId":"alibaba","slug":"alibaba-qwen3-vl-235b-a22b-thinking","name":"Qwen3 VL 235B A22B Thinking","provider":"Alibaba","description":"Qwen3 VL 235B A22B Thinking is a model from Alibaba. It has no scored result.","logoUrl":null,"releaseDate":"2025-09-23T23:04:50.000Z","contextWindowTokens":131072,"modalities":["text","image"],"openWeights":true,"details":{"apiId":"qwen/qwen3-vl-235b-a22b-thinking","maxOutputTokens":32768,"knowledgeCutoff":"2025-03-31","retiresAt":"2026-10-09","tools":true,"structuredOutput":true,"cacheReadPerMillion":null,"weightsUrl":"https://huggingface.co/Qwen/Qwen3-VL-235B-A22B-Thinking"},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":0.39999999999999997,"outputPerMillion":4,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":51,"timeToFirstToken":1.3695,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":{"elo":1400.527358336466,"eloUpper":1407.368905270666,"eloLower":1393.6858114022662,"votes":7835,"rank":157,"rankUpper":null,"rankLower":null,"arenaName":"Qwen3-VL-235B-A22B-Thinking","organization":"alibaba","updatedAt":"2026-11-09"},"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"qwen3vl30ba3binstruct","providerId":"alibaba","slug":"alibaba-qwen3-vl-30b-a3b-instruct","name":"Qwen3 VL 30B A3B Instruct","provider":"Alibaba","description":"Qwen3 VL 30B A3B Instruct is a model from Alibaba. It has no scored result.","logoUrl":null,"releaseDate":"2025-10-06T23:47:56.000Z","contextWindowTokens":262144,"modalities":["text","image"],"openWeights":true,"details":{"apiId":"qwen/qwen3-vl-30b-a3b-instruct","maxOutputTokens":16384,"knowledgeCutoff":"2025-03-31","retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":null,"weightsUrl":"https://huggingface.co/Qwen/Qwen3-VL-30B-A3B-Instruct"},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":0.15,"outputPerMillion":0.6,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":25,"timeToFirstToken":0.495,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[{"host":"deepinfra","name":"Deep Infra","kind":"host","url":"https://deepinfra.com/models","inputPerMillion":0.15,"outputPerMillion":0.6,"cacheReadPerMillion":null,"contextTokens":262144,"maxOutputTokens":32768,"deprecated":false},{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":0.15,"outputPerMillion":0.6,"cacheReadPerMillion":null,"contextTokens":262144,"maxOutputTokens":16384,"deprecated":false},{"host":"novita-ai","name":"Novita AI","kind":"host","url":"https://novita.ai/docs/guides/introduction","inputPerMillion":0.2,"outputPerMillion":0.7,"cacheReadPerMillion":null,"contextTokens":131072,"maxOutputTokens":32768,"deprecated":false},{"host":"siliconflow","name":"SiliconFlow","kind":"host","url":"https://cloud.siliconflow.com/models","inputPerMillion":0.29,"outputPerMillion":1,"cacheReadPerMillion":null,"contextTokens":262000,"maxOutputTokens":262000,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"qwen3vl30ba3bthinking","providerId":"alibaba","slug":"alibaba-qwen3-vl-30b-a3b-thinking","name":"Qwen3 VL 30B A3B Thinking","provider":"Alibaba","description":"Qwen3 VL 30B A3B Thinking is a model from Alibaba. It has no scored result.","logoUrl":null,"releaseDate":"2025-10-06T23:47:59.000Z","contextWindowTokens":262144,"modalities":["text","image"],"openWeights":true,"details":{"apiId":"qwen/qwen3-vl-30b-a3b-thinking","maxOutputTokens":235929,"knowledgeCutoff":"2025-03-31","retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":null,"weightsUrl":"https://huggingface.co/Qwen/Qwen3-VL-30B-A3B-Thinking"},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":0.29,"outputPerMillion":1,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":29,"timeToFirstToken":1.585,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"qwen3vl32binstruct","providerId":"alibaba","slug":"alibaba-qwen3-vl-32b-instruct","name":"Qwen3 VL 32B Instruct","provider":"Alibaba","description":"Qwen3 VL 32B Instruct is a model from Alibaba. It has no scored result.","logoUrl":null,"releaseDate":"2025-10-23T14:55:32.000Z","contextWindowTokens":131072,"modalities":["text","image"],"openWeights":true,"details":{"apiId":"qwen/qwen3-vl-32b-instruct","maxOutputTokens":32768,"knowledgeCutoff":null,"retiresAt":"2026-10-09","tools":true,"structuredOutput":true,"cacheReadPerMillion":null,"weightsUrl":"https://huggingface.co/Qwen/Qwen3-VL-32B-Instruct"},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":0.10400000000000001,"outputPerMillion":0.41600000000000004,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":63,"timeToFirstToken":0.806,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"qwen3vl8binstruct","providerId":"alibaba","slug":"alibaba-qwen3-vl-8b-instruct","name":"Qwen3 VL 8B Instruct","provider":"Alibaba","description":"Qwen3 VL 8B Instruct is a model from Alibaba. It has no scored result.","logoUrl":null,"releaseDate":"2025-10-14T17:35:08.000Z","contextWindowTokens":262144,"modalities":["image","text"],"openWeights":true,"details":{"apiId":"qwen/qwen3-vl-8b-instruct","maxOutputTokens":235929,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.12,"weightsUrl":"https://huggingface.co/Qwen/Qwen3-VL-8B-Instruct"},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":0.25,"outputPerMillion":0.75,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":57,"timeToFirstToken":0.7245,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"qwen3vl8bthinking","providerId":"alibaba","slug":"alibaba-qwen3-vl-8b-thinking","name":"Qwen3 VL 8B Thinking","provider":"Alibaba","description":"Qwen3 VL 8B Thinking is a model from Alibaba. It has no scored result.","logoUrl":null,"releaseDate":"2025-10-14T17:42:26.000Z","contextWindowTokens":131072,"modalities":["image","text"],"openWeights":true,"details":{"apiId":"qwen/qwen3-vl-8b-thinking","maxOutputTokens":32768,"knowledgeCutoff":null,"retiresAt":"2026-10-09","tools":true,"structuredOutput":true,"cacheReadPerMillion":null,"weightsUrl":"https://huggingface.co/Qwen/Qwen3-VL-8B-Thinking"},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":0.18,"outputPerMillion":2.0999999999999996,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":128.5,"timeToFirstToken":0.4755,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"qwen3vlplus","providerId":"alibaba","slug":"alibaba-qwen3-vl-plus","name":"Qwen3 VL Plus","provider":"Alibaba","description":"Qwen3 VL Plus is a model from Alibaba. It has no scored result.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[{"benchmarkId":"vals-ai:medscribe:overall","benchmarkName":"MedScribe","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":77.129,"unit":"percent","family":"vals-ai:medscribe:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medscribe","measuredAt":"2026-10-29","stderr":1.916,"costPerTest":0.02022,"latency":71.346},{"benchmarkId":"vals-ai:medcode:overall","benchmarkName":"MedCode","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":31.651,"unit":"percent","family":"vals-ai:medcode:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medcode","measuredAt":"2026-10-29","stderr":1.845,"costPerTest":0.002519,"latency":9.952}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"qwen3235ba22bjul2025","providerId":"alibaba","slug":"alibaba-qwen3-235b-a22b-jul-2025","name":"Qwen3-235B-A22B (Jul 2025)","provider":"Alibaba","description":"Qwen3-235B-A22B (Jul 2025) is a model from Alibaba. It has 6 scored results in 3 categories. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":45.65073227241674,"multimodal":null,"knowledge":54.119372895991845,"multilingual":null,"instruction":null,"math":46.6480753198234},"capabilityCounts":{"math":3,"knowledge":2,"reasoning":1},"benchmarks":[{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":86.66666666666667,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":59.341241506765535,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":5.124707431905383,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":80.05050505050505,"unit":"percent","family":"gpqa-diamond","index":50.62923658370101,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.600093351563155,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-simpleqa-verified:overall","benchmarkName":"SimpleQA Verified","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":40.44044044044044,"unit":"percent","family":"epoch-ai:epoch-simpleqa-verified:overall","index":57.609509208282674,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.5535255725079344,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-2025-02-28-public:overall","benchmarkName":"FrontierMath-2025-02-28-Public","category":"math","domain":null,"eligible":false,"ranked":true,"score":20,"unit":"percent","family":"epoch-ai:epoch-frontiermath-2025-02-28-public:overall","index":null,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":13.333333333333334,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-mystery-game-puzzles:overall","benchmarkName":"Mystery Game Puzzles","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":9,"unit":"percent","family":"epoch-ai:epoch-mystery-game-puzzles:overall","index":45.65073227241674,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.8762349126466145,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-2025-02-28-private:overall","benchmarkName":"FrontierMath-2025-02-28-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":8.480565371024735,"unit":"percent","family":"epoch-ai:epoch-frontiermath-2025-02-28-private:overall","index":37.40062672553905,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.6589928458951777,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-private:overall","benchmarkName":"FrontierMath-Tier-4-2025-07-01-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":0,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-private:overall","index":43.20235772716562,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-public:overall","benchmarkName":"FrontierMath-Tier-4-2025-07-01-Public","category":"math","domain":null,"eligible":false,"ranked":true,"score":0,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-public:overall","index":null,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":40,"benchmarkCount":6,"categoryCount":3,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"qwen330ba3bjul2025","providerId":"alibaba","slug":"alibaba-qwen3-30b-a3b-jul-2025","name":"Qwen3-30B-A3B (Jul 2025)","provider":"Alibaba","description":"Qwen3-30B-A3B (Jul 2025) is a model from Alibaba. It has 2 scored results in 2 categories. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":34.850855654683244,"multilingual":null,"instruction":null,"math":47.46294640638169},"capabilityCounts":{"math":1,"knowledge":1},"benchmarks":[{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":66.25,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":47.46294640638169,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":6.041848075762483,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":62.84722222222223,"unit":"percent","family":"gpqa-diamond","index":34.850855654683244,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.832789761354549,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":25,"benchmarkCount":2,"categoryCount":2,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"qwen3-asr-0-6b","providerId":"alibaba","slug":"qwen3-asr-0-6b","name":"Qwen3-ASR 0.6B","provider":"Alibaba","description":"Qwen3-ASR 0.6B is a standard model from Alibaba. It has no scored result.","logoUrl":null,"releaseDate":"2026-06-26","contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"qwen3-asr-1-7b","providerId":"alibaba","slug":"qwen3-asr-1-7b","name":"Qwen3-ASR 1.7B","provider":"Alibaba","description":"Qwen3-ASR 1.7B is a standard model from Alibaba. It has no scored result.","logoUrl":null,"releaseDate":"2026-06-26","contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"qwen3-coder-480b-a35b","providerId":"alibaba","slug":"qwen3-coder-480b-a35b","name":"Qwen3-Coder 480B-A35B","provider":"Alibaba","description":"Qwen3-Coder 480B-A35B is a standard model from Alibaba. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":"2025-07-22","contextWindowTokens":256000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":52.5619349432166,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"coding":1},"benchmarks":[{"benchmarkId":"sweVerified","benchmarkName":"Software Engineering Benchmark Verified","category":"coding","score":69.6,"unit":"percent","family":"swe-bench-verified","index":52.5619349432166,"aboveFit":false,"run":null,"source":"Carlos E. Jimenez, John Yang, Alexander Wettig, Shunyu Yao, Kexin Pei, Ofir Press, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2310.06770","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"bfclV3","benchmarkName":"Berkeley Function Calling Leaderboard v3","category":"agentic","score":68.7,"unit":"percent","family":"bfclV3","index":null,"aboveFit":false,"run":null,"source":"Shishir G. Patil et al.","sourceUrl":"https://gorilla.cs.berkeley.edu/leaderboard.html","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":20,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"qwen3coder480ba35binstruct","providerId":"alibaba","slug":"alibaba-qwen3-coder-480b-a35b-instruct","name":"Qwen3-Coder 480B/A35B Instruct","provider":"Alibaba","description":"Qwen3-Coder 480B/A35B Instruct is a model from Alibaba. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":44.70694673982029,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"coding":1},"benchmarks":[{"benchmarkId":"swebench-experiments:swe-bench-verified:overall","benchmarkName":"SWE-bench Verified","category":"coding","domain":null,"eligible":true,"ranked":true,"score":62.5,"unit":"percent","family":"swe-bench-verified","index":47.3252761409524,"aboveFit":false,"run":"OpenHands","source":"SWE-bench team","sourceUrl":"https://github.com/SWE-bench/experiments/tree/main/evaluation/verified","measuredAt":"2026-09-01","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"swe-bench:swe-bench-verified:overall","benchmarkName":"SWE-bench Verified","category":"coding","domain":null,"eligible":true,"ranked":true,"score":55.4,"unit":"percent","family":"swe-bench-verified","index":42.08861733868818,"aboveFit":false,"run":"mini-SWE-agent","source":"SWE-bench team","sourceUrl":"https://www.swebench.com/","measuredAt":"2026-02-26","stderr":null,"costPerTest":0.247927108,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":{"elo":1356.5798096204558,"eloUpper":1361.5763290996545,"eloLower":1351.583290141257,"votes":25155,"rank":205,"rankUpper":null,"rankLower":null,"arenaName":"Qwen3-Coder-480B-A35B-Instruct","organization":"alibaba","updatedAt":"2026-11-09"},"quality":{"coveragePercent":20,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:swe-bench-team","name":"SWE-bench team","url":"https://github.com","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"qwen3-omni-30b-a3b-instruct","providerId":"alibaba","slug":"qwen3-omni-30b-a3b-instruct","name":"Qwen3-Omni-30B-A3B-Instruct","provider":"Alibaba","description":"Qwen3-Omni-30B-A3B-Instruct is a standard model from Alibaba. It has 7 scored results in 5 categories. It has no index.","logoUrl":null,"releaseDate":"2025-09-22","contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":13.672039180853211,"coding":null,"reasoning":22.073582735971804,"multimodal":27.381195741949348,"knowledge":31.537798761663367,"multilingual":null,"instruction":18.18036057474539,"math":null},"capabilityCounts":{"knowledge":2,"multimodal":1,"instruction":1,"agentic":1,"reasoning":2},"benchmarks":[{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":62,"unit":"percent","family":"gpqa-diamond","index":34.0738065828762,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaMmmuPro","benchmarkName":"Artificial Analysis MMMU-Pro","category":"multimodal","score":55.5,"unit":"percent","family":"mmmu-pro","index":27.381195741949348,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmmu-pro","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaIfBench","benchmarkName":"Artificial Analysis IFBench","category":"instruction","score":31.2,"unit":"percent","family":"ifbench-family","index":18.18036057474539,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"tau2Bench","benchmarkName":"τ²-Bench Tool-Agent-User Evaluation","category":"agentic","score":16.4,"unit":"percent","family":"tau2-bench","index":13.672039180853211,"aboveFit":false,"run":null,"source":"Victor Barres, Honghua Dong, Soham Ray, Xujie Si, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2506.07982","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":4.6,"unit":"percent","family":"hle","index":29.001790940450533,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":0,"unit":"percent","family":"lcr","index":7.298049459170706,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":0,"unit":"percent","family":"critpt","index":36.8491160127729,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":97,"timeToFirstToken":1.91,"source":"BenchLM","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":65,"benchmarkCount":7,"categoryCount":5,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"qwen3-omni-30b-a3b-thinking","providerId":"alibaba","slug":"qwen3-omni-30b-a3b-thinking","name":"Qwen3-Omni-30B-A3B-Thinking","provider":"Alibaba","description":"Qwen3-Omni-30B-A3B-Thinking is a standard model from Alibaba. It has 7 scored results in 5 categories. It has no index.","logoUrl":null,"releaseDate":"2025-09-22","contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":16.923547429269078,"coding":null,"reasoning":22.073582735971804,"multimodal":33.497833388009425,"knowledge":37.853588419031425,"multilingual":null,"instruction":29.581390075665325,"math":null},"capabilityCounts":{"knowledge":2,"multimodal":1,"instruction":1,"agentic":1,"reasoning":2},"benchmarks":[{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":72.6,"unit":"percent","family":"gpqa-diamond","index":43.79583693702592,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaMmmuPro","benchmarkName":"Artificial Analysis MMMU-Pro","category":"multimodal","score":60.2,"unit":"percent","family":"mmmu-pro","index":33.497833388009425,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/mmmu-pro","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaIfBench","benchmarkName":"Artificial Analysis IFBench","category":"instruction","score":43.4,"unit":"percent","family":"ifbench-family","index":29.581390075665325,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"tau2Bench","benchmarkName":"τ²-Bench Tool-Agent-User Evaluation","category":"agentic","score":21.3,"unit":"percent","family":"tau2-bench","index":16.923547429269078,"aboveFit":false,"run":null,"source":"Victor Barres, Honghua Dong, Soham Ray, Xujie Si, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2506.07982","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":7.5,"unit":"percent","family":"hle","index":31.911339901036932,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":0,"unit":"percent","family":"lcr","index":7.298049459170706,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":0,"unit":"percent","family":"critpt","index":36.8491160127729,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":65,"benchmarkCount":7,"categoryCount":5,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"qwen3508b","providerId":"alibaba","slug":"alibaba-qwen3-5-0-8b","name":"Qwen3.5 0.8B","provider":"Alibaba","description":"Qwen3.5 0.8B is a model from Alibaba. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":12.969123639513938,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-portuguese:overall","benchmarkName":"EuroEval Portuguese","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":34.58125,"unit":"percent","family":"euroeval","index":19.139034789574104,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/portuguese","measuredAt":null,"stderr":0.6045982653795825,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-dutch:overall","benchmarkName":"EuroEval Dutch","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":31.90777777777778,"unit":"percent","family":"euroeval","index":15.511748132451508,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/dutch","measuredAt":null,"stderr":0.5776516956449461,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-french:overall","benchmarkName":"EuroEval French","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":31.70375,"unit":"percent","family":"euroeval","index":15.234929383461271,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/french","measuredAt":null,"stderr":0.5777096480932269,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-italian:overall","benchmarkName":"EuroEval Italian","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":31.401249999999997,"unit":"percent","family":"euroeval","index":14.824506486728296,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/italian","measuredAt":null,"stderr":0.6274140877442903,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":28.666250000000005,"unit":"percent","family":"euroeval","index":11.113740792299595,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":0.7262424698404797,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":28.16,"unit":"percent","family":"euroeval","index":10.426876027105969,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.6271428571428572,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-spanish:overall","benchmarkName":"EuroEval Spanish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":26.497500000000002,"unit":"percent","family":"euroeval","index":8.171246057457815,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/spanish","measuredAt":null,"stderr":0.6543854464304658,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":24.991250000000004,"unit":"percent","family":"euroeval","index":6.127611385708924,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.6437257276977517,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"qwen3508bbase","providerId":"alibaba","slug":"alibaba-qwen3-5-0-8b-base","name":"Qwen3.5 0.8B Base","provider":"Alibaba","description":"Qwen3.5 0.8B Base is a model from Alibaba. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":5,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-dutch:overall","benchmarkName":"EuroEval Dutch","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":26.98111111111111,"unit":"percent","family":"euroeval","index":8.827395059489128,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/dutch","measuredAt":null,"stderr":0.6944542221533877,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-italian:overall","benchmarkName":"EuroEval Italian","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":24.3875,"unit":"percent","family":"euroeval","index":5.308461554626156,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/italian","measuredAt":null,"stderr":0.8018572192105026,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-french:overall","benchmarkName":"EuroEval French","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":23.84125,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/french","measuredAt":null,"stderr":0.8184073710567372,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-portuguese:overall","benchmarkName":"EuroEval Portuguese","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":23.185,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/portuguese","measuredAt":null,"stderr":0.813232362243412,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-spanish:overall","benchmarkName":"EuroEval Spanish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":21.753749999999997,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/spanish","measuredAt":null,"stderr":0.6482235339757421,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":21.680000000000003,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.8803709793175204,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":21.0075,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.6260690856447074,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":20.044999999999998,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":0.6760535759094837,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"qwen352b","providerId":"alibaba","slug":"alibaba-qwen3-5-2b","name":"Qwen3.5 2B","provider":"Alibaba","description":"Qwen3.5 2B is a model from Alibaba. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":23.815651061231843,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-french:overall","benchmarkName":"EuroEval French","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":46.098749999999995,"unit":"percent","family":"euroeval","index":34.765632188324616,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/french","measuredAt":null,"stderr":0.5753232786877305,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-portuguese:overall","benchmarkName":"EuroEval Portuguese","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":40.96375,"unit":"percent","family":"euroeval","index":27.79861871816322,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/portuguese","measuredAt":null,"stderr":0.638053681127223,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-dutch:overall","benchmarkName":"EuroEval Dutch","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":40.20666666666667,"unit":"percent","family":"euroeval","index":26.771430834741995,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/dutch","measuredAt":null,"stderr":0.6219969830421851,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-italian:overall","benchmarkName":"EuroEval Italian","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":38.067499999999995,"unit":"percent","family":"euroeval","index":23.86907387630247,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/italian","measuredAt":null,"stderr":0.8463976975984753,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":37.98874999999999,"unit":"percent","family":"euroeval","index":23.76222824616123,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":0.9426384182177173,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":34.89999999999999,"unit":"percent","family":"euroeval","index":19.57150519728859,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.6496453359333845,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":34.61142857142857,"unit":"percent","family":"euroeval","index":19.179980167111218,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.6983259866806256,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-spanish:overall","benchmarkName":"EuroEval Spanish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":31.518750000000004,"unit":"percent","family":"euroeval","index":14.983926950748547,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/spanish","measuredAt":null,"stderr":0.540552379515621,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"qwen352bbase","providerId":"alibaba","slug":"alibaba-qwen3-5-2b-base","name":"Qwen3.5 2B Base","provider":"Alibaba","description":"Qwen3.5 2B Base is a model from Alibaba. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":12.96997162070555,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-dutch:overall","benchmarkName":"EuroEval Dutch","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":35.68444444444444,"unit":"percent","family":"euroeval","index":20.635815812875297,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/dutch","measuredAt":null,"stderr":0.7095373209892085,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-portuguese:overall","benchmarkName":"EuroEval Portuguese","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":31.262500000000003,"unit":"percent","family":"euroeval","index":14.63625466219375,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/portuguese","measuredAt":null,"stderr":0.8519573052682864,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":30.7375,"unit":"percent","family":"euroeval","index":13.923950461252225,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":0.8219118337753752,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-french:overall","benchmarkName":"EuroEval French","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":29.331250000000004,"unit":"percent","family":"euroeval","index":12.015992780158854,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/french","measuredAt":null,"stderr":0.7898753857413205,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-spanish:overall","benchmarkName":"EuroEval Spanish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":27.055,"unit":"percent","family":"euroeval","index":8.927645280362384,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/spanish","measuredAt":null,"stderr":0.6279356057749871,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":23.66375,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.6616645675869307,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"qwen352bdatask","providerId":"alibaba","slug":"alibaba-qwen3-5-2b-da-task","name":"Qwen3.5 2B Da Task","provider":"Alibaba","description":"Qwen3.5 2B Da Task is a model from Alibaba. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":26.458808435439877,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":39.97625,"unit":"percent","family":"euroeval","index":26.458808435439877,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":0.9928691064284355,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"qwen3535ba3bbase","providerId":"alibaba","slug":"alibaba-qwen3-5-35b-a3b-base","name":"Qwen3.5 35B A3B Base","provider":"Alibaba","description":"Qwen3.5 35B A3B Base is a model from Alibaba. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":49.618143899317246,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":57.04571428571428,"unit":"percent","family":"euroeval","index":49.618143899317246,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.5468649051546423,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"qwen3535ba3bfp8","providerId":"alibaba","slug":"alibaba-qwen3-5-35b-a3b-fp8","name":"Qwen3.5 35B A3B FP8","provider":"Alibaba","description":"Qwen3.5 35B A3B FP8 is a model from Alibaba. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":58.17839288831928,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-italian:overall","benchmarkName":"EuroEval Italian","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":65.74375,"unit":"percent","family":"euroeval","index":61.41937700260323,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/italian","measuredAt":null,"stderr":0.48905041917986325,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-portuguese:overall","benchmarkName":"EuroEval Portuguese","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":65.5075,"unit":"percent","family":"euroeval","index":61.09884011217953,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/portuguese","measuredAt":null,"stderr":0.5645490456993085,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-french:overall","benchmarkName":"EuroEval French","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":64.21124999999999,"unit":"percent","family":"euroeval","index":59.34012712080724,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/french","measuredAt":null,"stderr":0.502115835739125,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":63.36,"unit":"percent","family":"euroeval","index":58.18517673785206,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":0.6869224847098834,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-dutch:overall","benchmarkName":"EuroEval Dutch","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":63.349999999999994,"unit":"percent","family":"euroeval","index":58.1716090387865,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/dutch","measuredAt":null,"stderr":0.6054332190391022,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":62.30857142857143,"unit":"percent","family":"euroeval","index":56.758630093245365,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.5798293204179369,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-spanish:overall","benchmarkName":"EuroEval Spanish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":57.61999999999999,"unit":"percent","family":"euroeval","index":50.3973174742247,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/spanish","measuredAt":null,"stderr":0.6616964466430207,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":55.2625,"unit":"percent","family":"euroeval","index":47.19873241952063,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.5749388554446463,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"qwen35397ba17b","providerId":"alibaba","slug":"alibaba-qwen3-5-397b-a17b","name":"Qwen3.5 397B A17B","provider":"Alibaba","description":"Qwen3.5 397B A17B is a model from Alibaba. It has 5 scored results in 4 categories. It has no index.","logoUrl":null,"releaseDate":"2026-02-16T06:23:38.000Z","contextWindowTokens":262144,"modalities":["text","image","video"],"openWeights":true,"details":{"apiId":"qwen/qwen3.5-397b-a17b","maxOutputTokens":64000,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.12,"weightsUrl":"https://huggingface.co/Qwen/Qwen3.5-397B-A17B"},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":57.842783375914905,"coding":null,"reasoning":52.62046634276329,"multimodal":null,"knowledge":56.419468117285504,"multilingual":null,"instruction":null,"math":52.043567337558805},"capabilityCounts":{"agentic":1,"knowledge":1,"math":2,"reasoning":1},"benchmarks":[{"benchmarkId":"tau2-bench:telecom:overall","benchmarkName":"τ²-bench Telecom","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":97.81,"unit":"percent","family":"tau2-bench","index":67.69352622239114,"aboveFit":false,"run":"enabled effort · Sierra","source":"Sierra Research","sourceUrl":"https://github.com/sierra-research/tau2-bench","measuredAt":"2026-03-02","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":86.36363636363636,"unit":"percent","family":"gpqa-diamond","index":56.419468117285504,"aboveFit":false,"run":"none effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.445015597318985,"costPerTest":null,"latency":null},{"benchmarkId":"vals-ai:sage:overall","benchmarkName":"SAGE","category":null,"domain":"Education","eligible":false,"ranked":true,"score":85.53,"unit":"percent","family":"vals-ai:sage:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/sage","measuredAt":"2026-11-05","stderr":1.812,"costPerTest":0.013475,"latency":168.001},{"benchmarkId":"tau2-bench:retail:overall","benchmarkName":"τ²-bench Retail","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":84.43,"unit":"percent","family":"tau2-bench","index":58.81491798488006,"aboveFit":false,"run":"enabled effort · Sierra","source":"Sierra Research","sourceUrl":"https://github.com/sierra-research/tau2-bench","measuredAt":"2026-04-30","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":82.22222222222221,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":56.75549019239625,"aboveFit":false,"run":"none effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":5.763774795025092,"costPerTest":null,"latency":null},{"benchmarkId":"tau2-bench:airline:overall","benchmarkName":"τ²-bench Airline","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":81.5,"unit":"percent","family":"tau2-bench","index":56.87064876694975,"aboveFit":false,"run":"enabled effort · Sierra","source":"Sierra Research","sourceUrl":"https://github.com/sierra-research/tau2-bench","measuredAt":"2026-03-02","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tiers-1-3-v2-private:overall","benchmarkName":"FrontierMath-Tiers-1-3-v2-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":31.2280701754386,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tiers-1-3-v2-private:overall","index":47.33164448272136,"aboveFit":false,"run":"none effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.7499133473298816,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-mystery-game-puzzles:overall","benchmarkName":"Mystery Game Puzzles","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":18,"unit":"percent","family":"epoch-ai:epoch-mystery-game-puzzles:overall","index":52.62046634276329,"aboveFit":false,"run":"none effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":3.8612291966536914,"costPerTest":null,"latency":null},{"benchmarkId":"tau2-bench:banking-knowledge:overall","benchmarkName":"τ²-bench Banking","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":9.79,"unit":"percent","family":"tau2-bench","index":9.285820911051395,"aboveFit":false,"run":"enabled effort · Sierra","source":"Sierra Research","sourceUrl":"https://github.com/sierra-research/tau2-bench","measuredAt":"2026-10-20","stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":0.6,"outputPerMillion":3.5999999999999996,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":37,"timeToFirstToken":1.152,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":{"elo":1437.5752866772705,"eloUpper":1440.6069054285917,"eloLower":1434.5436679259492,"votes":97678,"rank":83,"rankUpper":null,"rankLower":null,"arenaName":"Qwen3.5-397B-A17B","organization":"alibaba","updatedAt":"2026-11-09"},"quality":{"coveragePercent":60,"benchmarkCount":5,"categoryCount":4,"confidence":"unranked","state":"partial"},"hosts":[{"host":"siliconflow","name":"SiliconFlow","kind":"host","url":"https://cloud.siliconflow.com/models","inputPerMillion":0.39,"outputPerMillion":2.34,"cacheReadPerMillion":null,"contextTokens":262144,"maxOutputTokens":262144,"deprecated":true},{"host":"chutes","name":"Chutes","kind":"host","url":"https://llm.chutes.ai/v1/models","inputPerMillion":0.45,"outputPerMillion":3,"cacheReadPerMillion":0.045,"contextTokens":262144,"maxOutputTokens":65536,"deprecated":false},{"host":"deepinfra","name":"Deep Infra","kind":"host","url":"https://deepinfra.com/models","inputPerMillion":0.45,"outputPerMillion":3,"cacheReadPerMillion":0.22,"contextTokens":262144,"maxOutputTokens":81920,"deprecated":false},{"host":"digitalocean","name":"DigitalOcean","kind":"cloud","url":"https://docs.digitalocean.com/products/gradient-ai-platform/details/models/","inputPerMillion":0.55,"outputPerMillion":3.5,"cacheReadPerMillion":0.111,"contextTokens":131072,"maxOutputTokens":131072,"deprecated":false},{"host":"novita-ai","name":"Novita AI","kind":"host","url":"https://novita.ai/docs/guides/introduction","inputPerMillion":0.6,"outputPerMillion":3.6,"cacheReadPerMillion":null,"contextTokens":262144,"maxOutputTokens":65536,"deprecated":false},{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":0.6,"outputPerMillion":3.6,"cacheReadPerMillion":0.12,"contextTokens":262144,"maxOutputTokens":64000,"deprecated":false},{"host":"together-ai","name":"Together AI","kind":"host","url":"https://docs.together.ai/docs/serverless-models","inputPerMillion":0.6,"outputPerMillion":3.6,"cacheReadPerMillion":0.35,"contextTokens":262144,"maxOutputTokens":130000,"deprecated":true},{"host":"scaleway","name":"Scaleway","kind":"cloud","url":"https://www.scaleway.com/en/docs/generative-apis/","inputPerMillion":0.6729,"outputPerMillion":4.0374,"cacheReadPerMillion":null,"contextTokens":256000,"maxOutputTokens":16384,"deprecated":false},{"host":"ovhcloud","name":"OVHcloud","kind":"cloud","url":"https://www.ovhcloud.com/en/public-cloud/ai-endpoints/catalog//","inputPerMillion":0.71,"outputPerMillion":4.25,"cacheReadPerMillion":null,"contextTokens":262144,"maxOutputTokens":262144,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:sierra-research","name":"Sierra Research","url":"https://github.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"qwen354bbase","providerId":"alibaba","slug":"alibaba-qwen3-5-4b-base","name":"Qwen3.5 4B Base","provider":"Alibaba","description":"Qwen3.5 4B Base is a model from Alibaba. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":33.2116460846379,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-dutch:overall","benchmarkName":"EuroEval Dutch","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":48.32555555555555,"unit":"percent","family":"euroeval","index":37.786894953852524,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/dutch","measuredAt":null,"stderr":0.9026203281502823,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-portuguese:overall","benchmarkName":"EuroEval Portuguese","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":47.93,"unit":"percent","family":"euroeval","index":37.25021707970399,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/portuguese","measuredAt":null,"stderr":0.9050733740973712,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":47.005,"unit":"percent","family":"euroeval","index":35.99520491614035,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":0.818346274507314,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-italian:overall","benchmarkName":"EuroEval Italian","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":46.412499999999994,"unit":"percent","family":"euroeval","index":35.19131874650633,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/italian","measuredAt":null,"stderr":0.6743492233257187,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":43.49428571428571,"unit":"percent","family":"euroeval","index":31.23197342276946,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.7419004829022638,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-french:overall","benchmarkName":"EuroEval French","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":37.00625,"unit":"percent","family":"euroeval","index":22.42920181297068,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/french","measuredAt":null,"stderr":0.8206342821257225,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-spanish:overall","benchmarkName":"EuroEval Spanish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":35.41875,"unit":"percent","family":"euroeval","index":20.275329586314164,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/spanish","measuredAt":null,"stderr":0.5904129910494856,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":32.9675,"unit":"percent","family":"euroeval","index":16.949547352870518,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.813956387038028,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"qwen359b","providerId":"alibaba","slug":"alibaba-qwen3-5-9b","name":"Qwen3.5 9B","provider":"Alibaba","description":"Qwen3.5 9B is a model from Alibaba. It has 3 scored results in 3 categories. It has no index.","logoUrl":null,"releaseDate":"2026-03-10T14:19:56.000Z","contextWindowTokens":262144,"modalities":["text","image","video"],"openWeights":true,"details":{"apiId":"qwen/qwen3.5-9b","maxOutputTokens":40960,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.114,"weightsUrl":"https://huggingface.co/Qwen/Qwen3.5-9B"},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":49.58699490765579,"multilingual":45.040135726225216,"instruction":null,"math":44.79639036343838},"capabilityCounts":{"knowledge":1,"math":1,"multilingual":1},"benchmarks":[{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":78.91414141414141,"unit":"percent","family":"gpqa-diamond","index":49.58699490765579,"aboveFit":false,"run":"none effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.388848562991196,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":61.66666666666667,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":44.79639036343838,"aboveFit":false,"run":"none effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":6.003996985174109,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-italian:overall","benchmarkName":"EuroEval Italian","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":56.948750000000004,"unit":"percent","family":"euroeval","index":49.486585674449486,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/italian","measuredAt":null,"stderr":0.5761089849151808,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-french:overall","benchmarkName":"EuroEval French","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":55.46000000000001,"unit":"percent","family":"euroeval","index":47.46669447606531,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/french","measuredAt":null,"stderr":0.6052362658499572,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-portuguese:overall","benchmarkName":"EuroEval Portuguese","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":54.99125,"unit":"percent","family":"euroeval","index":46.83070858236751,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/portuguese","measuredAt":null,"stderr":0.5728437832428663,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":54.618750000000006,"unit":"percent","family":"euroeval","index":46.32531179217567,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":0.9289400882188259,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":52.72428571428572,"unit":"percent","family":"euroeval","index":43.75495966027477,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.5372435315989369,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-dutch:overall","benchmarkName":"EuroEval Dutch","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":52.20222222222222,"unit":"percent","family":"euroeval","index":43.04663962493184,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/dutch","measuredAt":null,"stderr":0.5163941933861795,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":46.677499999999995,"unit":"percent","family":"euroeval","index":35.55086277174348,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.7016041886277475,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-spanish:overall","benchmarkName":"EuroEval Spanish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":46.14375,"unit":"percent","family":"euroeval","index":34.826686834119606,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/spanish","measuredAt":null,"stderr":0.7426136276691938,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":0.126,"outputPerMillion":0.186,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":126,"timeToFirstToken":0.205,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":30,"benchmarkCount":3,"categoryCount":3,"confidence":"unranked","state":"partial"},"hosts":[{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":0.056,"outputPerMillion":0.083,"cacheReadPerMillion":0.051,"contextTokens":262144,"maxOutputTokens":40960,"deprecated":true},{"host":"deepinfra","name":"Deep Infra","kind":"host","url":"https://deepinfra.com/models","inputPerMillion":0.1,"outputPerMillion":0.15,"cacheReadPerMillion":null,"contextTokens":262144,"maxOutputTokens":65536,"deprecated":false},{"host":"siliconflow","name":"SiliconFlow","kind":"host","url":"https://cloud.siliconflow.com/models","inputPerMillion":0.1,"outputPerMillion":0.15,"cacheReadPerMillion":null,"contextTokens":262144,"maxOutputTokens":262144,"deprecated":false},{"host":"ovhcloud","name":"OVHcloud","kind":"cloud","url":"https://www.ovhcloud.com/en/public-cloud/ai-endpoints/catalog//","inputPerMillion":0.12,"outputPerMillion":0.18,"cacheReadPerMillion":null,"contextTokens":262144,"maxOutputTokens":262144,"deprecated":false},{"host":"together-ai","name":"Together AI","kind":"host","url":"https://docs.together.ai/docs/serverless-models","inputPerMillion":0.17,"outputPerMillion":0.25,"cacheReadPerMillion":null,"contextTokens":262144,"maxOutputTokens":65536,"deprecated":false},{"host":"regolo","name":"Regolo","kind":"host","url":"https://docs.regolo.ai/","inputPerMillion":0.078505,"outputPerMillion":0.392525,"cacheReadPerMillion":null,"contextTokens":262144,"maxOutputTokens":8192,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"qwen359bbase","providerId":"alibaba","slug":"alibaba-qwen3-5-9b-base","name":"Qwen3.5 9B Base","provider":"Alibaba","description":"Qwen3.5 9B Base is a model from Alibaba. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":38.71637655997528,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-portuguese:overall","benchmarkName":"EuroEval Portuguese","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":57.09,"unit":"percent","family":"euroeval","index":49.678229423750416,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/portuguese","measuredAt":null,"stderr":0.7899386843293599,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":56.46125,"unit":"percent","family":"euroeval","index":48.82516034500377,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":0.7602168687289174,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-dutch:overall","benchmarkName":"EuroEval Dutch","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":55.599999999999994,"unit":"percent","family":"euroeval","index":47.65664226298303,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/dutch","measuredAt":null,"stderr":0.7218819711330334,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-italian:overall","benchmarkName":"EuroEval Italian","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":51.27374999999999,"unit":"percent","family":"euroeval","index":41.786916454748216,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/italian","measuredAt":null,"stderr":0.7186456445982262,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-french:overall","benchmarkName":"EuroEval French","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":46.747499999999995,"unit":"percent","family":"euroeval","index":35.64583666520235,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/french","measuredAt":null,"stderr":0.9864836541980815,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":43.99571428571429,"unit":"percent","family":"euroeval","index":31.912296618770768,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.9624172373640812,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-spanish:overall","benchmarkName":"EuroEval Spanish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":40.057500000000005,"unit":"percent","family":"euroeval","index":26.5690459903475,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/spanish","measuredAt":null,"stderr":0.5271992626132931,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":39.411249999999995,"unit":"percent","family":"euroeval","index":25.692233438236133,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.7709004475287325,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"qwen3-5-flash","providerId":"alibaba","slug":"qwen3-5-flash","name":"Qwen3.5 Flash","provider":"Alibaba","description":"Qwen3.5 Flash is a reasoning model from Alibaba. It has 7 scored results in 5 categories. It has no index.","logoUrl":null,"releaseDate":"2026-03-04","contextWindowTokens":1000000,"modalities":["text"],"openWeights":false,"details":null,"reasoning":true,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":41.43485264321955,"coding":53.585625796794154,"reasoning":null,"multimodal":61.74889083921626,"knowledge":51.78210281999916,"multilingual":null,"instruction":null,"math":55.92929883040928},"capabilityCounts":{"math":1,"knowledge":2,"coding":2,"multimodal":1,"agentic":1},"benchmarks":[{"benchmarkId":"vals-ai:aime:overall","benchmarkName":"AIME","category":"math","domain":null,"eligible":true,"ranked":true,"score":92.5,"unit":"percent","family":"vals-ai:aime:overall","index":55.92929883040928,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/aime","measuredAt":"2026-04-16","stderr":0.891,"costPerTest":0.011117,"latency":239.603},{"benchmarkId":"vals-ai:sage:overall","benchmarkName":"SAGE","category":null,"domain":"Education","eligible":false,"ranked":true,"score":84.905,"unit":"percent","family":"vals-ai:sage:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/sage","measuredAt":"2026-11-05","stderr":1.82,"costPerTest":0.005971,"latency":101.818},{"benchmarkId":"vals-ai:legal_bench:overall","benchmarkName":"LegalBench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":84.276,"unit":"percent","family":"vals-ai:legal_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_bench","measuredAt":"2026-10-13","stderr":0.41,"costPerTest":0.00076,"latency":16.73},{"benchmarkId":"vals-ai:mmlu_pro:overall","benchmarkName":"MMLU Pro","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":84.061,"unit":"percent","family":"mmlu-pro","index":50.387526583892836,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmlu_pro","measuredAt":"2026-09-01","stderr":0.36,"costPerTest":0.001734,"latency":33.303},{"benchmarkId":"vals-ai:lcb:overall","benchmarkName":"LiveCodeBench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":83.28,"unit":"percent","family":"livecodebench","index":58.4446160548469,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/lcb","measuredAt":"2026-09-01","stderr":1.04,"costPerTest":0.009542,"latency":209.294},{"benchmarkId":"vals-ai:gpqa:overall","benchmarkName":"GPQA Diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":82.828,"unit":"percent","family":"gpqa-diamond","index":53.176679056105485,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/gpqa","measuredAt":"2026-09-01","stderr":1.895,"costPerTest":0.003413,"latency":79.884},{"benchmarkId":"vals-ai:mmmu:overall","benchmarkName":"MMMU Pro","category":"multimodal","domain":null,"eligible":true,"ranked":true,"score":81.908,"unit":"percent","family":"mmmu-pro","index":61.74889083921626,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mmmu","measuredAt":"2026-09-01","stderr":0.926,"costPerTest":0.002615,"latency":45.836},{"benchmarkId":"vals-ai:tax_eval_v2:overall","benchmarkName":"TaxEval v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":72.158,"unit":"percent","family":"vals-ai:tax_eval_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_eval_v2","measuredAt":"2026-09-01","stderr":0.876,"costPerTest":0.003099,"latency":73.003},{"benchmarkId":"vals-ai:medscribe:overall","benchmarkName":"MedScribe","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":70.619,"unit":"percent","family":"vals-ai:medscribe:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medscribe","measuredAt":"2026-10-29","stderr":2.09,"costPerTest":0.004425,"latency":80.406},{"benchmarkId":"vals-ai:mortgage_tax:overall","benchmarkName":"MortgageTax","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":67.368,"unit":"percent","family":"vals-ai:mortgage_tax:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/mortgage_tax","measuredAt":"2026-09-01","stderr":0.922,"costPerTest":0.002158,"latency":36.822},{"benchmarkId":"vals-ai:swebench:overall","benchmarkName":"SWE-bench","category":"coding","domain":null,"eligible":true,"ranked":true,"score":64.4,"unit":"percent","family":"swe-bench-verified","index":48.72663553874141,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/swebench","measuredAt":"2026-09-01","stderr":2.143,"costPerTest":0.263227,"latency":363.101},{"benchmarkId":"vals-ai:corp_fin_v2:overall","benchmarkName":"CorpFin v2","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":63.559,"unit":"percent","family":"vals-ai:corp_fin_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/corp_fin_v2","measuredAt":"2026-08-12","stderr":0.947,"costPerTest":0.009638,"latency":35.198},{"benchmarkId":"vals-ai:case_law_v2:overall","benchmarkName":"CaseLaw v2","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":55.948,"unit":"percent","family":"vals-ai:case_law_v2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/case_law_v2","measuredAt":"2026-05-04","stderr":1.159,"costPerTest":0.004909,"latency":43.543},{"benchmarkId":"vals-ai:finance_agent:overall","benchmarkName":"Finance Agent v1.1","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":45.639,"unit":"percent","family":"vals-ai:finance_agent:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/finance_agent","measuredAt":"2026-06-04","stderr":2.775,"costPerTest":0.054887,"latency":248.888},{"benchmarkId":"vals-ai:medcode:overall","benchmarkName":"MedCode","category":null,"domain":"Healthcare","eligible":false,"ranked":true,"score":32.997,"unit":"percent","family":"vals-ai:medcode:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/medcode","measuredAt":"2026-10-29","stderr":1.787,"costPerTest":0.003934,"latency":63.074},{"benchmarkId":"vals-ai:terminal-bench-2:overall","benchmarkName":"Terminal-Bench 2.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":24.719,"unit":"percent","family":"vals-ai:terminal-bench-2:overall","index":41.43485264321955,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-2","measuredAt":"2026-06-04","stderr":4.599,"costPerTest":0.110414,"latency":716.499}],"price":{"inputPerMillion":0.1,"outputPerMillion":0.4,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":{"elo":1397.134784465809,"eloUpper":1400.7913621184784,"eloLower":1393.4782068131394,"votes":61168,"rank":161,"rankUpper":null,"rankLower":null,"arenaName":"Qwen3.5-Flash","organization":"alibaba","updatedAt":"2026-11-09"},"quality":{"coveragePercent":75,"benchmarkCount":7,"categoryCount":5,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"qwen3-5-plus","providerId":"alibaba","slug":"qwen3-5-plus","name":"Qwen3.5 Plus","provider":"Alibaba","description":"Qwen3.5 Plus is a reasoning model from Alibaba. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":"2026-03-04","contextWindowTokens":1000000,"modalities":["text","image","video"],"openWeights":false,"details":{"apiId":"qwen/qwen3.5-plus-20260420","maxOutputTokens":65536,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":null,"weightsUrl":null},"reasoning":true,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":46.72592334316041,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"agentic":1},"benchmarks":[{"benchmarkId":"jobBench","benchmarkName":"JobBench","category":"agentic","score":18.5,"unit":"percent","family":"jobBench","index":46.72592334316041,"aboveFit":false,"run":null,"source":"Yuetai Li, Yichen Feng, Zhangchen Xu, Zixian Ma, Kaiyuan Zheng, Fengqing Jiang, Xinghua Sun, Rulin Shao, Zichen Chen, Yue Huang, Xinyang Han, Brian Lee, Kayla Xu, Shenglai Zeng, Hang Hua, Xiangliang Zhang, Basel Alomair, Ranjay Krishna, Luke Zettlemoyer, Pang Wei Koh, Bhaskar Ramasubramanian, Luyao Niu, Xiang Yue, Radha Poovendran","sourceUrl":"https://arxiv.org/abs/2605.26329","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":0.3,"outputPerMillion":1.7999999999999998,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":84,"timeToFirstToken":0.991,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":20,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"qwen35plus0215","providerId":"alibaba","slug":"alibaba-qwen3-5-plus-2026-02-15","name":"Qwen3.5 Plus 2026-02-15","provider":"Alibaba","description":"Qwen3.5 Plus 2026-02-15 is a model from Alibaba. It has no scored result.","logoUrl":null,"releaseDate":"2026-02-16T08:10:16.000Z","contextWindowTokens":1000000,"modalities":["text","image","video"],"openWeights":null,"details":{"apiId":"qwen/qwen3.5-plus-02-15","maxOutputTokens":65536,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":null,"weightsUrl":null},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":0.26,"outputPerMillion":1.56,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":60,"timeToFirstToken":1.441,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"mghafiri-qwen3-5-0-8b-decision-model","providerId":"mourad-ghafiri","slug":"mghafiri-qwen3-5-0-8b-decision-model","name":"Qwen3.5-0.8B Decision Model","provider":"Mourad Ghafiri","description":"Qwen3.5-0.8B Decision Model is a standard model from Mourad Ghafiri. It has 2 scored results in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":53.91143283934313,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"reasoning":2},"benchmarks":[{"benchmarkId":"jevBench14","benchmarkName":"jevBench14","category":"reasoning","score":14.540627482985634,"unit":"percent","family":"jevBench14","index":56.348937996495486,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"jevBench15","benchmarkName":"jevBench15","category":"reasoning","score":0,"unit":"percent","family":"jevBench15","index":51.47392768219078,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":15,"benchmarkCount":2,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"qwen35-9b-jev-data-mix-v2","providerId":"jsaurabh","slug":"qwen35-9b-jev-data-mix-v2","name":"Qwen3.5-9B Jev-like data-mix v2","provider":"jsaurabh","description":"Qwen3.5-9B Jev-like data-mix v2 is a standard model from jsaurabh. It has 2 scored results in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":79.50313866862265,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"reasoning":2},"benchmarks":[{"benchmarkId":"jevBench15","benchmarkName":"jevBench15","category":"reasoning","score":53.02414313946459,"unit":"percent","family":"jevBench15","index":83.58380787680466,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"jevBench14","benchmarkName":"jevBench14","category":"reasoning","score":35.235946407850655,"unit":"percent","family":"jevBench14","index":75.42246946044067,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":15,"benchmarkCount":2,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"qwen3627bfp8","providerId":"alibaba","slug":"alibaba-qwen3-6-27b-fp8","name":"Qwen3.6 27B FP8","provider":"Alibaba","description":"Qwen3.6 27B FP8 is a model from Alibaba. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":61.55323689070532,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-portuguese:overall","benchmarkName":"EuroEval Portuguese","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":68.71,"unit":"percent","family":"euroeval","index":65.44389573792283,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/portuguese","measuredAt":null,"stderr":0.4681429402436824,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-italian:overall","benchmarkName":"EuroEval Italian","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":67.73125,"unit":"percent","family":"euroeval","index":64.11595719188186,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/italian","measuredAt":null,"stderr":0.4614058002452939,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-french:overall","benchmarkName":"EuroEval French","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":66.9525,"unit":"percent","family":"euroeval","index":63.05937262715193,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/french","measuredAt":null,"stderr":0.4968808961914314,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":66.64625,"unit":"percent","family":"euroeval","index":62.64386184326936,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":0.6110301240528163,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":65.03857142857143,"unit":"percent","family":"euroeval","index":60.46261193814129,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.43372614325970577,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-dutch:overall","benchmarkName":"EuroEval Dutch","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":63.99444444444445,"unit":"percent","family":"euroeval","index":59.04597186745547,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/dutch","measuredAt":null,"stderr":0.43421562601608804,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-spanish:overall","benchmarkName":"EuroEval Spanish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":61.3625,"unit":"percent","family":"euroeval","index":55.47502884950787,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/spanish","measuredAt":null,"stderr":0.5758648387425647,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":56.723749999999995,"unit":"percent","family":"euroeval","index":49.18131244547453,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.6502475490149886,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"qwen36flash","providerId":"alibaba","slug":"alibaba-qwen3-6-flash","name":"Qwen3.6 Flash","provider":"Alibaba","description":"Qwen3.6 Flash is a model from Alibaba. It has 8 scored results in 3 categories. It has no index.","logoUrl":null,"releaseDate":"2026-04-27T03:42:42.000Z","contextWindowTokens":1000000,"modalities":["text","image","video"],"openWeights":null,"details":{"apiId":"qwen/qwen3.6-flash","maxOutputTokens":65536,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":null,"weightsUrl":null},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":47.85666220485169,"multimodal":null,"knowledge":45.40928613830697,"multilingual":null,"instruction":null,"math":45.752900870930425},"capabilityCounts":{"math":4,"knowledge":2,"reasoning":2},"benchmarks":[{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":84.44444444444444,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":58.04836584958089,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":5.463890236888291,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":83.33333333333334,"unit":"percent","family":"gpqa-diamond","index":53.64015698116496,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.6552207828215257,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tiers-1-3-v2-private:overall","benchmarkName":"FrontierMath-Tiers-1-3-v2-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":22.45614035087719,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tiers-1-3-v2-private:overall","index":42.714524208171184,"aboveFit":false,"run":"none effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.4761793289496805,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-2025-02-28-public:overall","benchmarkName":"FrontierMath-2025-02-28-Public","category":"math","domain":null,"eligible":false,"ranked":true,"score":20,"unit":"percent","family":"epoch-ai:epoch-frontiermath-2025-02-28-public:overall","index":null,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":13.333333333333334,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-mystery-game-puzzles:overall","benchmarkName":"Mystery Game Puzzles","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":18,"unit":"percent","family":"epoch-ai:epoch-mystery-game-puzzles:overall","index":52.62046634276329,"aboveFit":false,"run":"none effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":3.8612291966536914,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-simpleqa-verified:overall","benchmarkName":"SimpleQA Verified","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":15.915915915915916,"unit":"percent","family":"epoch-ai:epoch-simpleqa-verified:overall","index":37.17841529544897,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.157997032121365,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-chess-puzzles:overall","benchmarkName":"Chess Puzzles","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":12,"unit":"percent","family":"epoch-ai:epoch-chess-puzzles:overall","index":43.09285806694008,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":3.2659863237109046,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-2025-02-28-private:overall","benchmarkName":"FrontierMath-2025-02-28-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":10.344827586206897,"unit":"percent","family":"epoch-ai:epoch-frontiermath-2025-02-28-private:overall","index":39.046355698804014,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.791432224407271,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-private:overall","benchmarkName":"FrontierMath-Tier-4-2025-07-01-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":0,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-private:overall","index":43.20235772716562,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":0.1875,"outputPerMillion":1.125,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":75,"timeToFirstToken":0.582,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":40,"benchmarkCount":8,"categoryCount":3,"confidence":"unranked","state":"partial"},"hosts":[{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":0.1875,"outputPerMillion":1.125,"cacheReadPerMillion":null,"contextTokens":1000000,"maxOutputTokens":65536,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"qwen3-7-flash","providerId":"alibaba","slug":"qwen3-7-flash","name":"Qwen3.7 Flash","provider":"Alibaba","description":"Qwen3.7 Flash is a reasoning model from Alibaba. It has 4 scored results in 3 categories. It has no index.","logoUrl":null,"releaseDate":"2026-07-27","contextWindowTokens":1000000,"modalities":["text","image","video"],"openWeights":false,"details":{"apiId":"qwen/qwen3.7-flash","maxOutputTokens":65536,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":false,"cacheReadPerMillion":0.006,"weightsUrl":null},"reasoning":true,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":49.522806755942604,"multimodal":null,"knowledge":51.32406436773115,"multilingual":null,"instruction":null,"math":47.611049893680054},"capabilityCounts":{"knowledge":1,"math":2,"reasoning":1},"benchmarks":[{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":80.8080808080808,"unit":"percent","family":"gpqa-diamond","index":51.32406436773115,"aboveFit":false,"run":"none effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.805779167298906,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":77.77777777777779,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":54.16973887802699,"aboveFit":false,"run":"none effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":6.267511942419625,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-frontiermath-tiers-1-3-v2-private:overall","benchmarkName":"FrontierMath-Tiers-1-3-v2-Private","category":"math","domain":null,"eligible":true,"ranked":true,"score":19.298245614035086,"unit":"percent","family":"epoch-ai:epoch-frontiermath-tiers-1-3-v2-private:overall","index":41.052360909333125,"aboveFit":false,"run":"none effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.341753795065531,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-mystery-game-puzzles:overall","benchmarkName":"Mystery Game Puzzles","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":14.000000000000002,"unit":"percent","family":"epoch-ai:epoch-mystery-game-puzzles:overall","index":49.522806755942604,"aboveFit":false,"run":"none effort","source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":3.4873508801977695,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":0.03,"outputPerMillion":0.13,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":56,"timeToFirstToken":0.67,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":40,"benchmarkCount":4,"categoryCount":3,"confidence":"unranked","state":"partial"},"hosts":[{"host":"alibaba-cloud","name":"Alibaba Cloud","kind":"maker","url":"https://www.alibabacloud.com/help/en/model-studio/models","inputPerMillion":0.03,"outputPerMillion":0.13,"cacheReadPerMillion":0.006,"contextTokens":1000000,"maxOutputTokens":131072,"deprecated":false},{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":0.03,"outputPerMillion":0.13,"cacheReadPerMillion":0.006,"contextTokens":1000000,"maxOutputTokens":65536,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"qwen3824ta95b","providerId":"alibaba","slug":"alibaba-qwen3-8-2-4t-a95b","name":"Qwen3.8 2.4T A95B","provider":"Alibaba","description":"Qwen3.8 2.4T A95B is a model from Alibaba. It has no scored result.","logoUrl":null,"releaseDate":"2026-08-12T16:21:42.000Z","contextWindowTokens":1048576,"modalities":["text"],"openWeights":true,"details":{"apiId":"qwen/qwen3.8-2.4t-a95b","maxOutputTokens":131072,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.25,"weightsUrl":"https://huggingface.co/Qwen/Qwen3.8-2.4T-A95B"},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":2,"outputPerMillion":6,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":50,"timeToFirstToken":1.483,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[{"host":"deepinfra","name":"Deep Infra","kind":"host","url":"https://deepinfra.com/models","inputPerMillion":2,"outputPerMillion":6,"cacheReadPerMillion":0.2,"contextTokens":262144,"maxOutputTokens":131072,"deprecated":true},{"host":"fireworks-ai","name":"Fireworks AI","kind":"host","url":"https://fireworks.ai/docs/","inputPerMillion":2,"outputPerMillion":6,"cacheReadPerMillion":0.25,"contextTokens":262144,"maxOutputTokens":131072,"deprecated":false},{"host":"modal","name":"Modal","kind":"host","url":"https://modal.com/docs/guide/endpoints","inputPerMillion":2,"outputPerMillion":6,"cacheReadPerMillion":0.25,"contextTokens":1010000,"maxOutputTokens":131072,"deprecated":false},{"host":"novita-ai","name":"Novita AI","kind":"host","url":"https://novita.ai/docs/guides/introduction","inputPerMillion":2,"outputPerMillion":6,"cacheReadPerMillion":0.25,"contextTokens":1000000,"maxOutputTokens":131072,"deprecated":false},{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":2,"outputPerMillion":6,"cacheReadPerMillion":0.25,"contextTokens":1048576,"maxOutputTokens":131072,"deprecated":false},{"host":"siliconflow","name":"SiliconFlow","kind":"host","url":"https://cloud.siliconflow.com/models","inputPerMillion":2,"outputPerMillion":6,"cacheReadPerMillion":0.25,"contextTokens":1049000,"maxOutputTokens":131000,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"qwen38flash","providerId":"alibaba","slug":"alibaba-qwen3-8-flash","name":"Qwen3.8 Flash","provider":"Alibaba","description":"Qwen3.8 Flash is a model from Alibaba. It has no scored result.","logoUrl":null,"releaseDate":"2026-08-26T19:37:40.000Z","contextWindowTokens":1000000,"modalities":["text","image","video"],"openWeights":true,"details":{"apiId":"qwen/qwen3.8-flash","maxOutputTokens":131072,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.016,"weightsUrl":"https://huggingface.co/Qwen/Qwen3.8-Flash-Next"},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":0.15,"outputPerMillion":0.47,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":60,"timeToFirstToken":0.996,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[{"host":"deepinfra","name":"Deep Infra","kind":"host","url":"https://deepinfra.com/models","inputPerMillion":0.113,"outputPerMillion":0.382,"cacheReadPerMillion":0.0141,"contextTokens":1000000,"maxOutputTokens":131072,"deprecated":false},{"host":"alibaba-cloud","name":"Alibaba Cloud","kind":"maker","url":"https://www.alibabacloud.com/help/en/model-studio/models","inputPerMillion":0.15,"outputPerMillion":0.47,"cacheReadPerMillion":0.016,"contextTokens":1000000,"maxOutputTokens":131072,"deprecated":false},{"host":"novita-ai","name":"Novita AI","kind":"host","url":"https://novita.ai/docs/guides/introduction","inputPerMillion":0.15,"outputPerMillion":0.47,"cacheReadPerMillion":0.016,"contextTokens":1000000,"maxOutputTokens":131072,"deprecated":false},{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":0.15,"outputPerMillion":0.47,"cacheReadPerMillion":0.016,"contextTokens":1000000,"maxOutputTokens":131072,"deprecated":false},{"host":"gmi-cloud","name":"GMI Cloud","kind":"host","url":"https://docs.gmicloud.ai/inference-engine/api-reference/llm-api-reference","inputPerMillion":0.16,"outputPerMillion":0.47,"cacheReadPerMillion":0.016,"contextTokens":1048575,"maxOutputTokens":131072,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"qwen38maxinternal","providerId":"alibaba","slug":"alibaba-qwen3-8-max-internal","name":"Qwen3.8 Max Internal","provider":"Alibaba","description":"Qwen3.8 Max Internal is a model from Alibaba. It has 3 scored results in 2 categories. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":72.98201300310362,"coding":71.43014060546076,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"coding":2,"agentic":1},"benchmarks":[{"benchmarkId":"vals-ai:vibe-code:overall","benchmarkName":"Vibe Code Bench v1.1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":77.141,"unit":"percent","family":"vals-ai:vibe-code:overall","index":70.08320736696902,"aboveFit":false,"run":"OpenHands","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","measuredAt":"2026-11-09","stderr":3.721,"costPerTest":11.668329,"latency":5079.378},{"benchmarkId":"vals-ai:emb:overall","benchmarkName":"EMB","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":71.979,"unit":"percent","family":"vals-ai:emb:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/emb","measuredAt":"2026-10-20","stderr":2.387,"costPerTest":4.430287,"latency":3193.384},{"benchmarkId":"vals-ai:fabv2:overall","benchmarkName":"Finance Agent (v2)","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":57.872,"unit":"percent","family":"vals-ai:fabv2:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/fabv2","measuredAt":"2026-10-29","stderr":0.109,"costPerTest":3.026128,"latency":2945.201},{"benchmarkId":"vals-ai:tax_agent_bench:overall","benchmarkName":"Tax Agent Bench","category":null,"domain":"Finance","eligible":false,"ranked":true,"score":49.663,"unit":"percent","family":"vals-ai:tax_agent_bench:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/tax_agent_bench","measuredAt":"2026-11-09","stderr":3.603,"costPerTest":4.068548,"latency":5231.239},{"benchmarkId":"vals-ai:legal_research:overall","benchmarkName":"Legal Research Bench","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":46.154,"unit":"percent","family":"vals-ai:legal_research:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/legal_research","measuredAt":"2026-11-09","stderr":3.465,"costPerTest":5.312295,"latency":7649.584},{"benchmarkId":"vals-ai:terminal-bench-4:overall","benchmarkName":"Terminal-Bench 4.0","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":38.889,"unit":"percent","family":"terminal-bench-4","index":72.98201300310362,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/terminal-bench-4","measuredAt":"2026-11-09","stderr":1.336,"costPerTest":12.146852,"latency":8591.435},{"benchmarkId":"vals-ai:vcb-1-100:overall","benchmarkName":"Vibe Code Bench 1-100","category":"coding","domain":null,"eligible":true,"ranked":true,"score":24.833,"unit":"percent","family":"vals-ai:vcb-1-100:overall","index":72.7770738439525,"aboveFit":false,"run":"OpenHands","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vcb-1-100","measuredAt":"2026-10-29","stderr":4.489,"costPerTest":11.589207,"latency":9477.164},{"benchmarkId":"vals-ai:hlab:overall","benchmarkName":"HLAB-Vals-1.1","category":null,"domain":"Legal","eligible":false,"ranked":true,"score":15.417,"unit":"percent","family":"vals-ai:hlab:overall","index":null,"aboveFit":false,"run":null,"source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/hlab","measuredAt":"2026-11-09","stderr":3.063,"costPerTest":3.358663,"latency":2950.56}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":40,"benchmarkCount":3,"categoryCount":2,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"qwen38maxprime","providerId":"alibaba","slug":"alibaba-qwen3-8-max-prime","name":"Qwen3.8 Max Prime","provider":"Alibaba","description":"Qwen3.8 Max Prime is a model from Alibaba. It has no scored result.","logoUrl":null,"releaseDate":"2026-09-23T19:20:28.000Z","contextWindowTokens":1000000,"modalities":["text","image","video"],"openWeights":null,"details":{"apiId":"qwen/qwen3.8-max-prime","maxOutputTokens":131072,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.5,"weightsUrl":null},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":4,"outputPerMillion":12,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":56,"timeToFirstToken":1.5725,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":4,"outputPerMillion":12,"cacheReadPerMillion":0.5,"contextTokens":1000000,"maxOutputTokens":131072,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"qwen3-8-27b-chutes","providerId":"alibaba","slug":"qwen3-8-27b-chutes","name":"Qwen3.8-27B (Chutes TEE)","provider":"Alibaba","description":"Qwen3.8-27B (Chutes TEE) is a reasoning model from Alibaba. It has 2 scored results in 1 category. It has no index.","logoUrl":null,"releaseDate":"2026-08-14","contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":true,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":47.21085681031145,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"reasoning":2},"benchmarks":[{"benchmarkId":"jevBench14","benchmarkName":"jevBench14","category":"reasoning","score":0,"unit":"percent","family":"jevBench14","index":42.94778593843212,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"jevBench15","benchmarkName":"jevBench15","category":"reasoning","score":0,"unit":"percent","family":"jevBench15","index":51.47392768219078,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":15,"benchmarkCount":2,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"qwen4-27b","providerId":"alibaba","slug":"qwen4-27b","name":"Qwen4 27B","provider":"Alibaba","description":"Qwen4 27B is a model from Alibaba. It has no scored result.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"qwen4-flash-plus","providerId":"alibaba","slug":"qwen4-flash-plus","name":"Qwen4 Flash & Plus","provider":"Alibaba","description":"Qwen4 Flash & Plus is a model from Alibaba. It has no scored result.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"qwen4-max","providerId":"alibaba","slug":"qwen4-max","name":"Qwen4 Max","provider":"Alibaba","description":"Qwen4 Max is a model from Alibaba. It has no scored result.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"qwqplus","providerId":"alibaba","slug":"alibaba-qwq-plus","name":"QWQ Plus","provider":"Alibaba","description":"QWQ Plus is a model from Alibaba. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":37.19589942578497,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"knowledge":1},"benchmarks":[{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":65.40404040404042,"unit":"percent","family":"gpqa-diamond","index":37.19589942578497,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.8469238770829937,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":15,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"deepseekr1distillllama70b","providerId":"deepseek","slug":"deepseek-r1-distill-llama-70b","name":"R1 Distill Llama 70B","provider":"DeepSeek","description":"R1 Distill Llama 70B is a model from DeepSeek. It has 3 scored results in 2 categories. It has no index.","logoUrl":null,"releaseDate":"2025-01-23T20:12:49.000Z","contextWindowTokens":8192,"modalities":["text"],"openWeights":true,"details":{"apiId":"deepseek/deepseek-r1-distill-llama-70b","maxOutputTokens":7372,"knowledgeCutoff":"2024-07-31","retiresAt":"2026-09-28","tools":false,"structuredOutput":false,"cacheReadPerMillion":null,"weightsUrl":"https://huggingface.co/deepseek-ai/DeepSeek-R1-Distill-Llama-70B"},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":28.336845179400676,"multilingual":null,"instruction":null,"math":40.74160306881627},"capabilityCounts":{"math":2,"knowledge":1},"benchmarks":[{"benchmarkId":"epoch-ai:epoch-math-level-5:overall","benchmarkName":"MATH level 5","category":"math","domain":null,"eligible":true,"ranked":true,"score":89.89803625377644,"unit":"percent","family":"epoch-ai:epoch-math-level-5:overall","index":42.666365688673096,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0.6818015200174493,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":55.744949494949495,"unit":"percent","family":"gpqa-diamond","index":28.336845179400676,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.973465219179067,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":51.388888888888886,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":38.81684044895944,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":6.054026310473158,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":0.7999999999999999,"outputPerMillion":0.7999999999999999,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":19,"timeToFirstToken":1.041,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":25,"benchmarkCount":3,"categoryCount":2,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"raft30ba3b","providerId":"unknown","slug":"unknown-raft-30b-a3b","name":"RAFT 30B A3B","provider":"Unknown","description":"RAFT 30B A3B is a model. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":57.505117795015146,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"agentic":1},"benchmarks":[{"benchmarkId":"tau2-bench:retail:overall","benchmarkName":"τ²-bench Retail","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":82.45614035087719,"unit":"percent","family":"tau2-bench","index":57.505117795015146,"aboveFit":false,"run":"Northeastern University HAI Lab","source":"Sierra Research","sourceUrl":"https://github.com/sierra-research/tau2-bench","measuredAt":"2026-04-30","stderr":null,"costPerTest":0,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":20,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:sierra-research","name":"Sierra Research","url":"https://github.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"raw-phi-4-mini","providerId":"microsoft","slug":"raw-phi-4-mini","name":"Raw Phi-4 mini direct logits","provider":"Microsoft","description":"Raw Phi-4 mini direct logits is a standard model from Microsoft. It has 2 scored results in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":69.31132619100288,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"reasoning":2},"benchmarks":[{"benchmarkId":"jevBench14","benchmarkName":"jevBench14","category":"reasoning","score":37.95731820837918,"unit":"percent","family":"jevBench14","index":77.93058111848859,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"jevBench15","benchmarkName":"jevBench15","category":"reasoning","score":15.222235703588067,"unit":"percent","family":"jevBench15","index":60.69207126351716,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":15,"benchmarkCount":2,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"raw-qwen3-0-6b","providerId":"alibaba","slug":"raw-qwen3-0-6b","name":"Raw Qwen3 0.6B direct logits","provider":"Alibaba","description":"Raw Qwen3 0.6B direct logits is a standard model from Alibaba. It has 2 scored results in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":50.82593185188219,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"reasoning":2},"benchmarks":[{"benchmarkId":"jevBench14","benchmarkName":"jevBench14","category":"reasoning","score":7.135291650528435,"unit":"percent","family":"jevBench14","index":49.52392087478547,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"jevBench15","benchmarkName":"jevBench15","category":"reasoning","score":1.079997575465549,"unit":"percent","family":"jevBench15","index":52.12794282897892,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":15,"benchmarkCount":2,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"raw-qwen3-1-7b","providerId":"alibaba","slug":"raw-qwen3-1-7b","name":"Raw Qwen3 1.7B direct logits","provider":"Alibaba","description":"Raw Qwen3 1.7B direct logits is a standard model from Alibaba. It has 2 scored results in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":55.78965662232427,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"reasoning":2},"benchmarks":[{"benchmarkId":"jevBench14","benchmarkName":"jevBench14","category":"reasoning","score":18.055721418631492,"unit":"percent","family":"jevBench14","index":59.588571800819736,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"jevBench15","benchmarkName":"jevBench15","category":"reasoning","score":0.8534322366651821,"unit":"percent","family":"jevBench15","index":51.99074144382881,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":15,"benchmarkCount":2,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"raw-qwen3-4b-instruct-2507","providerId":"alibaba","slug":"raw-qwen3-4b-instruct-2507","name":"Raw Qwen3 4B Instruct 2507 direct logits","provider":"Alibaba","description":"Raw Qwen3 4B Instruct 2507 direct logits is a standard model from Alibaba. It has 2 scored results in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":84.89680054199553,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"reasoning":2},"benchmarks":[{"benchmarkId":"jevBench15","benchmarkName":"jevBench15","category":"reasoning","score":62.13695038108371,"unit":"percent","family":"jevBench15","index":89.10225912259605,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"jevBench14","benchmarkName":"jevBench14","category":"reasoning","score":40.95282149140936,"unit":"percent","family":"jevBench14","index":80.691341961395,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":15,"benchmarkCount":2,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"raw-qwen3-8b","providerId":"alibaba","slug":"raw-qwen3-8b","name":"Raw Qwen3 8B direct logits","provider":"Alibaba","description":"Raw Qwen3 8B direct logits is a standard model from Alibaba. It has 2 scored results in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":71.81438415557218,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"reasoning":2},"benchmarks":[{"benchmarkId":"jevBench15","benchmarkName":"jevBench15","category":"reasoning","score":45.223917256408555,"unit":"percent","family":"jevBench15","index":78.86021775792085,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"jevBench14","benchmarkName":"jevBench14","category":"reasoning","score":23.676144280945373,"unit":"percent","family":"jevBench14","index":64.76855055322352,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":15,"benchmarkCount":2,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"reflex-4b","providerId":"kshetrajna12","slug":"reflex-4b","name":"reflex 4B","provider":"kshetrajna12","description":"reflex 4B is a standard model from kshetrajna12. It has 2 scored results in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":91.84395107969988,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"reasoning":2},"benchmarks":[{"benchmarkId":"jevBench15","benchmarkName":"jevBench15","category":"reasoning","score":65.23881652418177,"unit":"percent","family":"jevBench15","index":90.98065911809692,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"jevBench14","benchmarkName":"jevBench14","category":"reasoning","score":53.99041264165809,"unit":"percent","family":"jevBench14","index":92.70724304130283,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":15,"benchmarkCount":2,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"reflex-27b","providerId":"kshetrajna12","slug":"reflex-27b","name":"reflex-27b","provider":"kshetrajna12","description":"reflex-27b is a standard model from kshetrajna12. It has 2 scored results in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":59.42714890227636,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"reasoning":2},"benchmarks":[{"benchmarkId":"jevBench14","benchmarkName":"jevBench14","category":"reasoning","score":17.844519244120388,"unit":"percent","family":"jevBench14","index":59.39392047240019,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"jevBench15","benchmarkName":"jevBench15","category":"reasoning","score":13.188297397843682,"unit":"percent","family":"jevBench15","index":59.460377332152525,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":15,"benchmarkCount":2,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"rekaedge","providerId":"reka-ai","slug":"reka-ai-reka-edge","name":"Reka Edge","provider":"Reka AI","description":"Reka Edge is a model from Reka AI. It has no scored result.","logoUrl":null,"releaseDate":"2026-03-20T17:16:05.000Z","contextWindowTokens":16384,"modalities":["image","text","video"],"openWeights":true,"details":{"apiId":"rekaai/reka-edge","maxOutputTokens":14745,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":null,"weightsUrl":"https://huggingface.co/RekaAI/reka-edge-2603"},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":0.09999999999999999,"outputPerMillion":0.09999999999999999,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":113.5,"timeToFirstToken":0.472,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":0.1,"outputPerMillion":0.1,"cacheReadPerMillion":null,"contextTokens":16384,"maxOutputTokens":14745,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"rekaflash3","providerId":"reka-ai","slug":"reka-ai-reka-flash-3","name":"Reka Flash 3","provider":"Reka AI","description":"Reka Flash 3 is a model from Reka AI. It has no scored result.","logoUrl":null,"releaseDate":"2025-03-12T20:53:33.000Z","contextWindowTokens":65536,"modalities":["text"],"openWeights":true,"details":{"apiId":"rekaai/reka-flash-3","maxOutputTokens":58982,"knowledgeCutoff":"2025-01-31","retiresAt":null,"tools":false,"structuredOutput":true,"cacheReadPerMillion":null,"weightsUrl":"https://huggingface.co/RekaAI/reka-flash-3"},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":0.09999999999999999,"outputPerMillion":0.19999999999999998,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":81,"timeToFirstToken":1.6075,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":0.1,"outputPerMillion":0.2,"cacheReadPerMillion":null,"contextTokens":65536,"maxOutputTokens":58982,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"relaceapply3","providerId":"relace","slug":"relace-relace-apply-3","name":"Relace Apply 3","provider":"Relace","description":"Relace Apply 3 is a model from Relace. It has no scored result.","logoUrl":null,"releaseDate":"2025-09-26T12:59:32.000Z","contextWindowTokens":256000,"modalities":["text"],"openWeights":null,"details":{"apiId":"relace/relace-apply-3","maxOutputTokens":128000,"knowledgeCutoff":null,"retiresAt":null,"tools":false,"structuredOutput":false,"cacheReadPerMillion":null,"weightsUrl":null},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":0.85,"outputPerMillion":1.25,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":4214.5,"timeToFirstToken":0.549,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":0.85,"outputPerMillion":1.25,"cacheReadPerMillion":null,"contextTokens":256000,"maxOutputTokens":128000,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"relacesearch","providerId":"relace","slug":"relace-relace-search","name":"Relace Search","provider":"Relace","description":"Relace Search is a model from Relace. It has no scored result.","logoUrl":null,"releaseDate":"2025-12-08T17:06:00.000Z","contextWindowTokens":256000,"modalities":["text"],"openWeights":null,"details":{"apiId":"relace/relace-search","maxOutputTokens":128000,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":false,"cacheReadPerMillion":null,"weightsUrl":null},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":1,"outputPerMillion":3,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":87.5,"timeToFirstToken":0.982,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":1,"outputPerMillion":3,"cacheReadPerMillion":null,"contextTokens":256000,"maxOutputTokens":128000,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"remmslerpl213b","providerId":"undi95","slug":"undi95-remm-slerp-13b","name":"ReMM SLERP 13B","provider":"Undi95","description":"ReMM SLERP 13B is a model from Undi95. It has no scored result.","logoUrl":null,"releaseDate":"2023-07-22T00:00:00.000Z","contextWindowTokens":6144,"modalities":["text"],"openWeights":true,"details":{"apiId":"undi95/remm-slerp-l2-13b","maxOutputTokens":5529,"knowledgeCutoff":"2023-06-30","retiresAt":null,"tools":false,"structuredOutput":true,"cacheReadPerMillion":0.19999999999999998,"weightsUrl":"https://huggingface.co/Undi95/ReMM-SLERP-L2-13B"},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":0.3,"outputPerMillion":0.5,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":55.5,"timeToFirstToken":0.764,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":0.3,"outputPerMillion":0.5,"cacheReadPerMillion":0.2,"contextTokens":6144,"maxOutputTokens":5529,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"decision-index-0-3-rsi-jev-v5-0-vl-3b","providerId":"shgao","slug":"decision-index-0-3-rsi-jev-v5-0-vl-3b","name":"RSI-Jev v5.0 VL 3B","provider":"shgao","description":"RSI-Jev v5.0 VL 3B is a model from shgao. It has no scored result.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"mistralsaba","providerId":"mistral","slug":"mistral-saba","name":"Saba","provider":"Mistral","description":"Saba is a model from Mistral. It has no scored result.","logoUrl":null,"releaseDate":"2025-02-17T14:40:39.000Z","contextWindowTokens":32768,"modalities":["text","file"],"openWeights":null,"details":{"apiId":"mistralai/mistral-saba","maxOutputTokens":26214,"knowledgeCutoff":"2024-09-30","retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.02,"weightsUrl":null},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":0.19999999999999998,"outputPerMillion":0.6,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":82,"timeToFirstToken":0.417,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":0.2,"outputPerMillion":0.6,"cacheReadPerMillion":0.02,"contextTokens":32768,"maxOutputTokens":26214,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"sage-a1-5-f16-gguf","providerId":"internscience","slug":"sage-a1-5-f16-gguf","name":"Sage-A1.5-F16-GGUF","provider":"InternScience","description":"Sage-A1.5-F16-GGUF is a reasoning model from InternScience in the Sage-A1.5 family. It has no scored result.","logoUrl":null,"releaseDate":"2026-10-21","contextWindowTokens":262000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":true,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"sage-a1-5-fp8","providerId":"internscience","slug":"sage-a1-5-fp8","name":"Sage-A1.5-FP8","provider":"InternScience","description":"Sage-A1.5-FP8 is a reasoning model from InternScience in the Sage-A1.5 family. It has no scored result.","logoUrl":null,"releaseDate":"2026-10-21","contextWindowTokens":262000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":true,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"sage-a1-5-q4-k-m-gguf","providerId":"internscience","slug":"sage-a1-5-q4-k-m-gguf","name":"Sage-A1.5-Q4_K_M-GGUF","provider":"InternScience","description":"Sage-A1.5-Q4_K_M-GGUF is a reasoning model from InternScience in the Sage-A1.5 family. It has no scored result.","logoUrl":null,"releaseDate":"2026-10-21","contextWindowTokens":262000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":true,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"sage-a1-5-q8-0-gguf","providerId":"internscience","slug":"sage-a1-5-q8-0-gguf","name":"Sage-A1.5-Q8_0-GGUF","provider":"InternScience","description":"Sage-A1.5-Q8_0-GGUF is a reasoning model from InternScience in the Sage-A1.5 family. It has no scored result.","logoUrl":null,"releaseDate":"2026-10-21","contextWindowTokens":262000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":true,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"sakana-fugu-ultra-v1-1","providerId":"sakana","slug":"sakana-fugu-ultra-v1-1","name":"Sakana Fugu-Ultra v1.1","provider":"Sakana","description":"Sakana Fugu-Ultra v1.1 is a reasoning model from Sakana in the Sakana Fugu family. It has no scored result.","logoUrl":null,"releaseDate":"2026-07-24","contextWindowTokens":1000000,"modalities":["text"],"openWeights":false,"details":null,"reasoning":true,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":5,"outputPerMillion":30,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"sakana-namazu","providerId":"sakana","slug":"sakana-namazu","name":"Sakana Namazu","provider":"Sakana","description":"Sakana Namazu is a reasoning model from Sakana. It has no scored result.","logoUrl":null,"releaseDate":"2026-08-03","contextWindowTokens":262144,"modalities":["text","image","file"],"openWeights":false,"details":{"apiId":"sakana/sakana-namazu","maxOutputTokens":65536,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.15,"weightsUrl":null},"reasoning":true,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":0.95,"outputPerMillion":4,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":102,"timeToFirstToken":0.855,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":0.95,"outputPerMillion":4,"cacheReadPerMillion":0.15,"contextTokens":262144,"maxOutputTokens":65536,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"salamandra7b","providerId":"unknown","slug":"unknown-salamandra-7b","name":"Salamandra 7B","provider":"Unknown","description":"Salamandra 7B is a model. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":5,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-dutch:overall","benchmarkName":"EuroEval Dutch","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":26.191111111111113,"unit":"percent","family":"euroeval","index":7.755546833310454,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/dutch","measuredAt":null,"stderr":0.3552411804623674,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-french:overall","benchmarkName":"EuroEval French","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":25.196249999999996,"unit":"percent","family":"euroeval","index":6.405749216552728,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/french","measuredAt":null,"stderr":0.4430416882190659,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-portuguese:overall","benchmarkName":"EuroEval Portuguese","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":23.19,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/portuguese","measuredAt":null,"stderr":0.5220886538702024,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-italian:overall","benchmarkName":"EuroEval Italian","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":18.566249999999997,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/italian","measuredAt":null,"stderr":0.5199579309905754,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-spanish:overall","benchmarkName":"EuroEval Spanish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":17.2075,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/spanish","measuredAt":null,"stderr":0.3117942029608633,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":16.335,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.637268340261777,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":15.405,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":0.332349590040367,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":15.38,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.4741264792936026,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"salamandra7binstruct","providerId":"unknown","slug":"unknown-salamandra-7b-instruct","name":"Salamandra 7B Instruct","provider":"Unknown","description":"Salamandra 7B Instruct is a model. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":5.402587466893422,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-dutch:overall","benchmarkName":"EuroEval Dutch","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":28.09444444444445,"unit":"percent","family":"euroeval","index":10.337932222120692,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/dutch","measuredAt":null,"stderr":0.28556204056622386,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":26.49285714285714,"unit":"percent","family":"euroeval","index":8.164946768605937,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.3755784654016646,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":25.668749999999996,"unit":"percent","family":"euroeval","index":7.046822997400113,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":0.3420983045851002,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-portuguese:overall","benchmarkName":"EuroEval Portuguese","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":25.21,"unit":"percent","family":"euroeval","index":6.424404802767874,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/portuguese","measuredAt":null,"stderr":0.3500557991235112,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-french:overall","benchmarkName":"EuroEval French","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":23.70375,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/french","measuredAt":null,"stderr":1.042819195738168,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":21.07125,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.409954265741924,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-italian:overall","benchmarkName":"EuroEval Italian","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":20.398750000000003,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/italian","measuredAt":null,"stderr":0.4747301865270419,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-spanish:overall","benchmarkName":"EuroEval Spanish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":18.9025,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/spanish","measuredAt":null,"stderr":0.3679886207479791,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"salmonn-13b","providerId":"salmonn-authors","slug":"salmonn-13b","name":"SALMONN 13B","provider":"SALMONN authors","description":"SALMONN 13B is a standard model from SALMONN authors. It has no scored result.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"salmonn-7b","providerId":"salmonn-authors","slug":"salmonn-7b","name":"SALMONN 7B","provider":"SALMONN authors","description":"SALMONN 7B is a standard model from SALMONN authors. It has no scored result.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"sambalingobulgarianbase","providerId":"unknown","slug":"unknown-sambalingo-bulgarian-base","name":"SambaLingo Bulgarian Base","provider":"Unknown","description":"SambaLingo Bulgarian Base is a model. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":5,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":11.657142857142857,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.29879690734131553,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"scaledown-classify","providerId":"scaledown","slug":"scaledown-classify","name":"ScaleDown Classify","provider":"ScaleDown","description":"ScaleDown Classify is a model from ScaleDown. It has no scored result.","logoUrl":null,"releaseDate":null,"contextWindowTokens":1000000,"modalities":["text"],"openWeights":false,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":0.05,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"scaledown-compress","providerId":"scaledown","slug":"scaledown-compress","name":"ScaleDown Compress","provider":"ScaleDown","description":"ScaleDown Compress is a model from ScaleDown. It has no scored result.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":false,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":0.05,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"scaledown-extract","providerId":"scaledown","slug":"scaledown-extract-scaledown-extract","name":"ScaleDown Extract","provider":"ScaleDown","description":"ScaleDown Extract is a model from ScaleDown. It has no scored result.","logoUrl":null,"releaseDate":null,"contextWindowTokens":1000000,"modalities":["text"],"openWeights":false,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":0.05,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"scaledown-summarize","providerId":"scaledown","slug":"scaledown-summarize","name":"ScaleDown Summarize","provider":"ScaleDown","description":"ScaleDown Summarize is a model from ScaleDown. It has no scored result.","logoUrl":null,"releaseDate":null,"contextWindowTokens":1000000,"modalities":["text"],"openWeights":false,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":0.05,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"schematronv2small","providerId":"inference-net","slug":"inference-net-schematron-v2-small","name":"Schematron V2 Small","provider":"Inference.Net","description":"Schematron V2 Small is a model from Inference.Net. It has no scored result.","logoUrl":null,"releaseDate":"2026-09-12T01:35:33.000Z","contextWindowTokens":128000,"modalities":["text"],"openWeights":null,"details":{"apiId":"inference-net/schematron-v2-small","maxOutputTokens":8192,"knowledgeCutoff":null,"retiresAt":null,"tools":false,"structuredOutput":true,"cacheReadPerMillion":0.049999999999999996,"weightsUrl":null},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":0.049999999999999996,"outputPerMillion":0.22999999999999998,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":244.5,"timeToFirstToken":0.2945,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":0.05,"outputPerMillion":0.23,"cacheReadPerMillion":0.05,"contextTokens":128000,"maxOutputTokens":8192,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"schematronv2turbo","providerId":"inference-net","slug":"inference-net-schematron-v2-turbo","name":"Schematron V2 Turbo","provider":"Inference.Net","description":"Schematron V2 Turbo is a model from Inference.Net. It has no scored result.","logoUrl":null,"releaseDate":"2026-09-12T01:35:49.000Z","contextWindowTokens":128000,"modalities":["text"],"openWeights":null,"details":{"apiId":"inference-net/schematron-v2-turbo","maxOutputTokens":8192,"knowledgeCutoff":null,"retiresAt":null,"tools":false,"structuredOutput":true,"cacheReadPerMillion":0.03,"weightsUrl":null},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":0.03,"outputPerMillion":0.15,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":202,"timeToFirstToken":0.5215,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":0.03,"outputPerMillion":0.15,"cacheReadPerMillion":0.03,"contextTokens":128000,"maxOutputTokens":8192,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"seed-1-6","providerId":"bytedance","slug":"seed-1-6","name":"Seed 1.6","provider":"ByteDance","description":"Seed 1.6 is a reasoning model from ByteDance. It has no scored result.","logoUrl":null,"releaseDate":"2025-06-01","contextWindowTokens":262144,"modalities":["image","text","video"],"openWeights":false,"details":{"apiId":"bytedance-seed/seed-1.6","maxOutputTokens":32768,"knowledgeCutoff":null,"retiresAt":"2026-11-11","tools":true,"structuredOutput":true,"cacheReadPerMillion":null,"weightsUrl":null},"reasoning":true,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":0.25,"outputPerMillion":2,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":37,"timeToFirstToken":0.817,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"seed-1-6-flash","providerId":"bytedance","slug":"seed-1-6-flash","name":"Seed 1.6 Flash","provider":"ByteDance","description":"Seed 1.6 Flash is a reasoning model from ByteDance in the Seed 1.6 family. It has no scored result.","logoUrl":null,"releaseDate":"2025-06-01","contextWindowTokens":262144,"modalities":["image","text","video"],"openWeights":false,"details":{"apiId":"bytedance-seed/seed-1.6-flash","maxOutputTokens":32768,"knowledgeCutoff":null,"retiresAt":"2026-11-11","tools":true,"structuredOutput":true,"cacheReadPerMillion":null,"weightsUrl":null},"reasoning":true,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":0.075,"outputPerMillion":0.3,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":56,"timeToFirstToken":0.55,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"seed20code","providerId":"bytedance","slug":"bytedance-seed-2-0-code","name":"Seed 2.0 Code","provider":"ByteDance","description":"Seed 2.0 Code is a model from ByteDance. It has no scored result.","logoUrl":null,"releaseDate":"2026-08-12T16:05:01.000Z","contextWindowTokens":262144,"modalities":["text","image","video"],"openWeights":null,"details":{"apiId":"bytedance-seed/seed-2.0-code","maxOutputTokens":131072,"knowledgeCutoff":null,"retiresAt":"2026-11-11","tools":true,"structuredOutput":true,"cacheReadPerMillion":null,"weightsUrl":null},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":0.5,"outputPerMillion":3,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":56,"timeToFirstToken":0.722,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"seedoss36binstruct","providerId":"bytedance","slug":"bytedance-seed-oss-36b-instruct","name":"Seed OSS 36B Instruct","provider":"ByteDance","description":"Seed OSS 36B Instruct is a model from ByteDance. It has 2 scored results in 2 categories. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":42.81242401336194,"multilingual":null,"instruction":null,"math":48.190188963548046},"capabilityCounts":{"knowledge":1,"math":1},"benchmarks":[{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":71.52777777777779,"unit":"percent","family":"gpqa-diamond","index":42.81242401336194,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.7411232709794677,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":67.5,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":48.190188963548046,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":6.324056203789373,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":25,"benchmarkCount":2,"categoryCount":2,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"seed-2-0-lite","providerId":"bytedance","slug":"seed-2-0-lite","name":"Seed-2.0-Lite","provider":"ByteDance","description":"Seed-2.0-Lite is a standard model from ByteDance in the Seed 2.0 family. It has no scored result.","logoUrl":null,"releaseDate":"2026-02-14","contextWindowTokens":262144,"modalities":["text","image","video"],"openWeights":false,"details":{"apiId":"bytedance-seed/seed-2.0-lite","maxOutputTokens":131072,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":null,"weightsUrl":null},"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":0.25,"outputPerMillion":2,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":38,"timeToFirstToken":2.3445,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[{"host":"volcengine","name":"Volcengine Ark","kind":"maker","url":"https://www.volcengine.com/docs/82379/1330310","inputPerMillion":0.08906,"outputPerMillion":0.53436,"cacheReadPerMillion":0.01781,"contextTokens":256000,"maxOutputTokens":131072,"deprecated":false},{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":0.25,"outputPerMillion":2,"cacheReadPerMillion":null,"contextTokens":262144,"maxOutputTokens":131072,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"seed-2-0-mini","providerId":"bytedance","slug":"seed-2-0-mini","name":"Seed-2.0-Mini","provider":"ByteDance","description":"Seed-2.0-Mini is a standard model from ByteDance in the Seed 2.0 family. It has no scored result.","logoUrl":null,"releaseDate":"2026-02-14","contextWindowTokens":262144,"modalities":["text","image","video"],"openWeights":false,"details":{"apiId":"bytedance-seed/seed-2.0-mini","maxOutputTokens":131072,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":null,"weightsUrl":null},"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":0.09999999999999999,"outputPerMillion":0.39999999999999997,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":101,"timeToFirstToken":0.405,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[{"host":"volcengine","name":"Volcengine Ark","kind":"maker","url":"https://www.volcengine.com/docs/82379/1330310","inputPerMillion":0.02969,"outputPerMillion":0.29687,"cacheReadPerMillion":0.00594,"contextTokens":256000,"maxOutputTokens":131072,"deprecated":false},{"host":"deepinfra","name":"Deep Infra","kind":"host","url":"https://deepinfra.com/models","inputPerMillion":0.1,"outputPerMillion":0.4,"cacheReadPerMillion":0.02,"contextTokens":256000,"maxOutputTokens":32000,"deprecated":false},{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":0.1,"outputPerMillion":0.4,"cacheReadPerMillion":null,"contextTokens":262144,"maxOutputTokens":131072,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"seedrealtime","providerId":"bytedance","slug":"seedrealtime","name":"SeedRealtime","provider":"ByteDance","description":"SeedRealtime is a standard model from ByteDance. It has no scored result.","logoUrl":null,"releaseDate":"2026-08-05","contextWindowTokens":null,"modalities":["text"],"openWeights":false,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"semif-qwen3-5-4b","providerId":"theodore-lee-theoleecj","slug":"semif-qwen3-5-4b","name":"SemIf Qwen3.5-4B","provider":"Theodore Lee (TheoLeeCJ)","description":"SemIf Qwen3.5-4B is a standard model from Theodore Lee (TheoLeeCJ). It has 2 scored results in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":89.97824321292177,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"reasoning":2},"benchmarks":[{"benchmarkId":"jevBench15","benchmarkName":"jevBench15","category":"reasoning","score":68.66437558511035,"unit":"percent","family":"jevBench15","index":93.05507813806454,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"jevBench14","benchmarkName":"jevBench14","category":"reasoning","score":47.69091838825422,"unit":"percent","family":"jevBench14","index":86.90140828777899,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":15,"benchmarkCount":2,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"decision-index-0-3-sieve-2b","providerId":"sthanika-ai","slug":"decision-index-0-3-sieve-2b","name":"Sieve 2B","provider":"sthanika-ai","description":"Sieve 2B is a model from sthanika-ai. It has no scored result.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"decision-index-0-3-sieve-9b","providerId":"sthanika-ai","slug":"decision-index-0-3-sieve-9b","name":"Sieve 9B","provider":"sthanika-ai","description":"Sieve 9B is a model from sthanika-ai. It has no scored result.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"simplejev-qwen3-5-0-8b","providerId":"featherless-ai","slug":"simplejev-qwen3-5-0-8b","name":"SimpleJev Qwen3.5-0.8B","provider":"Featherless AI","description":"SimpleJev Qwen3.5-0.8B is a standard model from Featherless AI. It has 2 scored results in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":51.85824702341582,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"reasoning":2},"benchmarks":[{"benchmarkId":"jevBench14","benchmarkName":"jevBench14","category":"reasoning","score":7.459762136724042,"unit":"percent","family":"jevBench14","index":49.8229642502484,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"jevBench15","benchmarkName":"jevBench15","category":"reasoning","score":3.995571708038271,"unit":"percent","family":"jevBench15","index":53.89352979658325,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":15,"benchmarkCount":2,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"simplejev-qwen3-6-35b-a3b","providerId":"featherless-ai","slug":"simplejev-qwen3-6-35b-a3b","name":"SimpleJev Qwen3.6-35B-A3B","provider":"Featherless AI","description":"SimpleJev Qwen3.6-35B-A3B is a standard model from Featherless AI. It has 2 scored results in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":58.667508287559855,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"reasoning":2},"benchmarks":[{"benchmarkId":"jevBench14","benchmarkName":"jevBench14","category":"reasoning","score":24.86157915547749,"unit":"percent","family":"jevBench14","index":65.86108889292893,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"jevBench15","benchmarkName":"jevBench15","category":"reasoning","score":0,"unit":"percent","family":"jevBench15","index":51.47392768219078,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":15,"benchmarkCount":2,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"simplejev-qwen3-8-27b","providerId":"featherless-ai","slug":"simplejev-qwen3-8-27b","name":"SimpleJev Qwen3.8-27B","provider":"Featherless AI","description":"SimpleJev Qwen3.8-27B is a standard model from Featherless AI. It has 2 scored results in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":64.15644847023069,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"reasoning":2},"benchmarks":[{"benchmarkId":"jevBench14","benchmarkName":"jevBench14","category":"reasoning","score":34.56619633355993,"unit":"percent","family":"jevBench14","index":74.80520431167238,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"jevBench15","benchmarkName":"jevBench15","category":"reasoning","score":3.3584255994370715,"unit":"percent","family":"jevBench15","index":53.50769262878898,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":15,"benchmarkCount":2,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"skyfall36bv2","providerId":"thedrummer","slug":"thedrummer-skyfall-36b-v2","name":"Skyfall 36B V2","provider":"TheDrummer","description":"Skyfall 36B V2 is a model from TheDrummer. It has no scored result.","logoUrl":null,"releaseDate":"2025-03-10T19:56:06.000Z","contextWindowTokens":32768,"modalities":["text"],"openWeights":true,"details":{"apiId":"thedrummer/skyfall-36b-v2","maxOutputTokens":29491,"knowledgeCutoff":"2024-06-30","retiresAt":null,"tools":false,"structuredOutput":true,"cacheReadPerMillion":0.25,"weightsUrl":"https://huggingface.co/TheDrummer/Skyfall-36B-v2"},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":0.55,"outputPerMillion":0.7999999999999999,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":41,"timeToFirstToken":0.232,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":0.55,"outputPerMillion":0.8,"cacheReadPerMillion":0.25,"contextTokens":32768,"maxOutputTokens":29491,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"slam-omni","providerId":"x-lance","slug":"slam-omni","name":"SLAM-Omni","provider":"X-LANCE","description":"SLAM-Omni is a standard model from X-LANCE. It has no scored result.","logoUrl":null,"releaseDate":"2024-12-20","contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"smalljev","providerId":"aditya-ishesatoshi","slug":"smalljev","name":"smalljev semantic-v9","provider":"Aditya (isHeSatoshi)","description":"smalljev semantic-v9 is a standard model from Aditya (isHeSatoshi). It has 2 scored results in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":52.903264980914756,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"reasoning":2},"benchmarks":[{"benchmarkId":"jevBench14","benchmarkName":"jevBench14","category":"reasoning","score":12.308142930422852,"unit":"percent","family":"jevBench14","index":54.291401965730714,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"jevBench15","benchmarkName":"jevBench15","category":"reasoning","score":0.06803548717120994,"unit":"percent","family":"jevBench15","index":51.5151279960988,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":15,"benchmarkCount":2,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"smaugagentic","providerId":"unknown","slug":"unknown-smaug-agentic","name":"Smaug Agentic","provider":"Unknown","description":"Smaug Agentic is a model. It has 6 scored results in 6 categories. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":75.83169665458435,"coding":70.81206894715858,"reasoning":71.84323423721133,"multimodal":null,"knowledge":69.95007177276959,"multilingual":null,"instruction":70.23413695838416,"math":56.735050503044896},"capabilityCounts":{"reasoning":1,"knowledge":1,"math":1,"coding":1,"instruction":1,"agentic":1},"benchmarks":[{"benchmarkId":"livebench:livebench-reasoning:overall","benchmarkName":"LiveBench Reasoning","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":90.274,"unit":"percent","family":"livebench","index":78.56372990355365,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Reasoning","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.18924455445544555,"latency":null},{"benchmarkId":"livebench:livebench-language:overall","benchmarkName":"LiveBench Language","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":84.35666666666667,"unit":"percent","family":"livebench:livebench-language:overall","index":69.95007177276959,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Language","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.139044,"latency":null},{"benchmarkId":"livebench:livebench-math:overall","benchmarkName":"LiveBench Mathematics","category":"math","domain":null,"eligible":true,"ranked":true,"score":83.9195,"unit":"percent","family":"livebench:livebench-math:overall","index":56.735050503044896,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Mathematics","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.17236411609498678,"latency":null},{"benchmarkId":"livebench:livebench-coding:overall","benchmarkName":"LiveBench Coding","category":"coding","domain":null,"eligible":true,"ranked":true,"score":82.471,"unit":"percent","family":"livebench:livebench-coding:overall","index":70.81206894715858,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Coding","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.058345299145299144,"latency":null},{"benchmarkId":"livebench:livebench-data-analysis:overall","benchmarkName":"LiveBench Data Analysis","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":79.89833333333333,"unit":"percent","family":"livebench","index":65.12273857086902,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Data%20Analysis","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.3312293333333333,"latency":null},{"benchmarkId":"livebench:livebench-instruction-following:overall","benchmarkName":"LiveBench Instruction Following","category":"instruction","domain":null,"eligible":true,"ranked":true,"score":70.99175,"unit":"percent","family":"livebench:livebench-instruction-following:overall","index":70.23413695838416,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=IF","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.1048895,"latency":null},{"benchmarkId":"livebench:livebench-agentic-coding:overall","benchmarkName":"LiveBench Agentic Coding","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":64.64666666666666,"unit":"percent","family":"livebench:livebench-agentic-coding:overall","index":75.83169665458435,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Agentic%20Coding","measuredAt":"2026-06-25","stderr":null,"costPerTest":1.807226388888889,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":85,"benchmarkCount":6,"categoryCount":6,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:livebench","name":"LiveBench","url":"https://livebench.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"smaugflash","providerId":"unknown","slug":"unknown-smaug-flash","name":"Smaug Flash","provider":"Unknown","description":"Smaug Flash is a model. It has 6 scored results in 6 categories. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":72.13351593019087,"coding":63.254845192367796,"reasoning":68.72129228291465,"multimodal":null,"knowledge":64.13469474340991,"multilingual":null,"instruction":66.72646927210971,"math":65.29910713173965},"capabilityCounts":{"math":1,"reasoning":1,"knowledge":1,"coding":1,"instruction":1,"agentic":1},"benchmarks":[{"benchmarkId":"livebench:livebench-math:overall","benchmarkName":"LiveBench Mathematics","category":"math","domain":null,"eligible":true,"ranked":true,"score":90.06424999999999,"unit":"percent","family":"livebench:livebench-math:overall","index":65.29910713173965,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Mathematics","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.005437994722955145,"latency":null},{"benchmarkId":"livebench:livebench-reasoning:overall","benchmarkName":"LiveBench Reasoning","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":86.23075,"unit":"percent","family":"livebench","index":73.32596639863408,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Reasoning","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.008641089108910892,"latency":null},{"benchmarkId":"livebench:livebench-language:overall","benchmarkName":"LiveBench Language","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":79.214,"unit":"percent","family":"livebench:livebench-language:overall","index":64.13469474340991,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Language","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.0062180000000000004,"latency":null},{"benchmarkId":"livebench:livebench-data-analysis:overall","benchmarkName":"LiveBench Data Analysis","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":79.12166666666667,"unit":"percent","family":"livebench","index":64.1166181671952,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Data%20Analysis","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.008366666666666666,"latency":null},{"benchmarkId":"livebench:livebench-coding:overall","benchmarkName":"LiveBench Coding","category":"coding","domain":null,"eligible":true,"ranked":true,"score":77.0975,"unit":"percent","family":"livebench:livebench-coding:overall","index":63.254845192367796,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Coding","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.006227350427350426,"latency":null},{"benchmarkId":"livebench:livebench-instruction-following:overall","benchmarkName":"LiveBench Instruction Following","category":"instruction","domain":null,"eligible":true,"ranked":true,"score":69.25825,"unit":"percent","family":"livebench:livebench-instruction-following:overall","index":66.72646927210971,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=IF","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.007252,"latency":null},{"benchmarkId":"livebench:livebench-agentic-coding:overall","benchmarkName":"LiveBench Agentic Coding","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":61.06066666666667,"unit":"percent","family":"livebench:livebench-agentic-coding:overall","index":72.13351593019087,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Agentic%20Coding","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.07808611111111112,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":85,"benchmarkCount":6,"categoryCount":6,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:livebench","name":"LiveBench","url":"https://livebench.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"smaugmini","providerId":"unknown","slug":"unknown-smaug-mini","name":"Smaug Mini","provider":"Unknown","description":"Smaug Mini is a model. It has 6 scored results in 6 categories. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":71.87294510019548,"coding":60.90828748605564,"reasoning":66.30422835177119,"multimodal":null,"knowledge":61.63032833278858,"multilingual":null,"instruction":76.1522512159159,"math":64.85102637888147},"capabilityCounts":{"math":1,"reasoning":1,"knowledge":1,"coding":1,"instruction":1,"agentic":1},"benchmarks":[{"benchmarkId":"livebench:livebench-math:overall","benchmarkName":"LiveBench Mathematics","category":"math","domain":null,"eligible":true,"ranked":true,"score":89.74275,"unit":"percent","family":"livebench:livebench-math:overall","index":64.85102637888147,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Mathematics","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.014894195250659631,"latency":null},{"benchmarkId":"livebench:livebench-reasoning:overall","benchmarkName":"LiveBench Reasoning","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":82.80775,"unit":"percent","family":"livebench","index":68.89169582982441,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Reasoning","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.018601980198019803,"latency":null},{"benchmarkId":"livebench:livebench-data-analysis:overall","benchmarkName":"LiveBench Data Analysis","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":78.813,"unit":"percent","family":"livebench","index":63.716760873717966,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Data%20Analysis","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.027478,"latency":null},{"benchmarkId":"livebench:livebench-language:overall","benchmarkName":"LiveBench Language","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":76.99933333333333,"unit":"percent","family":"livebench:livebench-language:overall","index":61.63032833278858,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Language","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.01800466666666667,"latency":null},{"benchmarkId":"livebench:livebench-coding:overall","benchmarkName":"LiveBench Coding","category":"coding","domain":null,"eligible":true,"ranked":true,"score":75.429,"unit":"percent","family":"livebench:livebench-coding:overall","index":60.90828748605564,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Coding","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.010233333333333334,"latency":null},{"benchmarkId":"livebench:livebench-instruction-following:overall","benchmarkName":"LiveBench Instruction Following","category":"instruction","domain":null,"eligible":true,"ranked":true,"score":73.9165,"unit":"percent","family":"livebench:livebench-instruction-following:overall","index":76.1522512159159,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=IF","measuredAt":"2026-06-25","stderr":null,"costPerTest":0.014252,"latency":null},{"benchmarkId":"livebench:livebench-agentic-coding:overall","benchmarkName":"LiveBench Agentic Coding","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":60.808,"unit":"percent","family":"livebench:livebench-agentic-coding:overall","index":71.87294510019548,"aboveFit":false,"run":null,"source":"LiveBench","sourceUrl":"https://livebench.ai/#/release/2026-06-25?cats=Agentic%20Coding","measuredAt":"2026-06-25","stderr":null,"costPerTest":1.0671111111111111,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":85,"benchmarkCount":6,"categoryCount":6,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:livebench","name":"LiveBench","url":"https://livebench.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"smaugnemotron35nvfp4","providerId":"unknown","slug":"unknown-smaug-nemotron-3-5-nvfp4","name":"Smaug Nemotron 3.5 Nvfp4","provider":"Unknown","description":"Smaug Nemotron 3.5 Nvfp4 is a model. It has 3 scored results in 3 categories. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":57.60598549637658,"coding":60.468189366100134,"reasoning":52.42964446529301,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"reasoning":1,"coding":1,"agentic":1},"benchmarks":[{"benchmarkId":"livebench:livebench-reasoning:overall","benchmarkName":"LiveBench Reasoning","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":75.1,"unit":"percent","family":"livebench","index":58.90681444602579,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Reasoning","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.012576381188118812,"latency":null},{"benchmarkId":"livebench:livebench-coding-2:overall","benchmarkName":"LiveBench Coding 2.0","category":"coding","domain":null,"eligible":true,"ranked":true,"score":72.2,"unit":"percent","family":"livebench:livebench-coding-2:overall","index":60.468189366100134,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Coding","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.09419655172413793,"latency":null},{"benchmarkId":"livebench:livebench-research:overall","benchmarkName":"LiveBench Research","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":67.8,"unit":"percent","family":"livebench-2-agentic","index":68.17371981797719,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Research","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.09319999999999999,"latency":null},{"benchmarkId":"livebench:livebench-data-analysis:overall","benchmarkName":"LiveBench Data Analysis","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":65.1,"unit":"percent","family":"livebench","index":45.95247448456023,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Data%20Analysis","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.016672786666666665,"latency":null},{"benchmarkId":"livebench:livebench-design:overall","benchmarkName":"LiveBench Design","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":62.7,"unit":"percent","family":"livebench-2-agentic","index":64.44393123388286,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Design","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.028003999999999998,"latency":null},{"benchmarkId":"livebench:livebench-browser:overall","benchmarkName":"LiveBench Browser Use","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":44,"unit":"percent","family":"livebench-2-agentic","index":50.76803975887031,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Browser","measuredAt":"2026-10-14","stderr":null,"costPerTest":0.216188,"latency":null},{"benchmarkId":"livebench:livebench-long-horizon:overall","benchmarkName":"LiveBench Long Agentic","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":19.6,"unit":"percent","family":"livebench-2-agentic","index":32.92356104281116,"aboveFit":false,"run":"max effort","source":"LiveBench","sourceUrl":"https://livebench.ai/#/?cats=Long%20Agentic","measuredAt":"2026-10-14","stderr":null,"costPerTest":1.0770689655172414,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":55.00000000000001,"benchmarkCount":3,"categoryCount":3,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:livebench","name":"LiveBench","url":"https://livebench.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"smollm2135m","providerId":"unknown","slug":"unknown-smollm2-135m","name":"SmolLM2 135M","provider":"Unknown","description":"SmolLM2 135M is a model. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":5,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-dutch:overall","benchmarkName":"EuroEval Dutch","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":9.867777777777778,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/dutch","measuredAt":null,"stderr":0.4197603666761534,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-french:overall","benchmarkName":"EuroEval French","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":9.75125,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/french","measuredAt":null,"stderr":0.5589247489600009,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-spanish:overall","benchmarkName":"EuroEval Spanish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":9.085000000000003,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/spanish","measuredAt":null,"stderr":0.4063730582851181,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-italian:overall","benchmarkName":"EuroEval Italian","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":8.453750000000001,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/italian","measuredAt":null,"stderr":0.3405831396296652,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":7.2612499999999995,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.5710954386089946,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-portuguese:overall","benchmarkName":"EuroEval Portuguese","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":6.813750000000001,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/portuguese","measuredAt":null,"stderr":0.5248303297257124,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":5.792500000000001,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":0.381403657559809,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":4.397142857142857,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.47309445922240384,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"smollm2135minstruct","providerId":"unknown","slug":"unknown-smollm2-135m-instruct","name":"SmolLM2 135M Instruct","provider":"Unknown","description":"SmolLM2 135M Instruct is a model. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":5,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-french:overall","benchmarkName":"EuroEval French","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":8.541250000000002,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/french","measuredAt":null,"stderr":0.47658780670512335,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-dutch:overall","benchmarkName":"EuroEval Dutch","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":7.477777777777779,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/dutch","measuredAt":null,"stderr":0.31970086326957053,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-spanish:overall","benchmarkName":"EuroEval Spanish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":6.811250000000001,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/spanish","measuredAt":null,"stderr":0.3343417039497167,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":6.686250000000001,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.641840614950472,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-italian:overall","benchmarkName":"EuroEval Italian","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":6.4425,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/italian","measuredAt":null,"stderr":0.4198288341693553,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":6.050000000000001,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":0.6044341361968234,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-portuguese:overall","benchmarkName":"EuroEval Portuguese","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":4.765,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/portuguese","measuredAt":null,"stderr":0.4032737128799744,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":4.541428571428573,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.37172241124313204,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"smollm2360m","providerId":"unknown","slug":"unknown-smollm2-360m","name":"SmolLM2 360M","provider":"Unknown","description":"SmolLM2 360M is a model. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":5,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-french:overall","benchmarkName":"EuroEval French","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":17.4575,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/french","measuredAt":null,"stderr":0.6040100371682577,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-dutch:overall","benchmarkName":"EuroEval Dutch","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":14.207777777777778,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/dutch","measuredAt":null,"stderr":0.4913021247438054,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-portuguese:overall","benchmarkName":"EuroEval Portuguese","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":14.067500000000003,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/portuguese","measuredAt":null,"stderr":0.5794596081695427,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-italian:overall","benchmarkName":"EuroEval Italian","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":12.22625,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/italian","measuredAt":null,"stderr":0.5209891433609726,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-spanish:overall","benchmarkName":"EuroEval Spanish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":12.157499999999999,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/spanish","measuredAt":null,"stderr":0.4909318308074961,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":9.77875,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.6592846502080872,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":7.08625,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":0.5510756527011513,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":6.238571428571428,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.4893602654361663,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"smollm2360minstruct","providerId":"unknown","slug":"unknown-smollm2-360m-instruct","name":"SmolLM2 360M Instruct","provider":"Unknown","description":"SmolLM2 360M Instruct is a model. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":5,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-french:overall","benchmarkName":"EuroEval French","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":9.467500000000001,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/french","measuredAt":null,"stderr":0.6533220396557887,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-spanish:overall","benchmarkName":"EuroEval Spanish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":8.862499999999999,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/spanish","measuredAt":null,"stderr":0.3164747225293041,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-italian:overall","benchmarkName":"EuroEval Italian","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":8.82375,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/italian","measuredAt":null,"stderr":0.3886052785282259,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":7.819999999999999,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.5597195168117689,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-dutch:overall","benchmarkName":"EuroEval Dutch","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":7.707777777777778,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/dutch","measuredAt":null,"stderr":0.42922985870619285,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-portuguese:overall","benchmarkName":"EuroEval Portuguese","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":7.353750000000001,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/portuguese","measuredAt":null,"stderr":0.4543384613919451,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":6.515,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":0.6227446808283471,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":6.450000000000001,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.45217840076948507,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"smollm33bbase","providerId":"unknown","slug":"unknown-smollm3-3b-base","name":"SmolLM3 3B Base","provider":"Unknown","description":"SmolLM3 3B Base is a model. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":19.817419742851754,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-french:overall","benchmarkName":"EuroEval French","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":43.15624999999999,"unit":"percent","family":"euroeval","index":30.77333673828568,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/french","measuredAt":null,"stderr":0.5258668438873095,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-portuguese:overall","benchmarkName":"EuroEval Portuguese","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":38.278749999999995,"unit":"percent","family":"euroeval","index":24.15569151906227,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/portuguese","measuredAt":null,"stderr":0.4777862361977373,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-dutch:overall","benchmarkName":"EuroEval Dutch","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":37.36222222222222,"unit":"percent","family":"euroeval","index":22.912174211651397,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/dutch","measuredAt":null,"stderr":0.39417361594375244,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-italian:overall","benchmarkName":"EuroEval Italian","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":36.407500000000006,"unit":"percent","family":"euroeval","index":21.61683583142071,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/italian","measuredAt":null,"stderr":0.4752581535334244,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":33.755,"unit":"percent","family":"euroeval","index":18.0180036542828,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.60615463167413,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":32.18375,"unit":"percent","family":"euroeval","index":15.886178938607813,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":0.3722167547276721,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":31.40142857142857,"unit":"percent","family":"euroeval","index":14.824748767068755,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.5044394747375243,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-spanish:overall","benchmarkName":"EuroEval Spanish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":30.787500000000005,"unit":"percent","family":"euroeval","index":13.991788956579995,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/spanish","measuredAt":null,"stderr":0.37682887362833545,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"smollm33bnothinking","providerId":"unknown","slug":"unknown-smollm3-3b-no-thinking","name":"SmolLM3 3B No Thinking","provider":"Unknown","description":"SmolLM3 3B No Thinking is a model. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":16.174209883353612,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-french:overall","benchmarkName":"EuroEval French","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":37.97625,"unit":"percent","family":"euroeval","index":23.745268622329306,"aboveFit":false,"run":"no-thinking effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/french","measuredAt":null,"stderr":0.4663220453720797,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-italian:overall","benchmarkName":"EuroEval Italian","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":33.16875,"unit":"percent","family":"euroeval","index":17.22259729656477,"aboveFit":false,"run":"no-thinking effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/italian","measuredAt":null,"stderr":0.3325140974455068,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-dutch:overall","benchmarkName":"EuroEval Dutch","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":31.623333333333335,"unit":"percent","family":"euroeval","index":15.125822470142452,"aboveFit":false,"run":"no-thinking effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/dutch","measuredAt":null,"stderr":0.3118681266097287,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":30.661250000000003,"unit":"percent","family":"euroeval","index":13.820496755877386,"aboveFit":false,"run":"no-thinking effort","source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.5678743258855783,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"solarmini4","providerId":"upstage","slug":"upstage-solar-mini-4","name":"Solar Mini 4","provider":"Upstage","description":"Solar Mini 4 is a model from Upstage. It has no scored result.","logoUrl":null,"releaseDate":"2026-09-23T10:45:58.000Z","contextWindowTokens":524288,"modalities":["text"],"openWeights":null,"details":{"apiId":"upstage/solar-mini4","maxOutputTokens":131072,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.005,"weightsUrl":null},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":0.049999999999999996,"outputPerMillion":0.19999999999999998,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":23,"timeToFirstToken":2.583,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":0.05,"outputPerMillion":0.2,"cacheReadPerMillion":0.005,"contextTokens":524288,"maxOutputTokens":131072,"deprecated":false},{"host":"upstage","name":"Upstage","kind":"maker","url":"https://developers.upstage.ai/docs/apis/chat","inputPerMillion":0.1,"outputPerMillion":0.4,"cacheReadPerMillion":0.01,"contextTokens":524288,"maxOutputTokens":131072,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"solar-pro-2","providerId":"upstage","slug":"solar-pro-2","name":"Solar Pro 2","provider":"Upstage","description":"Solar Pro 2 is a reasoning model from Upstage in the Solar family. It has 7 scored results in 4 categories. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":128000,"modalities":["text"],"openWeights":false,"details":null,"reasoning":true,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":23.9574224156381,"coding":null,"reasoning":22.073582735971804,"multimodal":null,"knowledge":29.247129026918767,"multilingual":null,"instruction":20.516637111819147,"math":null},"capabilityCounts":{"knowledge":3,"instruction":1,"agentic":1,"reasoning":2},"benchmarks":[{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":56.1,"unit":"percent","family":"gpqa-diamond","index":28.662487800849473,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaIfBench","benchmarkName":"Artificial Analysis IFBench","category":"instruction","score":33.7,"unit":"percent","family":"ifbench-family","index":20.516637111819147,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"tau2Bench","benchmarkName":"τ²-Bench Tool-Agent-User Evaluation","category":"agentic","score":31.9,"unit":"percent","family":"tau2-bench","index":23.9574224156381,"aboveFit":false,"run":null,"source":"Victor Barres, Honghua Dong, Soham Ray, Xujie Si, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2506.07982","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"omniscienceAccuracy","benchmarkName":"Artificial Analysis Omniscience Accuracy","category":"knowledge","score":16.1,"unit":"percent","family":"omniscienceAccuracy","index":30.98007180998312,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/omniscience","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":3.7,"unit":"percent","family":"hle","index":28.098827469923716,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":0,"unit":"percent","family":"lcr","index":7.298049459170706,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":0,"unit":"percent","family":"critpt","index":36.8491160127729,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":55.00000000000001,"benchmarkCount":7,"categoryCount":4,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"sonar","providerId":"perplexity","slug":"perplexity-sonar","name":"Sonar","provider":"Perplexity","description":"Sonar is a model from Perplexity. It has no scored result.","logoUrl":null,"releaseDate":"2025-01-27T21:36:48.000Z","contextWindowTokens":127072,"modalities":["text","image"],"openWeights":null,"details":{"apiId":"perplexity/sonar","maxOutputTokens":114364,"knowledgeCutoff":null,"retiresAt":null,"tools":false,"structuredOutput":false,"cacheReadPerMillion":null,"weightsUrl":null},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":1,"outputPerMillion":1,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":86,"timeToFirstToken":2.1535,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"sonardeepresearch","providerId":"perplexity","slug":"perplexity-sonar-deep-research","name":"Sonar Deep Research","provider":"Perplexity","description":"Sonar Deep Research is a model from Perplexity. It has no scored result.","logoUrl":null,"releaseDate":"2025-03-07T01:34:06.000Z","contextWindowTokens":128000,"modalities":["text"],"openWeights":null,"details":{"apiId":"perplexity/sonar-deep-research","maxOutputTokens":115200,"knowledgeCutoff":null,"retiresAt":null,"tools":false,"structuredOutput":false,"cacheReadPerMillion":null,"weightsUrl":null},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":2,"outputPerMillion":8,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":18,"timeToFirstToken":92.716,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"sonarpro","providerId":"perplexity","slug":"perplexity-sonar-pro","name":"Sonar Pro","provider":"Perplexity","description":"Sonar Pro is a model from Perplexity. It has no scored result.","logoUrl":null,"releaseDate":"2025-03-07T01:53:43.000Z","contextWindowTokens":200000,"modalities":["text","image"],"openWeights":null,"details":{"apiId":"perplexity/sonar-pro","maxOutputTokens":8000,"knowledgeCutoff":null,"retiresAt":null,"tools":false,"structuredOutput":false,"cacheReadPerMillion":null,"weightsUrl":null},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":3,"outputPerMillion":15,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":97,"timeToFirstToken":1.57,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"sonarprosearch","providerId":"perplexity","slug":"perplexity-sonar-pro-search","name":"Sonar Pro Search","provider":"Perplexity","description":"Sonar Pro Search is a model from Perplexity. It has no scored result.","logoUrl":null,"releaseDate":"2025-10-30T19:59:26.000Z","contextWindowTokens":200000,"modalities":["text","image"],"openWeights":null,"details":{"apiId":"perplexity/sonar-pro-search","maxOutputTokens":8000,"knowledgeCutoff":null,"retiresAt":null,"tools":false,"structuredOutput":true,"cacheReadPerMillion":null,"weightsUrl":null},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":3,"outputPerMillion":15,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":39,"timeToFirstToken":11.562,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"sonarreasoningpro","providerId":"perplexity","slug":"perplexity-sonar-reasoning-pro","name":"Sonar Reasoning Pro","provider":"Perplexity","description":"Sonar Reasoning Pro is a model from Perplexity. It has no scored result.","logoUrl":null,"releaseDate":"2025-03-07T02:08:28.000Z","contextWindowTokens":128000,"modalities":["text","image"],"openWeights":null,"details":{"apiId":"perplexity/sonar-reasoning-pro","maxOutputTokens":115200,"knowledgeCutoff":null,"retiresAt":null,"tools":false,"structuredOutput":false,"cacheReadPerMillion":null,"weightsUrl":null},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":2,"outputPerMillion":8,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":6,"timeToFirstToken":38.0295,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"soofi-s-30b-a3b","providerId":"soofi-project","slug":"soofi-s-30b-a3b","name":"Soofi S 30B-A3B","provider":"Soofi Project","description":"Soofi S 30B-A3B is a standard model from Soofi Project in the Soofi S family. It has 3 scored results in 2 categories. It has no index.","logoUrl":null,"releaseDate":"2026-07-10","contextWindowTokens":1000000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":15.033796178435438,"reasoning":null,"multimodal":null,"knowledge":13.126485417233653,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"coding":1,"knowledge":2},"benchmarks":[{"benchmarkId":"gsm8k","benchmarkName":"Grade School Math 8K","category":"math","score":86.1,"unit":"percent","family":"gsm8k","index":null,"aboveFit":false,"run":null,"source":"Karl Cobbe and colleagues (OpenAI)","sourceUrl":"https://arxiv.org/abs/2110.14168","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"bbh","benchmarkName":"BIG-Bench Hard","category":"reasoning","score":78.8,"unit":"percent","family":"bbh","index":null,"aboveFit":false,"run":null,"source":"Mirac Suzgun, Nathan Scales, Nathanael Schärli, Sebastian Gehrmann, Yi Tay, Hyung Won Chung, Aakanksha Chowdhery, Quoc V. Le, Ed H. Chi, Denny Zhou, Jason Wei","sourceUrl":"https://arxiv.org/abs/2210.09261","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"humaneval","benchmarkName":"Evaluating Large Language Models Trained on Code","category":"coding","score":73.8,"unit":"percent","family":"humaneval","index":15.033796178435438,"aboveFit":false,"run":null,"source":"Mark Chen, Jerry Tworek, Heewoo Jun, Qiming Yuan, Henrique Ponde de Oliveira Pinto, Jared Kaplan, Harri Edwards, Yuri Burda, Nicholas Joseph, Greg Brockman, Alex Ray, Raul Puri, Gretchen Krueger, Michael Petrov, Heidy Khlaaf, Girish Sastry, Pamela Mishkin, Brooke Chan, Scott Gray, Nick Ryder, Mikhail Pavlov, Alethea Power, Lukasz Kaiser, Mohammad Bavarian, Clemens Winter, Philippe Tillet, Felipe Petroski Such, Dave Cummings, Matthias Plappert, Fotios Chantzis, Elizabeth Barnes, Ariel Herbert-Voss, William Hebgen Guss, Alex Nichol, Alex Paino, Nikolas Tezak, Jie Tang, Igor Babuschkin, Suchir Balaji, Shantanu Jain, William Saunders, Christopher Hesse, Andrew N. Carr, Jan Leike, Josh Achiam, Vedant Misra, Evan Morikawa, Alec Radford, Matthew Knight, Miles Brundage, Mira Murati, Katie Mayer, Peter Welinder, Bob McGrew, Dario Amodei, Sam McCandlish, Ilya Sutskever, Wojciech Zaremba","sourceUrl":"https://arxiv.org/abs/2107.03374","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"agieval","benchmarkName":"AGIEval","category":"knowledge","score":66.9,"unit":"percent","family":"agieval","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"drop","benchmarkName":"Discrete Reasoning Over Paragraphs","category":"reasoning","score":66.5,"unit":"percent","family":"drop","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmluPro","benchmarkName":"Massive Multitask Language Understanding Professional","category":"knowledge","score":51.4,"unit":"percent","family":"mmlu-pro","index":9.238576005099105,"aboveFit":false,"run":null,"source":"Yubo Wang, Xueguang Ma, Ge Zhang, Yuansheng Ni, Abhranil Chandra, Shiguang Guo, Weiming Ren, Aaran Arulraj, Xuan He, Ziyan Jiang, Tianle Li, Max Ku, Kai Wang, Alex Zhuang, Rongqi Fan, Xiang Yue, Wenhu Chen","sourceUrl":"https://arxiv.org/abs/2406.01574","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gpqa","benchmarkName":"Graduate-Level Google-Proof Q&A","category":"knowledge","score":43.4,"unit":"percent","family":"gpqa-diamond","index":17.0143948293682,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gpqaDiamond","benchmarkName":"GPQA Diamond","category":"knowledge","score":43.4,"unit":"percent","family":"gpqa-diamond","index":17.0143948293682,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":35,"benchmarkCount":3,"categoryCount":2,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"sparkx26","providerId":"unknown","slug":"unknown-spark-x2-6","name":"Spark X2.6","provider":"Unknown","description":"Spark X2.6 is a model. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":57.286636683769544,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"agentic":1},"benchmarks":[{"benchmarkId":"lmarena:agent-task-outcome:overall","benchmarkName":"Agent Arena task outcome","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":-10.160949312642565,"unit":"index","family":"agent-arena","index":58.176908030130335,"aboveFit":false,"run":null,"source":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset","measuredAt":"2026-11-11","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lmarena:agent-steerability:overall","benchmarkName":"Agent Arena steerability","category":"agentic","domain":null,"eligible":true,"ranked":true,"score":-12.32664660398296,"unit":"index","family":"agent-arena","index":56.39636533740876,"aboveFit":false,"run":null,"source":"LMArena","sourceUrl":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset","measuredAt":"2026-11-11","stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":20,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:lmarena","name":"LMArena","url":"https://huggingface.co","license":"CC BY 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"spark-s1-4b-v6","providerId":"abhishek-rai-abhishek085","slug":"spark-s1-4b-v6","name":"spark-s1-4b-v6","provider":"Abhishek Rai (abhishek085)","description":"spark-s1-4b-v6 is a standard model from Abhishek Rai (abhishek085). It has 2 scored results in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":88.41151640320959,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"reasoning":2},"benchmarks":[{"benchmarkId":"jevBench15","benchmarkName":"jevBench15","category":"reasoning","score":68.15818403751406,"unit":"percent","family":"jevBench15","index":92.74854324685296,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"jevBench14","benchmarkName":"jevBench14","category":"reasoning","score":44.623631936050636,"unit":"percent","family":"jevBench14","index":84.07448955956619,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":15,"benchmarkCount":2,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"standardone-8b","providerId":"standard-thinking","slug":"standardone-8b","name":"Standard One 8B","provider":"Standard Thinking","description":"Standard One 8B is a standard model from Standard Thinking. It has 2 scored results in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":75.07562343489361,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"reasoning":2},"benchmarks":[{"benchmarkId":"jevBench15","benchmarkName":"jevBench15","category":"reasoning","score":47.79069264973117,"unit":"percent","family":"jevBench15","index":80.41458234518619,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"jevBench14","benchmarkName":"jevBench14","category":"reasoning","score":29.066687887780347,"unit":"percent","family":"jevBench14","index":69.73666452460104,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":15,"benchmarkCount":2,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"step-3-5-flash","providerId":"stepfun","slug":"step-3-5-flash","name":"Step 3.5 Flash","provider":"StepFun","description":"Step 3.5 Flash is a standard model from StepFun. It has 3 scored results in 2 categories. It has no index.","logoUrl":null,"releaseDate":"2026-02-01","contextWindowTokens":262144,"modalities":["text"],"openWeights":true,"details":{"apiId":"stepfun/step-3.5-flash","maxOutputTokens":65536,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":false,"cacheReadPerMillion":null,"weightsUrl":"https://huggingface.co/stepfun-ai/Step-3.5-Flash"},"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":55.72930996590039,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":51.80471368926614},"capabilityCounts":{"math":1,"coding":2},"benchmarks":[{"benchmarkId":"aime2025","benchmarkName":"American Invitational Mathematics Examination 2025","category":"math","score":97.3,"unit":"percent","family":"aime2025","index":51.80471368926614,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"liveCodeBenchV6","benchmarkName":"LiveCodeBench v6","category":"coding","score":86.4,"unit":"percent","family":"liveCodeBenchV6","index":55.35640861522245,"aboveFit":false,"run":null,"source":"LiveCodeBench maintainers","sourceUrl":"https://github.com/LiveCodeBench/LiveCodeBench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"sweVerified","benchmarkName":"Software Engineering Benchmark Verified","category":"coding","score":74.4,"unit":"percent","family":"swe-bench-verified","index":56.102211316578334,"aboveFit":false,"run":null,"source":"Carlos E. Jimenez, John Yang, Alexander Wettig, Shunyu Yao, Kexin Pei, Ofir Press, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2310.06770","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":0.09999999999999999,"outputPerMillion":0.3,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":42,"timeToFirstToken":0.994,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":{"elo":1402.6613526441433,"eloUpper":1406.180438069871,"eloLower":1399.1422672184153,"votes":59777,"rank":154,"rankUpper":null,"rankLower":null,"arenaName":"Step-3.5-Flash","organization":"stepfun","updatedAt":"2026-11-09"},"quality":{"coveragePercent":30,"benchmarkCount":3,"categoryCount":2,"confidence":"unranked","state":"partial"},"hosts":[{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":0.1,"outputPerMillion":0.3,"cacheReadPerMillion":null,"contextTokens":262144,"maxOutputTokens":65536,"deprecated":false},{"host":"siliconflow","name":"SiliconFlow","kind":"host","url":"https://cloud.siliconflow.com/models","inputPerMillion":0.1,"outputPerMillion":0.3,"cacheReadPerMillion":null,"contextTokens":262000,"maxOutputTokens":66000,"deprecated":false},{"host":"stepfun","name":"StepFun","kind":"maker","url":"https://platform.stepfun.ai/docs/en/overview/concept","inputPerMillion":0.1,"outputPerMillion":0.3,"cacheReadPerMillion":0.02,"contextTokens":256000,"maxOutputTokens":256000,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"step-audio","providerId":"stepfun","slug":"step-audio","name":"Step-Audio","provider":"StepFun","description":"Step-Audio is a standard model from StepFun. It has no scored result.","logoUrl":null,"releaseDate":"2025-02-17","contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"step-audio-chat-130b","providerId":"stepfun","slug":"step-audio-chat-130b","name":"Step-Audio-Chat 130B","provider":"StepFun","description":"Step-Audio-Chat 130B is a standard model from StepFun. It has no scored result.","logoUrl":null,"releaseDate":"2025-02-17","contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"subq-snap-1-1","providerId":"subquadratic","slug":"subq-snap-1-1","name":"SubQ Snap 1.1","provider":"Subquadratic","description":"SubQ Snap 1.1 is a model from Subquadratic in the SubQ Snap family. It has no scored result.","logoUrl":null,"releaseDate":"2026-10-22","contextWindowTokens":384000,"modalities":["text"],"openWeights":false,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":0.04,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"summarize","providerId":"scaledown","slug":"scaledown-summarize-summarize","name":"Summarize","provider":"ScaleDown","description":"Summarize is a model from ScaleDown. It has no scored result.","logoUrl":null,"releaseDate":"2026-10-13T12:09:20.000Z","contextWindowTokens":1000000,"modalities":["text","image","file"],"openWeights":null,"details":{"apiId":"scaledown/summarize","maxOutputTokens":4000,"knowledgeCutoff":null,"retiresAt":null,"tools":false,"structuredOutput":false,"cacheReadPerMillion":null,"weightsUrl":null},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":0.04,"outputPerMillion":0,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":136,"timeToFirstToken":0.5,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"decision-index-rune-26b-a4b-v3","providerId":"surogate","slug":"decision-index-rune-26b-a4b-v3","name":"Surogate Rune 26B-A4B v3","provider":"Surogate","description":"Surogate Rune 26B-A4B v3 is a model from Surogate. It has no scored result.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"surogate-rune-26b-a4b-v3-rtxpro6000-a2","providerId":"surogate","slug":"surogate-rune-26b-a4b-v3-rtxpro6000-a2","name":"Surogate Rune 26B-A4B v3 (RTX PRO 6000)","provider":"Surogate","description":"Surogate Rune 26B-A4B v3 (RTX PRO 6000) is a standard model from Surogate. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":91.72629506621706,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"reasoning":1},"benchmarks":[{"benchmarkId":"jevBench15","benchmarkName":"jevBench15","category":"reasoning","score":66.47011066179378,"unit":"percent","family":"jevBench15","index":91.72629506621706,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":15,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"swanone","providerId":"blockbrain","slug":"swanone","name":"swanOne","provider":"blockbrain","description":"swanOne is a standard model from blockbrain. It has 2 scored results in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":76.79497878072974,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"reasoning":2},"benchmarks":[{"benchmarkId":"jevBench15","benchmarkName":"jevBench15","category":"reasoning","score":46.56179491157818,"unit":"percent","family":"jevBench15","index":79.67039758693868,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"jevBench14","benchmarkName":"jevBench14","category":"reasoning","score":33.60524727984119,"unit":"percent","family":"jevBench14","index":73.91955997452081,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":15,"benchmarkCount":2,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"swe16fast","providerId":"unknown","slug":"unknown-swe-1-6-fast","name":"Swe 1.6 Fast","provider":"Unknown","description":"Swe 1.6 Fast is a model. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":44.85461202566902,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"coding":1},"benchmarks":[{"benchmarkId":"vals-ai:vibe-code:overall","benchmarkName":"Vibe Code Bench v1.1","category":"coding","domain":null,"eligible":true,"ranked":true,"score":7.22,"unit":"percent","family":"vals-ai:vibe-code:overall","index":44.85461202566902,"aboveFit":false,"run":"Devin CLI","source":"Vals AI","sourceUrl":"https://www.vals.ai/benchmarks/vibe-code","measuredAt":"2026-11-09","stderr":2.635,"costPerTest":null,"latency":189.132}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":20,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:vals-ai","name":"Vals AI","url":"https://www.vals.ai","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"sweagentlm32b","providerId":"unknown","slug":"unknown-swe-agent-lm-32b","name":"SWE Agent LM 32B","provider":"Unknown","description":"SWE Agent LM 32B is a model. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":30.877742156376065,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"coding":1},"benchmarks":[{"benchmarkId":"swe-bench:swe-bench-verified:overall","benchmarkName":"SWE-bench Verified","category":"coding","domain":null,"eligible":true,"ranked":true,"score":40.2,"unit":"percent","family":"swe-bench-verified","index":30.877742156376065,"aboveFit":false,"run":"SWE-agent","source":"SWE-bench team","sourceUrl":"https://www.swebench.com/","measuredAt":"2026-02-26","stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":20,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:swe-bench-team","name":"SWE-bench team","url":"https://www.swebench.com","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"swellama","providerId":"unknown","slug":"unknown-swe-llama","name":"SWE Llama","provider":"Unknown","description":"SWE Llama is a model. It has 2 scored results in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":5.209659467686862,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"coding":2},"benchmarks":[{"benchmarkId":"swebench-experiments:swe-bench-verified:overall","benchmarkName":"SWE-bench Verified","category":"coding","domain":null,"eligible":true,"ranked":true,"score":1.4000000000000001,"unit":"percent","family":"swe-bench-verified","index":5,"aboveFit":false,"run":"RAG baseline","source":"SWE-bench team","sourceUrl":"https://github.com/SWE-bench/experiments/tree/main/evaluation/verified","measuredAt":"2026-09-01","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"swebench-experiments:swe-bench-lite:overall","benchmarkName":"SWE-bench Lite","category":"coding","domain":null,"eligible":true,"ranked":true,"score":1.3333333333333335,"unit":"percent","family":"swe-bench-lite","index":5.419318935373724,"aboveFit":false,"run":"RAG baseline","source":"SWE-bench team","sourceUrl":"https://github.com/SWE-bench/experiments/tree/main/evaluation/lite","measuredAt":"2025-09-11","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"swebench-experiments:swe-bench-test:overall","benchmarkName":"SWE-bench (full test split)","category":"coding","domain":null,"eligible":true,"ranked":true,"score":0.6974716652136007,"unit":"percent","family":"swebench-experiments:swe-bench-test:overall","index":null,"aboveFit":false,"run":"RAG baseline","source":"SWE-bench team","sourceUrl":"https://github.com/SWE-bench/experiments/tree/main/evaluation/test","measuredAt":"2025-12-19","stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":20,"benchmarkCount":2,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:swe-bench-team","name":"SWE-bench team","url":"https://github.com","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"swe-1-7","providerId":"cognition","slug":"swe-1-7","name":"SWE-1.7","provider":"Cognition","description":"SWE-1.7 is a reasoning model from Cognition in the SWE family. It has 3 scored results in 2 categories. It has no index.","logoUrl":null,"releaseDate":"2026-07-08","contextWindowTokens":256000,"modalities":["text"],"openWeights":false,"details":null,"reasoning":true,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":68.80683341190269,"coding":65.72435309610114,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"agentic":1,"coding":2},"benchmarks":[{"benchmarkId":"terminalBench21","benchmarkName":"Terminal-Bench 2.1","category":"agentic","score":81.5,"unit":"percent","family":"terminal-bench-2-1","index":68.80683341190269,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"frontierCode","benchmarkName":"FrontierCode 1.1 Main","category":"coding","score":42.3,"unit":"percent","family":"frontierCode","index":67.41108472254832,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"frontierCode2","benchmarkName":"FrontierCode 2.0","category":"coding","score":16.2,"unit":"percent","family":"frontierCode2","index":64.03762146965397,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":40,"benchmarkCount":3,"categoryCount":2,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"swe-2","providerId":"cognition","slug":"swe-2","name":"SWE-2","provider":"Cognition","description":"SWE-2 is a reasoning model from Cognition in the SWE family. It has 4 scored results in 2 categories. It has no index.","logoUrl":null,"releaseDate":"2026-09-10","contextWindowTokens":1000000,"modalities":["text"],"openWeights":false,"details":null,"reasoning":true,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":74.78069089743609,"coding":71.69556825094534,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"agentic":2,"coding":2},"benchmarks":[{"benchmarkId":"terminalBench21","benchmarkName":"Terminal-Bench 2.1","category":"agentic","score":92.8,"unit":"percent","family":"terminal-bench-2-1","index":75.41569903044733,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"deepSwe","benchmarkName":"DeepSWE","category":"agentic","score":73,"unit":"percent","family":"deepSwe","index":74.14568276442486,"aboveFit":false,"run":null,"source":"Datacurve AI","sourceUrl":"https://deepswe.datacurve.ai/blog","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"frontierCode","benchmarkName":"FrontierCode 1.1 Main","category":"coding","score":50,"unit":"percent","family":"frontierCode","index":73.71916458427832,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"frontierCode2","benchmarkName":"FrontierCode 2.0","category":"coding","score":23.8,"unit":"percent","family":"frontierCode2","index":69.67197191761235,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":40,"benchmarkCount":4,"categoryCount":2,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"swe-2-flash","providerId":"cognition","slug":"swe-2-flash","name":"SWE-2 Flash","provider":"Cognition","description":"SWE-2 Flash is a reasoning model from Cognition in the SWE family. It has 4 scored results in 2 categories. It has no index.","logoUrl":null,"releaseDate":"2026-11-04","contextWindowTokens":1000000,"modalities":["text"],"openWeights":false,"details":null,"reasoning":true,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":69.03898129962373,"coding":69.9264343515835,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"agentic":2,"coding":2},"benchmarks":[{"benchmarkId":"terminalBench21","benchmarkName":"Terminal-Bench 2.1","category":"agentic","score":89.2,"unit":"percent","family":"terminal-bench-2-1","index":73.31021971834461,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"deepSwe","benchmarkName":"DeepSWE","category":"agentic","score":60.1,"unit":"percent","family":"deepSwe","index":64.76774288090283,"aboveFit":false,"run":null,"source":"Datacurve AI","sourceUrl":"https://deepswe.datacurve.ai/blog","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"sweAtlasCodebaseQna","benchmarkName":"SWE-Atlas Codebase QnA","category":"coding","score":50,"unit":"percent","family":"sweAtlasCodebaseQna","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"frontierCode","benchmarkName":"FrontierCode 1.1 Main","category":"coding","score":45.5,"unit":"percent","family":"frontierCode","index":70.0326244053452,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"frontierCode2","benchmarkName":"FrontierCode 2.0","category":"coding","score":24,"unit":"percent","family":"frontierCode2","index":69.82024429782179,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":40,"benchmarkCount":4,"categoryCount":2,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"swellama13b","providerId":"unknown","slug":"unknown-swe-llama-13b","name":"SWE-Llama 13B","provider":"Unknown","description":"SWE-Llama 13B is a model. It has 2 scored results in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":5.074121816614458,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"coding":2},"benchmarks":[{"benchmarkId":"swe-bench:swe-bench-verified:overall","benchmarkName":"SWE-bench Verified","category":"coding","domain":null,"eligible":true,"ranked":true,"score":1.2,"unit":"percent","family":"swe-bench-verified","index":5,"aboveFit":false,"run":"RAG baseline","source":"SWE-bench team","sourceUrl":"https://www.swebench.com/","measuredAt":"2026-02-26","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"swe-bench:swe-bench-lite:overall","benchmarkName":"SWE-bench Lite","category":"coding","domain":null,"eligible":true,"ranked":true,"score":1,"unit":"percent","family":"swe-bench-lite","index":5.1482436332289225,"aboveFit":false,"run":"RAG baseline","source":"SWE-bench team","sourceUrl":"https://www.swebench.com/","measuredAt":"2025-09-11","stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":20,"benchmarkCount":2,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:swe-bench-team","name":"SWE-bench team","url":"https://www.swebench.com","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"swellama7b","providerId":"unknown","slug":"unknown-swe-llama-7b","name":"SWE-Llama 7B","provider":"Unknown","description":"SWE-Llama 7B is a model. It has 2 scored results in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":5.208304091176139,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"coding":2},"benchmarks":[{"benchmarkId":"swe-bench:swe-bench-verified:overall","benchmarkName":"SWE-bench Verified","category":"coding","domain":null,"eligible":true,"ranked":true,"score":1.4,"unit":"percent","family":"swe-bench-verified","index":5,"aboveFit":false,"run":"RAG baseline","source":"SWE-bench team","sourceUrl":"https://www.swebench.com/","measuredAt":"2026-02-26","stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"swe-bench:swe-bench-lite:overall","benchmarkName":"SWE-bench Lite","category":"coding","domain":null,"eligible":true,"ranked":true,"score":1.33,"unit":"percent","family":"swe-bench-lite","index":5.416608182352277,"aboveFit":false,"run":"RAG baseline","source":"SWE-bench team","sourceUrl":"https://www.swebench.com/","measuredAt":"2025-09-11","stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":20,"benchmarkCount":2,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:swe-bench-team","name":"SWE-bench team","url":"https://www.swebench.com","license":"No licence stated","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"switchyard","providerId":"nvidia","slug":"nvidia-switchyard","name":"Switchyard","provider":"NVIDIA","description":"Switchyard is a model from NVIDIA. It has no scored result.","logoUrl":null,"releaseDate":"2026-09-21T21:25:47.000Z","contextWindowTokens":1000000,"modalities":["text"],"openWeights":null,"details":{"apiId":"nvidia/switchyard","maxOutputTokens":null,"knowledgeCutoff":null,"retiresAt":null,"tools":null,"structuredOutput":null,"cacheReadPerMillion":null,"weightsUrl":null},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"system-one-sg","providerId":"sean-goedecke","slug":"system-one-sg","name":"system-one Qwen3-8B","provider":"Sean Goedecke","description":"system-one Qwen3-8B is a standard model from Sean Goedecke. It has 2 scored results in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":71.3435992944577,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"reasoning":2},"benchmarks":[{"benchmarkId":"jevBench15","benchmarkName":"jevBench15","category":"reasoning","score":44.13645499190376,"unit":"percent","family":"jevBench15","index":78.20168221237569,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"jevBench14","benchmarkName":"jevBench14","category":"reasoning","score":23.369044226391765,"unit":"percent","family":"jevBench14","index":64.48551637653974,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":15,"benchmarkCount":2,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"system-one-open","providerId":"mithalouni","slug":"system-one-open","name":"system-one-open","provider":"mithalouni","description":"system-one-open is a standard model from mithalouni. It has 2 scored results in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":80.84952416811423,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"reasoning":2},"benchmarks":[{"benchmarkId":"jevBench14","benchmarkName":"jevBench14","category":"reasoning","score":45.11471760046746,"unit":"percent","family":"jevBench14","index":84.52709132452343,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"jevBench15","benchmarkName":"jevBench15","category":"reasoning","score":42.43603455732882,"unit":"percent","family":"jevBench15","index":77.17195701170503,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":15,"benchmarkCount":2,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"ternary-bonsai-1-7b","providerId":"prism-ml","slug":"ternary-bonsai-1-7b","name":"Ternary Bonsai 1.7B","provider":"Prism ML","description":"Ternary Bonsai 1.7B is a standard model from Prism ML in the Ternary Bonsai family. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":"2026-04-16","contextWindowTokens":32000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":5,"math":null},"capabilityCounts":{"instruction":1},"benchmarks":[{"benchmarkId":"ifeval","benchmarkName":"Instruction-Following Eval","category":"instruction","score":70.1,"unit":"percent","family":"ifeval","index":5,"aboveFit":false,"run":null,"source":"Jeffrey Zhou, Tianjian Lu, Swaroop Mishra, Siddhartha Brahma, Sujoy Basu, Yi Luan, Denny Zhou, Le Hou","sourceUrl":"https://arxiv.org/abs/2311.07911","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"musr","benchmarkName":"Testing the Limits of Chain-of-thought with Multistep Soft Reasoning","category":"reasoning","score":50.8,"unit":"percent","family":"musr","index":null,"aboveFit":false,"run":null,"source":"Zayne Sprague, Xi Ye, Kaj Bostrom, Swarat Chaudhuri, Greg Durrett","sourceUrl":"https://arxiv.org/abs/2310.16049","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"ternarybonsai29b","providerId":"prismml","slug":"prismml-ternary-bonsai-2-9b","name":"Ternary Bonsai 2 9B","provider":"PrismML","description":"Ternary Bonsai 2 9B is a model from PrismML. It has no scored result.","logoUrl":null,"releaseDate":"2026-10-16T06:09:55.000Z","contextWindowTokens":262144,"modalities":["text","image"],"openWeights":true,"details":{"apiId":"prism-ml/ternary-bonsai-2-9b","maxOutputTokens":65536,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.04,"weightsUrl":"https://huggingface.co/prism-ml/Ternary-Bonsai-2-9B-gguf"},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":0.08,"outputPerMillion":0.13,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":52,"timeToFirstToken":4.161,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"ternary-bonsai-27b","providerId":"prism-ml","slug":"ternary-bonsai-27b","name":"Ternary Bonsai 27B","provider":"Prism ML","description":"Ternary Bonsai 27B is a reasoning model from Prism ML in the Ternary Bonsai family. It has no scored result.","logoUrl":null,"releaseDate":"2026-07-14","contextWindowTokens":262000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":true,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"ternary-bonsai-4b","providerId":"prism-ml","slug":"ternary-bonsai-4b","name":"Ternary Bonsai 4B","provider":"Prism ML","description":"Ternary Bonsai 4B is a standard model from Prism ML in the Ternary Bonsai family. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":"2026-04-16","contextWindowTokens":32000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":8.385988343062294,"math":null},"capabilityCounts":{"instruction":1},"benchmarks":[{"benchmarkId":"ifeval","benchmarkName":"Instruction-Following Eval","category":"instruction","score":72.1,"unit":"percent","family":"ifeval","index":8.385988343062294,"aboveFit":false,"run":null,"source":"Jeffrey Zhou, Tianjian Lu, Swaroop Mishra, Siddhartha Brahma, Sujoy Basu, Yi Luan, Denny Zhou, Le Hou","sourceUrl":"https://arxiv.org/abs/2311.07911","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"musr","benchmarkName":"Testing the Limits of Chain-of-thought with Multistep Soft Reasoning","category":"reasoning","score":45.1,"unit":"percent","family":"musr","index":null,"aboveFit":false,"run":null,"source":"Zayne Sprague, Xi Ye, Kaj Bostrom, Swarat Chaudhuri, Greg Durrett","sourceUrl":"https://arxiv.org/abs/2310.16049","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"ternary-bonsai-8b","providerId":"prism-ml","slug":"ternary-bonsai-8b","name":"Ternary Bonsai 8B","provider":"Prism ML","description":"Ternary Bonsai 8B is a standard model from Prism ML in the Ternary Bonsai family. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":"2026-04-16","contextWindowTokens":64000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":27.92080749893341,"math":null},"capabilityCounts":{"instruction":1},"benchmarks":[{"benchmarkId":"ifeval","benchmarkName":"Instruction-Following Eval","category":"instruction","score":81.8,"unit":"percent","family":"ifeval","index":27.92080749893341,"aboveFit":false,"run":null,"source":"Jeffrey Zhou, Tianjian Lu, Swaroop Mishra, Siddhartha Brahma, Sujoy Basu, Yi Luan, Denny Zhou, Le Hou","sourceUrl":"https://arxiv.org/abs/2311.07911","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"musr","benchmarkName":"Testing the Limits of Chain-of-thought with Multistep Soft Reasoning","category":"reasoning","score":56.2,"unit":"percent","family":"musr","index":null,"aboveFit":false,"run":null,"source":"Zayne Sprague, Xi Ye, Kaj Bostrom, Swarat Chaudhuri, Greg Durrett","sourceUrl":"https://arxiv.org/abs/2310.16049","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"teuken7bbasev06","providerId":"unknown","slug":"unknown-teuken-7b-base-v0-6","name":"Teuken 7B Base V0.6","provider":"Unknown","description":"Teuken 7B Base V0.6 is a model. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":5,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":0,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-french:overall","benchmarkName":"EuroEval French","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":0,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/french","measuredAt":null,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-spanish:overall","benchmarkName":"EuroEval Spanish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":0,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/spanish","measuredAt":null,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-italian:overall","benchmarkName":"EuroEval Italian","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":0,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/italian","measuredAt":null,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-portuguese:overall","benchmarkName":"EuroEval Portuguese","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":0,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/portuguese","measuredAt":null,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-dutch:overall","benchmarkName":"EuroEval Dutch","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":0,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/dutch","measuredAt":null,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":0,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":0,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"teuken7binstructv06","providerId":"unknown","slug":"unknown-teuken-7b-instruct-v0-6","name":"Teuken 7B Instruct V0.6","provider":"Unknown","description":"Teuken 7B Instruct V0.6 is a model. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":9.27786151249196,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-portuguese:overall","benchmarkName":"EuroEval Portuguese","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":32.765,"unit":"percent","family":"euroeval","index":16.67480144679307,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/portuguese","measuredAt":null,"stderr":0.37105003031936273,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-dutch:overall","benchmarkName":"EuroEval Dutch","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":30.07,"unit":"percent","family":"euroeval","index":13.018306548626569,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/dutch","measuredAt":null,"stderr":0.5410882952068696,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":28.557499999999997,"unit":"percent","family":"euroeval","index":10.966192064961696,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":0.6536328766976153,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-italian:overall","benchmarkName":"EuroEval Italian","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":28.496249999999996,"unit":"percent","family":"euroeval","index":10.883089908185184,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/italian","measuredAt":null,"stderr":0.5504600348799175,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-french:overall","benchmarkName":"EuroEval French","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":26.130000000000003,"unit":"percent","family":"euroeval","index":7.67263311679875,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/french","measuredAt":null,"stderr":0.7274677827917879,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-spanish:overall","benchmarkName":"EuroEval Spanish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":24.962499999999995,"unit":"percent","family":"euroeval","index":6.088604250895436,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/spanish","measuredAt":null,"stderr":0.5427562758734347,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":23.89428571428571,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.4903268381021636,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":22.875,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.4458804492237802,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"decision-index-0-3-tev1-0-8b-experimental","providerId":"togethercomputer","slug":"decision-index-0-3-tev1-0-8b-experimental","name":"Tev1-0.8B-experimental","provider":"togethercomputer","description":"Tev1-0.8B-experimental is a model from togethercomputer. It has no scored result.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"decision-index-0-3-tev1-4b-experimental","providerId":"togethercomputer","slug":"decision-index-0-3-tev1-4b-experimental","name":"Tev1-4B-experimental","provider":"togethercomputer","description":"Tev1-4B-experimental is a model from togethercomputer. It has no scored result.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"thunder-llm","providerId":"academic","slug":"thunder-llm","name":"Thunder-LLM 8B","provider":"Academic","description":"Thunder-LLM 8B is a standard model from Academic in the Thunder-LLM family. It has no scored result.","logoUrl":null,"releaseDate":null,"contextWindowTokens":32000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"tildeopen30b","providerId":"unknown","slug":"unknown-tildeopen-30b","name":"TildeOpen 30B","provider":"Unknown","description":"TildeOpen 30B is a model. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":5,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-french:overall","benchmarkName":"EuroEval French","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":24.402500000000003,"unit":"percent","family":"euroeval","index":5.3288131032244905,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/french","measuredAt":null,"stderr":0.5273503934766712,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-spanish:overall","benchmarkName":"EuroEval Spanish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":22.00875,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/spanish","measuredAt":null,"stderr":0.7127937990751604,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":17.57625,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.77026476779092,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"tinyrecursionmodeltrm","providerId":"unknown","slug":"unknown-tiny-recursion-model-trm","name":"Tiny Recursion Model (TRM)","provider":"Unknown","description":"Tiny Recursion Model (TRM) is a model. It has 2 scored results in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":43.71745513125509,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"reasoning":2},"benchmarks":[{"benchmarkId":"arc-prize:v1-public-eval:overall","benchmarkName":"ARC-AGI-1 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":44,"unit":"percent","family":"arc-prize:v1-public-eval:overall","index":null,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":0.39465,"latency":null},{"benchmarkId":"arc-prize:v1-semi-private:overall","benchmarkName":"ARC-AGI-1 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":40,"unit":"percent","family":"arc-prize:v1-semi-private:overall","index":45.37680297916666,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":1.7638,"latency":null},{"benchmarkId":"arc-prize:v2-semi-private:overall","benchmarkName":"ARC-AGI-2 (semi-private)","category":"reasoning","domain":null,"eligible":true,"ranked":true,"score":6.25,"unit":"percent","family":"arc-prize:v2-semi-private:overall","index":42.05810728334352,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":2.1,"latency":null},{"benchmarkId":"arc-prize:v2-public-eval:overall","benchmarkName":"ARC-AGI-2 (public eval)","category":"reasoning","domain":null,"eligible":false,"ranked":true,"score":5,"unit":"percent","family":"arc-prize:v2-public-eval:overall","index":null,"aboveFit":false,"run":null,"source":"ARC Prize Foundation","sourceUrl":"https://arcprize.org/leaderboard","measuredAt":null,"stderr":null,"costPerTest":1.8,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":15,"benchmarkCount":2,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:arc-prize-foundation","name":"ARC Prize Foundation","url":"https://arcprize.org","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"toast-1","providerId":"mixedbread","slug":"toast-1","name":"Toast 1","provider":"Mixedbread","description":"Toast 1 is a reasoning model from Mixedbread. It has no scored result.","logoUrl":null,"releaseDate":"2026-08-13","contextWindowTokens":131000,"modalities":["text"],"openWeights":false,"details":null,"reasoning":true,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":0.3,"outputPerMillion":0.72,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"trustllmv28b","providerId":"unknown","slug":"unknown-trustllm-v2-8b","name":"Trustllm V2 8B","provider":"Unknown","description":"Trustllm V2 8B is a model. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":5,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-dutch:overall","benchmarkName":"EuroEval Dutch","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":28.593333333333337,"unit":"percent","family":"euroeval","index":11.01480965327994,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/dutch","measuredAt":null,"stderr":0.4500466368151661,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":21.588749999999997,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":0.6684906038980652,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":17.26375,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.7742334919131308,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"trustllmv28bmidtrain","providerId":"unknown","slug":"unknown-trustllm-v2-8b-midtrain","name":"Trustllm V2 8B Midtrain","provider":"Unknown","description":"Trustllm V2 8B Midtrain is a model. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":5,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-dutch:overall","benchmarkName":"EuroEval Dutch","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":30.471111111111114,"unit":"percent","family":"euroeval","index":13.562522033367081,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/dutch","measuredAt":null,"stderr":0.42515356702673396,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":24.056250000000002,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":0.5517585522672033,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":19.46375,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.6038819731536951,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"trustllmv28bsft","providerId":"unknown","slug":"unknown-trustllm-v2-8b-sft","name":"Trustllm V2 8B SFT","provider":"Unknown","description":"Trustllm V2 8B SFT is a model. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":5,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-dutch:overall","benchmarkName":"EuroEval Dutch","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":23.36333333333333,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/dutch","measuredAt":null,"stderr":0.24706424415673903,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":21.03125,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":0.21642695072471912,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":17.7425,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.37578251688975634,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"trustllmv28bsft8k","providerId":"unknown","slug":"unknown-trustllm-v2-8b-sft-8k","name":"Trustllm V2 8B SFT 8K","provider":"Unknown","description":"Trustllm V2 8B SFT 8K is a model. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":5,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-dutch:overall","benchmarkName":"EuroEval Dutch","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":24.752222222222223,"unit":"percent","family":"euroeval","index":5.80330568998923,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/dutch","measuredAt":null,"stderr":0.19964474621526615,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":21.24,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":0.21648109039821467,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":17.861250000000002,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.499809338648249,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"trustllmv28bsftmixb2exp","providerId":"unknown","slug":"unknown-trustllm-v2-8b-sft-mixb2-exp","name":"Trustllm V2 8B SFT Mixb2 Exp","provider":"Unknown","description":"Trustllm V2 8B SFT Mixb2 Exp is a model. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":5,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-dutch:overall","benchmarkName":"EuroEval Dutch","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":24.016666666666666,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/dutch","measuredAt":null,"stderr":0.2550671661724795,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":20.424999999999997,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":0.2288763312358882,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":17.155,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.2910621711937159,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"tulu3tlu370b","providerId":"allen-institute-for-ai","slug":"allen-institute-for-ai-tulu-3-t-lu-3-70b","name":"Tulu 3 (Tülu 3) 70B","provider":"Allen Institute for AI","description":"Tulu 3 (Tülu 3) 70B is a model from Allen Institute for AI. It has 3 scored results in 2 categories. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":19.65149787902393,"multilingual":null,"instruction":null,"math":15.61357714072986},"capabilityCounts":{"knowledge":1,"math":2},"benchmarks":[{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":46.275252525252526,"unit":"percent","family":"gpqa-diamond","index":19.65149787902393,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.4058363620939613,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-math-level-5:overall","benchmarkName":"MATH level 5","category":"math","domain":null,"eligible":true,"ranked":true,"score":42.6642749244713,"unit":"percent","family":"epoch-ai:epoch-math-level-5:overall","index":19.722312090525712,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.0405854683575873,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","benchmarkName":"OTIS Mock AIME 2024-2025","category":"math","domain":null,"eligible":true,"ranked":true,"score":4.444444444444445,"unit":"percent","family":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","index":11.504842190934006,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.8272780318308628,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":25,"benchmarkCount":3,"categoryCount":2,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"typecastlm","providerId":"mikhail-gribov","slug":"typecastlm","name":"typecastlm","provider":"Mikhail Gribov","description":"typecastlm is a standard model from Mikhail Gribov. It has 2 scored results in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":65.46895487070934,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"reasoning":2},"benchmarks":[{"benchmarkId":"jevBench14","benchmarkName":"jevBench14","category":"reasoning","score":25.279442237462565,"unit":"percent","family":"jevBench14","index":66.24620615885749,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"jevBench15","benchmarkName":"jevBench15","category":"reasoning","score":21.826965316555995,"unit":"percent","family":"jevBench15","index":64.69170358256117,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":15,"benchmarkCount":2,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"ui-mate-27b","providerId":"tencent","slug":"ui-mate-27b","name":"UI-Mate-27B","provider":"Tencent","description":"UI-Mate-27B is a reasoning model from Tencent in the UI-Mate family. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":"2026-08-14","contextWindowTokens":262000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":true,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":62.63980430436236,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"agentic":1},"benchmarks":[{"benchmarkId":"osWorldVerified","benchmarkName":"OSWorld-Verified","category":"agentic","score":77,"unit":"percent","family":"osWorldVerified","index":62.63980430436236,"aboveFit":false,"run":null,"source":"Tianbao Xie, Danyang Zhang, Jixuan Chen, Xiaochuan Li, Siheng Zhao, Ruisheng Cao, Toh Jing Hua, Zhoujun Cheng, Dongchan Shin, Fangyu Lei, Yitao Liu, Yiheng Xu, Shuyan Zhou, Silvio Savarese, Caiming Xiong, Victor Zhong, Tao Yu","sourceUrl":"https://os-world.github.io/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":20,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"ui-mate-9b","providerId":"tencent","slug":"ui-mate-9b","name":"UI-Mate-9B","provider":"Tencent","description":"UI-Mate-9B is a reasoning model from Tencent in the UI-Mate family. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":"2026-08-14","contextWindowTokens":262000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":true,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":53.92142805985583,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"agentic":1},"benchmarks":[{"benchmarkId":"osWorldVerified","benchmarkName":"OSWorld-Verified","category":"agentic","score":66.2,"unit":"percent","family":"osWorldVerified","index":53.92142805985583,"aboveFit":false,"run":null,"source":"Tianbao Xie, Danyang Zhang, Jixuan Chen, Xiaochuan Li, Siheng Zhao, Ruisheng Cao, Toh Jing Hua, Zhoujun Cheng, Dongchan Shin, Fangyu Lei, Yitao Liu, Yiheng Xu, Shuyan Zhou, Silvio Savarese, Caiming Xiong, Victor Zhong, Tao Yu","sourceUrl":"https://os-world.github.io/","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":20,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"uitars157b","providerId":"bytedance","slug":"bytedance-ui-tars-7b","name":"UI-TARS 7B","provider":"ByteDance","description":"UI-TARS 7B is a model from ByteDance. It has no scored result.","logoUrl":null,"releaseDate":"2025-07-22T17:24:16.000Z","contextWindowTokens":128000,"modalities":["image","text"],"openWeights":true,"details":{"apiId":"bytedance/ui-tars-1.5-7b","maxOutputTokens":2048,"knowledgeCutoff":"2025-01-31","retiresAt":null,"tools":false,"structuredOutput":true,"cacheReadPerMillion":0.09999999999999999,"weightsUrl":"https://huggingface.co/ByteDance-Seed/UI-TARS-1.5-7B"},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":0.09999999999999999,"outputPerMillion":0.19999999999999998,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":35,"timeToFirstToken":1.002,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"ultramarine-1-flash","providerId":"paper-instruments","slug":"ultramarine-1-flash","name":"Ultramarine 1 Flash","provider":"Paper Instruments","description":"Ultramarine 1 Flash is a reasoning model from Paper Instruments in the Ultramarine 1 family. It has 4 scored results in 1 category. It has no index.","logoUrl":null,"releaseDate":"2026-10-14","contextWindowTokens":null,"modalities":["text"],"openWeights":false,"details":null,"reasoning":true,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":71.11139086585555,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"agentic":4},"benchmarks":[{"benchmarkId":"terminalBench21","benchmarkName":"Terminal-Bench 2.1","category":"agentic","score":83.2,"unit":"percent","family":"terminal-bench-2-1","index":69.80108753150675,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"jobBench","benchmarkName":"JobBench","category":"agentic","score":62.9,"unit":"percent","family":"jobBench","index":71.28353094965259,"aboveFit":false,"run":null,"source":"Yuetai Li, Yichen Feng, Zhangchen Xu, Zixian Ma, Kaiyuan Zheng, Fengqing Jiang, Xinghua Sun, Rulin Shao, Zichen Chen, Yue Huang, Xinyang Han, Brian Lee, Kayla Xu, Shenglai Zeng, Hang Hua, Xiangliang Zhang, Basel Alomair, Ranjay Krishna, Luke Zettlemoyer, Pang Wei Koh, Bhaskar Ramasubramanian, Luyao Niu, Xiang Yue, Radha Poovendran","sourceUrl":"https://arxiv.org/abs/2605.26329","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"deepSwe","benchmarkName":"DeepSWE","category":"agentic","score":61.5,"unit":"percent","family":"deepSwe","index":65.78550379849437,"aboveFit":false,"run":null,"source":"Datacurve AI","sourceUrl":"https://deepswe.datacurve.ai/blog","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"automationBench","benchmarkName":"AutomationBench","category":"agentic","score":50.2,"unit":"percent","family":"automationBench","index":77.57544118376853,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":0.15,"outputPerMillion":0.5,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":20,"benchmarkCount":4,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"ultravox-glm-4p6","providerId":"fixie-ai","slug":"ultravox-glm-4p6","name":"Ultravox GLM-4P6","provider":"Fixie AI","description":"Ultravox GLM-4P6 is a standard model from Fixie AI. It has no scored result.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"ultravox-glm-4p7","providerId":"fixie-ai","slug":"ultravox-glm-4p7","name":"Ultravox GLM-4P7","provider":"Fixie AI","description":"Ultravox GLM-4P7 is a standard model from Fixie AI. It has no scored result.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"ultravox-v0-4-1-llama-3-1-8b","providerId":"fixie-ai","slug":"ultravox-v0-4-1-llama-3-1-8b","name":"Ultravox v0.4.1 Llama 3.1 8B","provider":"Fixie AI","description":"Ultravox v0.4.1 Llama 3.1 8B is a standard model from Fixie AI. It has no scored result.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"ultravox-v0-5-llama-3-1-8b","providerId":"fixie-ai","slug":"ultravox-v0-5-llama-3-1-8b","name":"Ultravox v0.5 Llama 3.1 8B","provider":"Fixie AI","description":"Ultravox v0.5 Llama 3.1 8B is a standard model from Fixie AI. It has no scored result.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"ultravox-v0-5-llama-3-2-1b","providerId":"fixie-ai","slug":"ultravox-v0-5-llama-3-2-1b","name":"Ultravox v0.5 Llama 3.2 1B","provider":"Fixie AI","description":"Ultravox v0.5 Llama 3.2 1B is a standard model from Fixie AI. It has no scored result.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"ultravox-v0-6-llama-3-3-70b","providerId":"fixie-ai","slug":"ultravox-v0-6-llama-3-3-70b","name":"Ultravox v0.6 Llama 3.3 70B","provider":"Fixie AI","description":"Ultravox v0.6 Llama 3.3 70B is a standard model from Fixie AI. It has 7 scored results in 4 categories. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":27.23712991982319,"coding":null,"reasoning":27.459314861819237,"multimodal":null,"knowledge":25.44139907213778,"multilingual":null,"instruction":33.03907935053449,"math":null},"capabilityCounts":{"knowledge":2,"instruction":1,"agentic":2,"reasoning":2},"benchmarks":[{"benchmarkId":"aaGpqaDiamond","benchmarkName":"Artificial Analysis GPQA Diamond","category":"knowledge","score":49.8,"unit":"percent","family":"gpqa-diamond","index":22.884299948854824,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gpqa-diamond","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaIfBench","benchmarkName":"Artificial Analysis IFBench","category":"instruction","score":47.1,"unit":"percent","family":"ifbench-family","index":33.03907935053449,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/ifbench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"tau2Bench","benchmarkName":"τ²-Bench Tool-Agent-User Evaluation","category":"agentic","score":26.6,"unit":"percent","family":"tau2-bench","index":20.44048492245359,"aboveFit":false,"run":null,"source":"Victor Barres, Honghua Dong, Soham Ray, Xujie Si, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2506.07982","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"lcr","benchmarkName":"Artificial Analysis Long Context Reasoning","category":"reasoning","score":15.7,"unit":"percent","family":"lcr","index":18.06951371086557,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"aaHle","benchmarkName":"Artificial Analysis Humanity's Last Exam","category":"knowledge","score":3.6,"unit":"percent","family":"hle","index":27.998498195420737,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/hle","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gdpvalAaNormalized","benchmarkName":"GDPval-AA normalized","category":"agentic","score":0,"unit":"percent","family":"gdpvalAaNormalized","index":34.03377491719279,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/gdpval-aa","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"critpt","benchmarkName":"Critical Physics Tasks","category":"reasoning","score":0,"unit":"percent","family":"critpt","index":36.8491160127729,"aboveFit":false,"run":null,"source":"Artificial Analysis","sourceUrl":"https://artificialanalysis.ai/evaluations/critpt","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":55.00000000000001,"benchmarkCount":7,"categoryCount":4,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"ultravox-v0-7","providerId":"fixie-ai","slug":"ultravox-v0-7","name":"Ultravox v0.7","provider":"Fixie AI","description":"Ultravox v0.7 is a standard model from Fixie AI. It has no scored result.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"dolphinmistral24bveniceedition","providerId":"venice","slug":"venice-uncensored","name":"Uncensored","provider":"Venice","description":"Uncensored is a model from Venice. It has no scored result.","logoUrl":null,"releaseDate":"2025-07-09T21:02:46.000Z","contextWindowTokens":128000,"modalities":["text"],"openWeights":true,"details":{"apiId":"cognitivecomputations/dolphin-mistral-24b-venice-edition","maxOutputTokens":8192,"knowledgeCutoff":"2024-04-30","retiresAt":null,"tools":false,"structuredOutput":false,"cacheReadPerMillion":null,"weightsUrl":"https://huggingface.co/cognitivecomputations/Dolphin-Mistral-24B-Venice-Edition"},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":0.19999999999999998,"outputPerMillion":0.8999999999999999,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":90,"timeToFirstToken":0.804,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":0.2,"outputPerMillion":0.9,"cacheReadPerMillion":null,"contextTokens":128000,"maxOutputTokens":8192,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"universal35pro","providerId":"unknown","slug":"unknown-universal-3-5-pro","name":"Universal 3.5 Pro","provider":"Unknown","description":"Universal 3.5 Pro is a model. It has no scored result.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"universallanguagemodel","providerId":"unknown","slug":"unknown-universal-language-model","name":"Universal Language Model","provider":"Unknown","description":"Universal Language Model is a model. It has no scored result.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"unslopnemo12b","providerId":"thedrummer","slug":"thedrummer-unslopnemo-12b","name":"UnslopNemo 12B","provider":"TheDrummer","description":"UnslopNemo 12B is a model from TheDrummer. It has no scored result.","logoUrl":null,"releaseDate":"2024-11-08T22:04:08.000Z","contextWindowTokens":1024000,"modalities":["text"],"openWeights":true,"details":{"apiId":"thedrummer/unslopnemo-12b","maxOutputTokens":819200,"knowledgeCutoff":"2024-04-30","retiresAt":null,"tools":false,"structuredOutput":true,"cacheReadPerMillion":null,"weightsUrl":"https://huggingface.co/TheDrummer/UnslopNemo-12B-v4.1"},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":0.39999999999999997,"outputPerMillion":0.39999999999999997,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":46,"timeToFirstToken":0.537,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":0.4,"outputPerMillion":0.4,"cacheReadPerMillion":null,"contextTokens":1024000,"maxOutputTokens":819200,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"varco","providerId":"nc-ai","slug":"varco","name":"Varco","provider":"NC AI","description":"Varco is a standard model from NC AI. It has no scored result.","logoUrl":null,"releaseDate":null,"contextWindowTokens":64000,"modalities":["text"],"openWeights":false,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"verdict-small","providerId":"manavarya09-manav","slug":"verdict-small","name":"verdict-small","provider":"Manavarya09 (Manav)","description":"verdict-small is a standard model from Manavarya09 (Manav). It has 2 scored results in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":49.83220566973087,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"reasoning":2},"benchmarks":[{"benchmarkId":"jevBench14","benchmarkName":"jevBench14","category":"reasoning","score":5.688474707640219,"unit":"percent","family":"jevBench14","index":48.19048365727096,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"jevBench15","benchmarkName":"jevBench15","category":"reasoning","score":0,"unit":"percent","family":"jevBench15","index":51.47392768219078,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":15,"benchmarkCount":2,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"vibevoice-asr-streaming-1-5b","providerId":"microsoft","slug":"vibevoice-asr-streaming-1-5b","name":"VibeVoice-ASR-Streaming 1.5B","provider":"Microsoft","description":"VibeVoice-ASR-Streaming 1.5B is a standard model from Microsoft. It has no scored result.","logoUrl":null,"releaseDate":"2026-09-02","contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"vibevoice-asr-streaming-7b","providerId":"microsoft","slug":"vibevoice-asr-streaming-7b","name":"VibeVoice-ASR-Streaming 7B","provider":"Microsoft","description":"VibeVoice-ASR-Streaming 7B is a standard model from Microsoft. It has no scored result.","logoUrl":null,"releaseDate":"2026-09-02","contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"vita-1-0","providerId":"vita-mllm","slug":"vita-1-0","name":"VITA 1.0","provider":"VITA-MLLM","description":"VITA 1.0 is a standard model from VITA-MLLM. It has no scored result.","logoUrl":null,"releaseDate":"2024-08-09","contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"vita-1-5","providerId":"vita-mllm","slug":"vita-1-5","name":"VITA 1.5","provider":"VITA-MLLM","description":"VITA 1.5 is a standard model from VITA-MLLM. It has no scored result.","logoUrl":null,"releaseDate":"2025-01-03","contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"von-395m","providerId":"wfzyx-victor-hugo","slug":"von-395m","name":"Von 395M","provider":"wfzyx (Victor Hugo)","description":"Von 395M is a standard model from wfzyx (Victor Hugo). It has 2 scored results in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":59.87580234392918,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"reasoning":2},"benchmarks":[{"benchmarkId":"jevBench14","benchmarkName":"jevBench14","category":"reasoning","score":27.483645331198645,"unit":"percent","family":"jevBench14","index":68.27767700566758,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"jevBench15","benchmarkName":"jevBench15","category":"reasoning","score":0,"unit":"percent","family":"jevBench15","index":51.47392768219078,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":15,"benchmarkCount":2,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"voxtralmini","providerId":"mistral","slug":"mistral-voxtral-mini","name":"Voxtral Mini","provider":"Mistral","description":"Voxtral Mini is a model from Mistral. It has no scored result.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"voxtral-mini-4b-realtime-2602","providerId":"mistral","slug":"voxtral-mini-4b-realtime-2602","name":"Voxtral Mini 4B Realtime 2602","provider":"Mistral","description":"Voxtral Mini 4B Realtime 2602 is a standard model from Mistral. It has no scored result.","logoUrl":null,"releaseDate":"2026-02-04","contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"voxtralsmall24b2507","providerId":"mistral","slug":"mistral-voxtral-small-24b-2507","name":"Voxtral Small 24B 2507","provider":"Mistral","description":"Voxtral Small 24B 2507 is a model from Mistral. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":"2025-10-30T14:39:04.000Z","contextWindowTokens":32768,"modalities":["text","audio","file"],"openWeights":true,"details":{"apiId":"mistralai/voxtral-small-24b-2507","maxOutputTokens":26214,"knowledgeCutoff":null,"retiresAt":null,"tools":true,"structuredOutput":true,"cacheReadPerMillion":0.01,"weightsUrl":"https://huggingface.co/mistralai/Voxtral-Small-24B-2507"},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":46.95499839702159,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":55.082857142857144,"unit":"percent","family":"euroeval","index":46.95499839702159,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.24210808852785617,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":0.09999999999999999,"outputPerMillion":0.3,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":74.5,"timeToFirstToken":0.677,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[{"host":"amazon-bedrock","name":"Amazon Bedrock","kind":"cloud","url":"https://docs.aws.amazon.com/bedrock/latest/userguide/models-supported.html","inputPerMillion":0.1,"outputPerMillion":0.3,"cacheReadPerMillion":null,"contextTokens":32768,"maxOutputTokens":8192,"deprecated":true},{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":0.1,"outputPerMillion":0.3,"cacheReadPerMillion":0.01,"contextTokens":32768,"maxOutputTokens":26214,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"weaver","providerId":"mancer","slug":"mancer-weaver-alpha","name":"Weaver (alpha)","provider":"Mancer","description":"Weaver (alpha) is a model from Mancer. It has no scored result.","logoUrl":null,"releaseDate":"2023-08-02T00:00:00.000Z","contextWindowTokens":8000,"modalities":["text"],"openWeights":null,"details":{"apiId":"mancer/weaver","maxOutputTokens":6000,"knowledgeCutoff":"2023-06-30","retiresAt":null,"tools":false,"structuredOutput":false,"cacheReadPerMillion":0.3,"weightsUrl":null},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":0.3,"outputPerMillion":0.5,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":45,"timeToFirstToken":0.997,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[{"host":"openrouter","name":"OpenRouter","kind":"router","url":"https://openrouter.ai/models","inputPerMillion":0.3,"outputPerMillion":0.5,"cacheReadPerMillion":0.3,"contextTokens":8000,"maxOutputTokens":6000,"deprecated":false}],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:models-dev","name":"models.dev","url":"https://models.dev","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"winnow-12b","providerId":"eldan-ring","slug":"winnow-12b","name":"Winnow-12B Q8","provider":"Eldan Ring","description":"Winnow-12B Q8 is a standard model from Eldan Ring. It has 2 scored results in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":94.58403474036258,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"reasoning":2},"benchmarks":[{"benchmarkId":"jevBench15","benchmarkName":"jevBench15","category":"reasoning","score":73.23310265651646,"unit":"percent","family":"jevBench15","index":95,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"jevBench14","benchmarkName":"jevBench14","category":"reasoning","score":55.57545047877118,"unit":"percent","family":"jevBench14","index":94.16806948072515,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":15,"benchmarkCount":2,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"decision-index-0-3-winnow-e4b","providerId":"eldanring","slug":"decision-index-0-3-winnow-e4b","name":"Winnow-E4B","provider":"EldanRing","description":"Winnow-E4B is a model from EldanRing. It has no scored result.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"wispr-canto","providerId":"wispr","slug":"wispr-canto","name":"Wispr Canto","provider":"Wispr","description":"Wispr Canto is a standard model from Wispr. It has no scored result.","logoUrl":null,"releaseDate":"2026-09-17","contextWindowTokens":null,"modalities":["text"],"openWeights":false,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"wizardlm28x22b","providerId":"microsoft","slug":"microsoft-wizardlm-2-8x22b","name":"WizardLM-2 8x22B","provider":"Microsoft","description":"WizardLM-2 8x22B is a model from Microsoft. It has 2 scored results in 2 categories. It has no index.","logoUrl":null,"releaseDate":"2024-04-16T00:00:00.000Z","contextWindowTokens":65535,"modalities":["text"],"openWeights":null,"details":{"apiId":"microsoft/wizardlm-2-8x22b","maxOutputTokens":8000,"knowledgeCutoff":"2024-04-30","retiresAt":null,"tools":false,"structuredOutput":false,"cacheReadPerMillion":null,"weightsUrl":null},"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":17.04589368891091,"multilingual":null,"instruction":null,"math":11.499508152592824},"capabilityCounts":{"knowledge":1,"math":1},"benchmarks":[{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":43.43434343434344,"unit":"percent","family":"gpqa-diamond","index":17.04589368891091,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.323209333344425,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-math-level-5:overall","benchmarkName":"MATH level 5","category":"math","domain":null,"eligible":true,"ranked":true,"score":25.736404833836858,"unit":"percent","family":"epoch-ai:epoch-math-level-5:overall","index":11.499508152592824,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0.8719521575190187,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":0.62,"outputPerMillion":0.62,"batch":null,"freeTier":false,"state":"available","updatedAt":"2026-11-12T14:09:31.783Z"},"runtime":{"tokensPerSecond":18.5,"timeToFirstToken":1.493,"source":"OpenRouter","state":"available","updatedAt":"2026-11-12T14:39:40.309Z"},"arena":null,"quality":{"coveragePercent":25,"benchmarkCount":2,"categoryCount":2,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:openrouter","name":"OpenRouter","url":"https://openrouter.ai","license":"Terms restrict reuse","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"decision-index-0-3-xor-26b-a4b","providerId":"juspay","slug":"decision-index-0-3-xor-26b-a4b","name":"Xor 26B-A4B","provider":"Juspay","description":"Xor 26B-A4B is a model from Juspay. It has no scored result.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"yi1534b","providerId":"01-ai","slug":"01-ai-yi-1-5-34b","name":"Yi 1.5 34B","provider":"01.AI","description":"Yi 1.5 34B is a model from 01.AI. It has 2 scored results in 2 categories. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":6.536623455455036,"multilingual":null,"instruction":null,"math":11.375684557319985},"capabilityCounts":{"knowledge":1,"math":1},"benchmarks":[{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":31.9760101010101,"unit":"percent","family":"gpqa-diamond","index":6.536623455455036,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":2.000529166190088,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-math-level-5:overall","benchmarkName":"MATH level 5","category":"math","domain":null,"eligible":true,"ranked":true,"score":25.48149546827795,"unit":"percent","family":"epoch-ai:epoch-math-level-5:overall","index":11.375684557319985,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0.8772368987739622,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":25,"benchmarkCount":2,"categoryCount":2,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"yi34b","providerId":"01-ai","slug":"01-ai-yi-34b","name":"Yi 34B","provider":"01.AI","description":"Yi 34B is a model from 01.AI. It has 2 scored results in 2 categories. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":5,"multilingual":null,"instruction":null,"math":5},"capabilityCounts":{"knowledge":1,"math":1},"benchmarks":[{"benchmarkId":"epoch-ai:epoch-gpqa-diamond:overall","benchmarkName":"GPQA diamond","category":"knowledge","domain":null,"eligible":true,"ranked":true,"score":14.741161616161616,"unit":"percent","family":"gpqa-diamond","index":5,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":1.1372823721448395,"costPerTest":null,"latency":null},{"benchmarkId":"epoch-ai:epoch-math-level-5:overall","benchmarkName":"MATH level 5","category":"math","domain":null,"eligible":true,"ranked":true,"score":5.145392749244713,"unit":"percent","family":"epoch-ai:epoch-math-level-5:overall","index":5,"aboveFit":false,"run":null,"source":"Epoch AI","sourceUrl":"https://epoch.ai/benchmarks","measuredAt":null,"stderr":0.33693214361751755,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":25,"benchmarkCount":2,"categoryCount":2,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:epoch-ai","name":"Epoch AI","url":"https://epoch.ai","license":"CC BY","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"yugogpt","providerId":"unknown","slug":"unknown-yugogpt","name":"YugoGPT","provider":"Unknown","description":"YugoGPT is a model. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":14.398335367865656,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":31.087142857142855,"unit":"percent","family":"euroeval","index":14.398335367865656,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.9374084989360567,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"yulanminiinstruct","providerId":"unknown","slug":"unknown-yulan-mini-instruct","name":"YuLan Mini Instruct","provider":"Unknown","description":"YuLan Mini Instruct is a model. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":5.190121068332168,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-french:overall","benchmarkName":"EuroEval French","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":31.573750000000004,"unit":"percent","family":"euroeval","index":15.05854929560909,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/french","measuredAt":null,"stderr":0.3108154597184638,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-portuguese:overall","benchmarkName":"EuroEval Portuguese","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":27.905,"unit":"percent","family":"euroeval","index":10.080899700934374,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/portuguese","measuredAt":null,"stderr":0.3663225184178554,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-italian:overall","benchmarkName":"EuroEval Italian","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":24.3625,"unit":"percent","family":"euroeval","index":5.2745423069622674,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/italian","measuredAt":null,"stderr":0.4571309987301233,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-swedish:overall","benchmarkName":"EuroEval Swedish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":24.345,"unit":"percent","family":"euroeval","index":5.250798833597557,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/swedish","measuredAt":null,"stderr":0.3685295951480695,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-dutch:overall","benchmarkName":"EuroEval Dutch","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":24.255555555555556,"unit":"percent","family":"euroeval","index":5.129443303066779,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/dutch","measuredAt":null,"stderr":0.4623343880999477,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-german:overall","benchmarkName":"EuroEval German","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":21.715,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/german","measuredAt":null,"stderr":0.5519383457416236,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-spanish:overall","benchmarkName":"EuroEval Spanish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":20.47,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/spanish","measuredAt":null,"stderr":0.5610592660316733,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-polish:overall","benchmarkName":"EuroEval Polish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":20.31714285714286,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/polish","measuredAt":null,"stderr":0.3949476844714851,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"z-1","providerId":"z","slug":"z-1","name":"Z-1","provider":"Z","description":"Z-1 is a standard model from Z. It has no scored result.","logoUrl":null,"releaseDate":null,"contextWindowTokens":128000,"modalities":["text"],"openWeights":false,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"zagreus04bita","providerId":"unknown","slug":"unknown-zagreus-0-4b-ita","name":"Zagreus 0.4B Ita","provider":"Unknown","description":"Zagreus 0.4B Ita is a model. It has 1 scored result in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":null,"details":null,"reasoning":null,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":5,"instruction":null,"math":null},"capabilityCounts":{"multilingual":1},"benchmarks":[{"benchmarkId":"euroeval:euroeval-french:overall","benchmarkName":"EuroEval French","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":8.24,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/french","measuredAt":null,"stderr":0.7151835428755334,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-portuguese:overall","benchmarkName":"EuroEval Portuguese","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":6.9075,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/portuguese","measuredAt":null,"stderr":0.5754821348226199,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-italian:overall","benchmarkName":"EuroEval Italian","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":6.1525,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/italian","measuredAt":null,"stderr":0.7006403321248357,"costPerTest":null,"latency":null},{"benchmarkId":"euroeval:euroeval-spanish:overall","benchmarkName":"EuroEval Spanish","category":"multilingual","domain":null,"eligible":true,"ranked":true,"score":4.91375,"unit":"percent","family":"euroeval","index":5,"aboveFit":false,"run":null,"source":"EuroEval","sourceUrl":"https://euroeval.com/leaderboards/spanish","measuredAt":null,"stderr":0.3677040249983674,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":5,"benchmarkCount":1,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"},{"id":"direct:euroeval","name":"EuroEval","url":"https://euroeval.com","license":"MIT","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"zaya1-74b-preview","providerId":"zyphra","slug":"zaya1-74b-preview","name":"ZAYA1-74B-Preview","provider":"Zyphra","description":"ZAYA1-74B-Preview is a reasoning model from Zyphra in the ZAYA1 family. It has 5 scored results in 3 categories. It has no index.","logoUrl":null,"releaseDate":"2026-05-07","contextWindowTokens":256000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":true,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":39.41325816938705,"reasoning":null,"multimodal":null,"knowledge":30.020836038618036,"multilingual":null,"instruction":null,"math":40.2944322840331},"capabilityCounts":{"math":1,"knowledge":2,"coding":2},"benchmarks":[{"benchmarkId":"aime2026","benchmarkName":"AIME 2026","category":"math","score":76.4,"unit":"percent","family":"aime2026","index":40.2944322840331,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmluPro","benchmarkName":"Massive Multitask Language Understanding Professional","category":"knowledge","score":68.1,"unit":"percent","family":"mmlu-pro","index":30.278577066482868,"aboveFit":false,"run":null,"source":"Yubo Wang, Xueguang Ma, Ge Zhang, Yuansheng Ni, Abhranil Chandra, Shiguang Guo, Weiming Ren, Aaran Arulraj, Xuan He, Ziyan Jiang, Tianle Li, Max Ku, Kai Wang, Alex Zhuang, Rongqi Fan, Xiang Yue, Wenhu Chen","sourceUrl":"https://arxiv.org/abs/2406.01574","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"liveCodeBenchV6","benchmarkName":"LiveCodeBench v6","category":"coding","score":65.7,"unit":"percent","family":"liveCodeBenchV6","index":38.36052567121004,"aboveFit":false,"run":null,"source":"LiveCodeBench maintainers","sourceUrl":"https://github.com/LiveCodeBench/LiveCodeBench","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gpqa","benchmarkName":"Graduate-Level Google-Proof Q&A","category":"knowledge","score":57.3,"unit":"percent","family":"gpqa-diamond","index":29.763095010753208,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"gpqaDiamond","benchmarkName":"GPQA Diamond","category":"knowledge","score":57.3,"unit":"percent","family":"gpqa-diamond","index":29.763095010753208,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"tau2Airline","benchmarkName":"τ²-Bench Airline Domain","category":"agentic","score":56.1,"unit":"percent","family":"tau2Airline","index":null,"aboveFit":false,"run":null,"source":"Victor Barres, Honghua Dong, Soham Ray, Xujie Si, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2506.07982","measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"sweVerified","benchmarkName":"Software Engineering Benchmark Verified","category":"coding","score":53.2,"unit":"percent","family":"swe-bench-verified","index":40.46599066756406,"aboveFit":false,"run":null,"source":"Carlos E. Jimenez, John Yang, Alexander Wettig, Shunyu Yao, Kexin Pei, Ofir Press, Karthik Narasimhan","sourceUrl":"https://arxiv.org/abs/2310.06770","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":45,"benchmarkCount":5,"categoryCount":3,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"zaya1-vl-8b","providerId":"zyphra","slug":"zaya1-vl-8b","name":"ZAYA1-VL-8B","provider":"Zyphra","description":"ZAYA1-VL-8B is a standard model from Zyphra in the ZAYA1 family. It has 4 scored results in 1 category. It has no index.","logoUrl":null,"releaseDate":"2026-05-08","contextWindowTokens":131000,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":26.592720330193327,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"multimodal":4},"benchmarks":[{"benchmarkId":"countBench","benchmarkName":"CountBench","category":"multimodal","score":88.1,"unit":"percent","family":"countBench","index":35.704425037059394,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"ai2dTest","benchmarkName":"AI2D test split","category":"multimodal","score":87.5,"unit":"percent","family":"ai2dTest","index":null,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":false,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"refcocoAvg","benchmarkName":"RefCOCO average","category":"multimodal","score":84.3,"unit":"percent","family":"refcocoAvg","index":34.583223145930766,"aboveFit":false,"run":null,"source":"RefCOCO dataset authors","sourceUrl":"https://github.com/lichengunc/refer","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"realWorldQa","benchmarkName":"RealWorldQA","category":"multimodal","score":65,"unit":"percent","family":"realWorldQa","index":24.41944421420122,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"mmmu","benchmarkName":"Massive Multi-discipline Multimodal Understanding","category":"multimodal","score":46,"unit":"percent","family":"mmmu","index":11.663788923581933,"aboveFit":false,"run":null,"source":"MMMU authors","sourceUrl":"https://arxiv.org/abs/2401.05508","measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":10,"benchmarkCount":4,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"zerank-2","providerId":"zeroentropy","slug":"zerank-2","name":"ZeroEntropy zerank-2","provider":"ZeroEntropy","description":"ZeroEntropy zerank-2 is a standard model from ZeroEntropy. It has 2 scored results in 1 category. It has no index.","logoUrl":null,"releaseDate":null,"contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":65.7788145926991,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{"reasoning":2},"benchmarks":[{"benchmarkId":"jevBench14","benchmarkName":"jevBench14","category":"reasoning","score":40.20589573277385,"unit":"percent","family":"jevBench14","index":80.00294899628594,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null},{"benchmarkId":"jevBench15","benchmarkName":"jevBench15","category":"reasoning","score":0.13334937595297605,"unit":"percent","family":"jevBench15","index":51.55468018911228,"aboveFit":false,"run":null,"source":"","sourceUrl":null,"measuredAt":null,"domain":null,"eligible":true,"ranked":true,"stderr":null,"costPerTest":null,"latency":null}],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":15,"benchmarkCount":2,"categoryCount":1,"confidence":"unranked","state":"partial"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]},{"modelId":"zonos-2","providerId":"zyphra","slug":"zonos-2","name":"Zonos 2","provider":"Zyphra","description":"Zonos 2 is a standard model from Zyphra. It has no scored result.","logoUrl":null,"releaseDate":"2026-06-12","contextWindowTokens":null,"modalities":["text"],"openWeights":true,"details":null,"reasoning":false,"overallScore":null,"overallScoreRange":null,"badges":[],"capabilityScores":{"agentic":null,"coding":null,"reasoning":null,"multimodal":null,"knowledge":null,"multilingual":null,"instruction":null,"math":null},"capabilityCounts":{},"benchmarks":[],"price":{"inputPerMillion":null,"outputPerMillion":null,"batch":null,"freeTier":false,"state":"not-reported","updatedAt":null},"runtime":{"tokensPerSecond":null,"timeToFirstToken":null,"source":null,"state":"not-reported","updatedAt":null},"arena":null,"quality":{"coveragePercent":0,"benchmarkCount":0,"categoryCount":0,"confidence":"unranked","state":"not-reported"},"hosts":[],"sources":[{"id":"benchlm","name":"BenchLM","url":"https://benchlm.ai","license":"CC BY-NC 4.0","retrievedAt":"2026-11-12T14:39:40.309Z"}]}],"benchmarks":[{"id":"aokvqa","slug":"aokvqa","name":"A Benchmark for Visual Question Answering using World Knowledge","description":"A visual question answering benchmark whose questions cannot be answered from the image alone and require commonsense or world knowledge.","category":"multimodal","domain":null,"unit":"percent","publisher":"Dustin Schwenk et al.","url":"https://arxiv.org/abs/2206.01718","year":"2022","tasks":"Knowledge-grounded visual question answering","counted":false,"results":1,"models":1},{"id":"aaBriefcaseRubric","slug":"aa-briefcase-rubric","name":"AA-Briefcase v1.1, rubric pass rate","description":"The rubric pass rate on Artificial Analysis's AA-Briefcase v1.1 knowledge-work benchmark, alongside its Elo headline.","category":"agentic","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"Professional knowledge-work briefs","counted":false,"results":5,"models":5},{"id":"lmarena:agent-bash-recovery:overall","slug":"lmarena-agent-bash-recovery","name":"Agent Arena command recovery","description":"Estimated effect of the model on how quickly an agent recovers from a failed shell command.","category":"agentic","domain":null,"unit":"index","publisher":"LMArena","url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset","year":null,"tasks":null,"counted":true,"results":50,"models":50},{"id":"lmarena:agent-steerability:overall","slug":"lmarena-agent-steerability","name":"Agent Arena steerability","description":"Estimated effect of the model on whether an agent acts on a correction the user gives it.","category":"agentic","domain":null,"unit":"index","publisher":"LMArena","url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset","year":null,"tasks":null,"counted":true,"results":54,"models":54},{"id":"lmarena:agent-task-outcome:overall","slug":"lmarena-agent-task-outcome","name":"Agent Arena task outcome","description":"Estimated effect of the model on whether a user marks an agent session as finished.","category":"agentic","domain":null,"unit":"index","publisher":"LMArena","url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset","year":null,"tasks":null,"counted":true,"results":54,"models":54},{"id":"agentIf","slug":"agent-if","name":"AgentIF: instruction following in agentic scenarios","description":"Instruction-following benchmark built from real agentic applications, with long instructions that carry many constraints.","category":"instruction","domain":null,"unit":"percent","publisher":"AgentIF authors","url":"https://agentif.github.io/","year":"2025","tasks":"Agentic system prompts and instructions","counted":false,"results":2,"models":2},{"id":"agentsLastExam","slug":"agents-last-exam","name":"Agents' Last Exam","description":"An agent benchmark reported in DeepSeek's V4 Flash 0731 launch comparison.","category":"agentic","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"Agent tasks","counted":true,"results":21,"models":21},{"id":"agentsLastExamCli","slug":"agents-last-exam-cli","name":"Agents' Last Exam v1.1, CLI subset","description":"The command-line subset of Agents' Last Exam v1.1: 86 of the release's 122 agent tasks.","category":"agentic","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"86 CLI agent tasks","counted":false,"results":2,"models":2},{"id":"agentsLastExamLinux","slug":"agents-last-exam-linux","name":"Agents' Last Exam, Linux subset","description":"The Linux subset of Agents' Last Exam, an end-to-end agent task benchmark, as reported by Meta.","category":"agentic","domain":null,"unit":"percent","publisher":"Meta (reported run)","url":"https://research.meta.ai/blog/muse-ember","year":"2026","tasks":"Linux agent tasks (subset size not published)","counted":false,"results":3,"models":3},{"id":"agieval","slug":"agieval","name":"AGIEval","description":"A human-centric exam benchmark for general knowledge and reasoning reported in DeepSeek-V4 base-model evaluations.","category":"knowledge","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"General academic and professional exam questions","counted":false,"results":1,"models":1},{"id":"aiNeedle","slug":"ai-needle","name":"AI-Needle","description":"A long-context retrieval benchmark that measures whether a model can recover relevant information embedded deep inside very long contexts.","category":"reasoning","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"Long-context retrieval","counted":false,"results":4,"models":4},{"id":"ai2dTest","slug":"ai2d-test","name":"AI2D test split","description":"A diagram understanding benchmark focused on scientific and educational visual question answering.","category":"multimodal","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"Diagram understanding","counted":false,"results":8,"models":8},{"id":"vals-ai:aime:overall","slug":"vals-ai-aime","name":"AIME","description":"Challenging national math exam given to top high-school students","category":"math","domain":null,"unit":"percent","publisher":"Vals AI","url":"https://www.vals.ai/benchmarks/aime","year":null,"tasks":null,"counted":true,"results":95,"models":94},{"id":"aime2026","slug":"aime2026","name":"AIME 2026","description":"A 2026 American Invitational Mathematics Examination snapshot used in frontier-model comparison tables for mathematical reasoning.","category":"math","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"Competition math problems","counted":true,"results":34,"models":34},{"id":"aime2025Arcee","slug":"aime2025-arcee","name":"AIME25 first-party comparison snapshot","description":null,"category":"math","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"15 problems","counted":false,"results":6,"models":6},{"id":"aime2024","slug":"aime2024","name":"American Invitational Mathematics Examination 2024","description":"The 2024 edition of AIME, maintaining the same format of 15 challenging mathematics problems with integer answers from 000 to 999.","category":"math","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"15 problems","counted":true,"results":27,"models":27},{"id":"aime2025","slug":"aime2025","name":"American Invitational Mathematics Examination 2025","description":"The most recent AIME examination, featuring 15 challenging mathematics problems testing olympiad-level mathematical reasoning with integer answers from 000-999.","category":"math","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"15 problems","counted":true,"results":53,"models":53},{"id":"androidWorld","slug":"android-world","name":"AndroidWorld","description":"A mobile GUI agent benchmark for completing Android app workflows and on-device tasks.","category":"agentic","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"Android app workflows","counted":true,"results":10,"models":10},{"id":"biomedicalImageAnalysis","slug":"biomedical-image-analysis","name":"Anthropic biomedical-image-analysis evaluation","description":"Quantitative biomedical image analysis across segmentation, detection, measurement, classification, registration, reconstruction, and tracking tasks.","category":"multimodal","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"91 cases across 31 biomedical imaging tasks","counted":false,"results":4,"models":4},{"id":"cveRediscovery","slug":"cve-rediscovery","name":"Anthropic CVE Rediscovery","description":"Anthropic's internal evaluation of whether a model can independently rediscover known vulnerabilities from 1,829 CVE advisories, given only the affected source code.","category":"agentic","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"1,829 CVE advisories","counted":false,"results":3,"models":3},{"id":"proteinBinderDesign","slug":"protein-binder-design","name":"Anthropic de novo protein-binder design evaluation","description":"Designs diverse miniprotein binders against target protein structures in a GPU-equipped research sandbox.","category":"knowledge","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"15 protein targets","counted":false,"results":4,"models":4},{"id":"medicinalChemistry","slug":"medicinal-chemistry","name":"Anthropic medicinal-chemistry evaluation","description":"Selects compounds with the highest or lowest measured ADME-related property from molecular structures and reference values.","category":"knowledge","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"504 molecular-property questions","counted":false,"results":4,"models":4},{"id":"organicChemistryV2","slug":"organic-chemistry-v2","name":"Anthropic Organic Chemistry V2 evaluation","description":"Chemistry tasks covering spectroscopy, synthesis planning, reaction prediction, and chemical structure images.","category":"knowledge","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"Organic chemistry reasoning tasks","counted":false,"results":1,"models":1},{"id":"proteinDesign","slug":"protein-design","name":"Anthropic Protein Design evaluation","description":"Generates novel protein sequences under family, topology, globularity, and structural-motif constraints.","category":"knowledge","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"Constrained protein-sequence design tasks","counted":false,"results":4,"models":4},{"id":"proteinDesignLibraryRanking","slug":"protein-design-library-ranking","name":"Anthropic protein-design library-ranking task","description":"Ranks held-out protein designs for experimental testing using wet-lab campaign measurements and a selection objective.","category":"knowledge","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"83 protein-design library-ranking problems","counted":false,"results":2,"models":2},{"id":"stemProfessorBench","slug":"stem-professor-bench","name":"Anthropic STEM Professor Bench","description":null,"category":"reasoning","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"64 research problems","counted":false,"results":3,"models":3},{"id":"apex","slug":"apex","name":"Apex","description":"A high-difficulty mathematical reasoning benchmark reported in DeepSeek-V4 model evaluations.","category":"math","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"Advanced mathematical reasoning","counted":false,"results":9,"models":9},{"id":"apexShortlist","slug":"apex-shortlist","name":"Apex Shortlist","description":"A shortlist subset of the Apex mathematical reasoning benchmark reported in DeepSeek-V4 model evaluations.","category":"math","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"Advanced mathematical reasoning","counted":false,"results":5,"models":5},{"id":"apexAgents","slug":"apex-agents","name":"APEX-Agents","description":"A professional-services agent benchmark covering long-horizon knowledge-work tasks.","category":"agentic","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"Professional-services agent tasks","counted":true,"results":13,"models":13},{"id":"apexAgentsAa","slug":"apex-agents-aa","name":"APEX-Agents-AA","description":"Artificial Analysis' implementation of the APEX-Agents benchmark for long-horizon professional-services agent tasks.","category":"agentic","domain":null,"unit":"percent","publisher":"Artificial Analysis / Mercor","url":"https://artificialanalysis.ai/evaluations/apex-agents-aa","year":"2026","tasks":"452 professional-services agent tasks","counted":true,"results":30,"models":30},{"id":"apprenticeBench","slug":"apprentice-bench","name":"ApprenticeBench: end-to-end computer use, continual learning, and long-horizon agency on a real accounts-payable job","description":"Tests whether a computer-use agent can learn a real accounts-payable job on the job, processing 100 vendor bills in a company ERP system with only the handbook, historical records, and mentor feedback a new hire would get.","category":"agentic","domain":null,"unit":"percent","publisher":"NeoCognition","url":"https://neocognition.io/blog/apprentice-bench/","year":"2026","tasks":"100 vendor bills processed in sequence inside a simulated construction company","counted":true,"results":18,"models":18},{"id":"arc-prize:v1-public-eval:overall","slug":"arc-prize-v1-public-eval","name":"ARC-AGI-1 (public eval)","description":"Public set of abstract reasoning puzzles.","category":"reasoning","domain":null,"unit":"percent","publisher":"ARC Prize Foundation","url":"https://arcprize.org/leaderboard","year":null,"tasks":null,"counted":false,"results":116,"models":116},{"id":"arc-prize:v1-semi-private:overall","slug":"arc-prize-v1-semi-private","name":"ARC-AGI-1 (semi-private)","description":"Abstract reasoning puzzles held out from training data.","category":"reasoning","domain":null,"unit":"percent","publisher":"ARC Prize Foundation","url":"https://arcprize.org/leaderboard","year":null,"tasks":null,"counted":true,"results":126,"models":126},{"id":"arc-prize:v2-public-eval:overall","slug":"arc-prize-v2-public-eval","name":"ARC-AGI-2 (public eval)","description":"Public set of abstract reasoning puzzles.","category":"reasoning","domain":null,"unit":"percent","publisher":"ARC Prize Foundation","url":"https://arcprize.org/leaderboard","year":null,"tasks":null,"counted":false,"results":122,"models":122},{"id":"arc-prize:v2-semi-private:overall","slug":"arc-prize-v2-semi-private","name":"ARC-AGI-2 (semi-private)","description":"Abstract reasoning puzzles held out from training data.","category":"reasoning","domain":null,"unit":"percent","publisher":"ARC Prize Foundation","url":"https://arcprize.org/leaderboard","year":null,"tasks":null,"counted":true,"results":131,"models":131},{"id":"arc-prize:v3-semi-private:overall","slug":"arc-prize-v3-semi-private","name":"ARC-AGI-3 (semi-private)","description":"Abstract reasoning puzzles held out from training data.","category":"reasoning","domain":null,"unit":"percent","publisher":"ARC Prize Foundation","url":"https://arcprize.org/leaderboard","year":null,"tasks":null,"counted":true,"results":26,"models":26},{"id":"aaAime2025","slug":"aa-aime2025","name":"Artificial Analysis AIME 2025","description":"An independently evaluated AIME 2025 result from Artificial Analysis.","category":"math","domain":null,"unit":"percent","publisher":"Artificial Analysis","url":"https://artificialanalysis.ai/evaluations/aime-2025","year":"2026","tasks":"30 AIME 2025 problems","counted":false,"results":12,"models":12},{"id":"aaAnalystAgent","slug":"aa-analyst-agent","name":"Artificial Analysis AnalystAgent","description":"Artificial Analysis' data-analysis benchmark, testing agents on spreadsheet and document work to answer the quantitative questions a business or data analyst faces day to day.","category":"agentic","domain":null,"unit":"percent","publisher":"Artificial Analysis","url":"https://artificialanalysis.ai/evaluations/aa-analyst-agent","year":"2026","tasks":"Spreadsheet and document analysis questions","counted":true,"results":26,"models":26},{"id":"aaAutomationBench","slug":"aa-automation-bench","name":"Artificial Analysis AutomationBench","description":"An independently evaluated automation benchmark from Artificial Analysis.","category":"agentic","domain":null,"unit":"percent","publisher":"Artificial Analysis","url":"https://artificialanalysis.ai/evaluations/automationbench-aa","year":"2026","tasks":"Business-process automation tasks","counted":true,"results":23,"models":23},{"id":"aaAutomationBenchStrict","slug":"aa-automation-bench-strict","name":"Artificial Analysis AutomationBench, strict pass@1","description":"The strict pass@1 score on Artificial Analysis's AutomationBench-AA, where a business-workflow task counts only when it is fully completed.","category":"agentic","domain":null,"unit":"percent","publisher":"Artificial Analysis","url":"https://artificialanalysis.ai/evaluations/automationbench-aa","year":"2026","tasks":"End-to-end business-workflow automation tasks","counted":false,"results":3,"models":3},{"id":"aaCweBench","slug":"aa-cwe-bench","name":"Artificial Analysis CWE-Bench-AA","description":"Artificial Analysis' implementation of Collinear AI's CWE-bench, testing whether agents can patch real-world software vulnerabilities. One of the three components of the AA Cyber Index.","category":"agentic","domain":null,"unit":"percent","publisher":"Artificial Analysis; dataset by Collinear AI","url":"https://artificialanalysis.ai/evaluations/cwe-bench-aa","year":"2026","tasks":"Real-world vulnerability patching tasks","counted":true,"results":19,"models":19},{"id":"aaCyberIndex","slug":"aa-cyber-index","name":"Artificial Analysis Cyber Index","description":"Artificial Analysis' composite index of cyber-defense capability: finding vulnerabilities in real codebases, reproducing and validating them, and patching them without breaking existing functionality. It averages CWE-Bench-AA, DeepsecBench-AA and CyberGym-E2E-AA.","category":"agentic","domain":null,"unit":"percent","publisher":"Artificial Analysis with the Cyber Index Alliance (Collinear AI et al.","url":"https://artificialanalysis.ai/evaluations/artificial-analysis-cyber-index","year":"2026","tasks":"Three defensive cybersecurity evaluations","counted":true,"results":19,"models":19},{"id":"aaCyberGymE2e","slug":"aa-cyber-gym-e2e","name":"Artificial Analysis CyberGym-E2E-AA","description":"Artificial Analysis' implementation of Berkeley RDI's CyberGym-E2E: agents find a real memory-safety vulnerability in a C/C++ open-source project, write a proof-of-concept input that crashes it, and patch it without breaking the project's tests.","category":"agentic","domain":null,"unit":"percent","publisher":"Artificial Analysis; dataset by Berkeley RDI","url":"https://artificialanalysis.ai/evaluations/cybergym-e2e-aa","year":"2026","tasks":"131 end-to-end vulnerability reproduction and patching tasks","counted":true,"results":19,"models":19},{"id":"aaDeepsecBench","slug":"aa-deepsec-bench","name":"Artificial Analysis DeepsecBench-AA","description":"Artificial Analysis' implementation of Vercel's DeepsecBench, testing agents on finding security vulnerabilities in real web application code. One of the three components of the AA Cyber Index.","category":"agentic","domain":null,"unit":"percent","publisher":"Artificial Analysis; dataset by Vercel","url":"https://artificialanalysis.ai/evaluations/deepsecbench-aa","year":"2026","tasks":"Vulnerability discovery in real codebases","counted":true,"results":21,"models":21},{"id":"aaEnterpriseOpsGym","slug":"aa-enterprise-ops-gym","name":"Artificial Analysis EnterpriseOps-Gym","description":"An independently evaluated enterprise-operations benchmark from Artificial Analysis.","category":"agentic","domain":null,"unit":"percent","publisher":"Artificial Analysis","url":"https://artificialanalysis.ai/evaluations/enterprise-ops-gym-aa","year":"2026","tasks":"Enterprise operations workflows","counted":false,"results":23,"models":23},{"id":"aaGdpPdf","slug":"aa-gdp-pdf","name":"Artificial Analysis GDP.pdf","description":"Artificial Analysis' document-output benchmark: professional tasks whose deliverable is a produced PDF, one of the ten components of its Intelligence Index v4.3.","category":"agentic","domain":null,"unit":"percent","publisher":"Artificial Analysis","url":"https://artificialanalysis.ai/evaluations/gdp-pdf","year":"2026","tasks":"Professional document-production tasks","counted":true,"results":22,"models":22},{"id":"aaGlobalMmluLite","slug":"aa-global-mmlu-lite","name":"Artificial Analysis Global-MMLU-Lite","description":"Global MMLU Lite is a multilingual version of MMLU with 400 questions per language, half culturally sensitive and half culturally agnostic. This page holds two independent runs: Artificial Analysis's, in the main table, and Kaggle's.","category":"multilingual","domain":null,"unit":"percent","publisher":"Artificial Analysis","url":"https://artificialanalysis.ai/evaluations/global-mmlu-lite","year":"2026","tasks":"Multilingual knowledge questions","counted":false,"results":14,"models":14},{"id":"aaGpqaDiamond","slug":"aa-gpqa-diamond","name":"Artificial Analysis GPQA Diamond","description":"An Artificial Analysis GPQA Diamond score.","category":"knowledge","domain":null,"unit":"percent","publisher":"Artificial Analysis","url":"https://artificialanalysis.ai/evaluations/gpqa-diamond","year":"2026","tasks":"Graduate-level science questions","counted":true,"results":200,"models":200},{"id":"aaHarveyLab","slug":"aa-harvey-lab","name":"Artificial Analysis Harvey LAB-AA v1.0","description":"Archived v1.0 legal-agent results from Artificial Analysis, kept separate from v1.1.","category":"agentic","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"Legal agent tasks","counted":false,"results":12,"models":12},{"id":"aaHealthBenchLengthAdjusted","slug":"aa-health-bench-length-adjusted","name":"Artificial Analysis HealthBench length-adjusted score","description":"Artificial Analysis independently evaluates 5,000 conversations and reports a length-adjusted rubric score.","category":"knowledge","domain":null,"unit":"percent","publisher":"OpenAI (benchmark); Artificial Analysis (evaluation)","url":"https://artificialanalysis.ai/evaluations/healthbench","year":"2026","tasks":"5,000 conversations","counted":false,"results":6,"models":6},{"id":"aaHealthBenchProfessional","slug":"aa-health-bench-professional","name":"Artificial Analysis HealthBench Professional length-adjusted score","description":"Artificial Analysis independently evaluates 525 conversations and reports a length-adjusted rubric score.","category":"knowledge","domain":null,"unit":"percent","publisher":"OpenAI (benchmark); Artificial Analysis (evaluation)","url":"https://artificialanalysis.ai/evaluations/healthbench-professional","year":"2026","tasks":"525 conversations","counted":false,"results":6,"models":6},{"id":"aaHealthBenchProfessionalRaw","slug":"aa-health-bench-professional-raw","name":"Artificial Analysis HealthBench Professional raw score","description":"Artificial Analysis independently evaluates 525 conversations and reports a raw rubric score.","category":"knowledge","domain":null,"unit":"percent","publisher":"OpenAI (benchmark); Artificial Analysis (evaluation)","url":"https://artificialanalysis.ai/evaluations/healthbench-professional","year":"2026","tasks":"525 conversations","counted":false,"results":6,"models":6},{"id":"aaHealthBench","slug":"aa-health-bench","name":"Artificial Analysis HealthBench raw score","description":"Artificial Analysis independently evaluates 5,000 conversations and reports a raw rubric score.","category":"knowledge","domain":null,"unit":"percent","publisher":"OpenAI (benchmark); Artificial Analysis (evaluation)","url":"https://artificialanalysis.ai/evaluations/healthbench","year":"2026","tasks":"5,000 conversations","counted":false,"results":6,"models":6},{"id":"aaHle","slug":"aa-hle","name":"Artificial Analysis Humanity's Last Exam","description":"An Artificial Analysis Humanity's Last Exam score.","category":"knowledge","domain":null,"unit":"percent","publisher":"Artificial Analysis","url":"https://artificialanalysis.ai/evaluations/hle","year":"2026","tasks":"Expert-level questions","counted":true,"results":223,"models":223},{"id":"aaIfBench","slug":"aa-if-bench","name":"Artificial Analysis IFBench","description":"An Artificial Analysis IFBench score.","category":"instruction","domain":null,"unit":"percent","publisher":"Artificial Analysis","url":"https://artificialanalysis.ai/evaluations/ifbench","year":"2026","tasks":"Verifiable instruction constraints","counted":true,"results":147,"models":147},{"id":"aaItbench","slug":"aa-itbench","name":"Artificial Analysis ITBench-AA","description":"An independently evaluated IT-operations benchmark from Artificial Analysis.","category":"agentic","domain":null,"unit":"percent","publisher":"Artificial Analysis","url":"https://artificialanalysis.ai/evaluations/itbench-aa","year":"2026","tasks":"IT incident-response tasks","counted":true,"results":24,"models":24},{"id":"aaLiveCodeBench","slug":"aa-live-code-bench","name":"Artificial Analysis LiveCodeBench","description":"An independently evaluated LiveCodeBench result from Artificial Analysis.","category":"coding","domain":null,"unit":"percent","publisher":"Artificial Analysis","url":"https://artificialanalysis.ai/evaluations/livecodebench","year":"2026","tasks":"Contamination-resistant coding tasks","counted":true,"results":12,"models":12},{"id":"lcr","slug":"lcr","name":"Artificial Analysis Long Context Reasoning","description":"An Artificial Analysis long-context reasoning evaluation.","category":"reasoning","domain":null,"unit":"percent","publisher":"Artificial Analysis","url":"https://artificialanalysis.ai/evaluations/artificial-analysis-long-context-reasoning","year":"2026","tasks":"Long-context reasoning tasks","counted":true,"results":218,"models":218},{"id":"aaMath500","slug":"aa-math500","name":"Artificial Analysis MATH-500","description":"An independently evaluated MATH-500 result from Artificial Analysis.","category":"math","domain":null,"unit":"percent","publisher":"Artificial Analysis","url":"https://artificialanalysis.ai/evaluations/math-500","year":"2026","tasks":"500 competition mathematics problems","counted":true,"results":9,"models":9},{"id":"aaMmluPro","slug":"aa-mmlu-pro","name":"Artificial Analysis MMLU-Pro","description":"An independently evaluated MMLU-Pro result from Artificial Analysis.","category":"knowledge","domain":null,"unit":"percent","publisher":"Artificial Analysis","url":"https://artificialanalysis.ai/evaluations/mmlu-pro","year":"2026","tasks":"Professional multi-subject questions","counted":true,"results":15,"models":15},{"id":"aaMmmuPro","slug":"aa-mmmu-pro","name":"Artificial Analysis MMMU-Pro","description":"An Artificial Analysis MMMU-Pro score.","category":"multimodal","domain":null,"unit":"percent","publisher":"Artificial Analysis","url":"https://artificialanalysis.ai/evaluations/mmmu-pro","year":"2026","tasks":"Multimodal academic reasoning","counted":true,"results":114,"models":114},{"id":"omniscienceAccuracy","slug":"omniscience-accuracy","name":"Artificial Analysis Omniscience Accuracy","description":"An Artificial Analysis knowledge metric for the proportion of correctly answered questions.","category":"knowledge","domain":null,"unit":"percent","publisher":"Artificial Analysis","url":"https://artificialanalysis.ai/evaluations/omniscience","year":"2026","tasks":"Knowledge questions","counted":true,"results":165,"models":165},{"id":"aaOpennessIndex","slug":"aa-openness-index","name":"Artificial Analysis Openness Index","description":null,"category":"knowledge","domain":null,"unit":"percent","publisher":"Artificial Analysis","url":"https://artificialanalysis.ai/evaluations/artificial-analysis-openness-index","year":"2026","tasks":"Model openness assessment","counted":false,"results":7,"models":7},{"id":"aaSciCode","slug":"aa-sci-code","name":"Artificial Analysis SciCode","description":"An Artificial Analysis SciCode score.","category":"coding","domain":null,"unit":"percent","publisher":"Artificial Analysis","url":"https://artificialanalysis.ai/evaluations/scicode","year":"2026","tasks":"Scientific coding subproblems","counted":true,"results":115,"models":115},{"id":"aaTau3Banking","slug":"aa-tau3-banking","name":"Artificial Analysis Tau3-Banking","description":"An independently evaluated Tau3 banking benchmark from Artificial Analysis.","category":"agentic","domain":null,"unit":"percent","publisher":"Artificial Analysis","url":"https://artificialanalysis.ai/evaluations/tau3-banking","year":"2026","tasks":"Banking tool-use workflows","counted":true,"results":22,"models":22},{"id":"aaTerminalBench21","slug":"aa-terminal-bench21","name":"Artificial Analysis Terminal-Bench v2.1","description":"An independently evaluated Terminal-Bench v2.1 result from Artificial Analysis.","category":"agentic","domain":null,"unit":"percent","publisher":"Artificial Analysis","url":"https://artificialanalysis.ai/evaluations/terminalbench-v2-1","year":"2026","tasks":"Terminal-based agent tasks","counted":true,"results":21,"models":21},{"id":"aaTerminalBench4","slug":"aa-terminal-bench4","name":"Artificial Analysis Terminal-Bench v4.0","description":"An independently evaluated Terminal-Bench v4.0 result from Artificial Analysis, one of the ten components of its Intelligence Index v4.3.","category":"agentic","domain":null,"unit":"percent","publisher":"Artificial Analysis","url":"https://artificialanalysis.ai/evaluations/terminalbench-v4-0","year":"2026","tasks":"Terminal-based agent tasks","counted":true,"results":22,"models":22},{"id":"arxivMathAugust2026WithTools","slug":"arxiv-math-august2026-with-tools","name":"ArXivMath August 2026 with tools","description":"The August 2026 ArXivMath release evaluated with a code-execution sandbox and no internet access.","category":"math","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"57 recent research-mathematics problems","counted":false,"results":2,"models":2},{"id":"arxivMathAugust2026","slug":"arxiv-math-august2026","name":"ArXivMath August 2026 without tools","description":"Final-answer research mathematics problems drawn from recent arXiv papers, including problems based on resolved or refuted conjectures.","category":"math","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"57 recent research-mathematics problems","counted":false,"results":2,"models":2},{"id":"arxivMathJune2026WithTools","slug":"arxiv-math-june2026-with-tools","name":"ArXivMath June 2026 with tools","description":"Final-answer research mathematics problems drawn from recent arXiv abstracts with tool access.","category":"math","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"49 recent research-mathematics problems","counted":false,"results":1,"models":1},{"id":"arxivMathJune2026","slug":"arxiv-math-june2026","name":"ArXivMath June 2026 without tools","description":"Final-answer research mathematics problems drawn from recent arXiv abstracts.","category":"math","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"49 recent research-mathematics problems","counted":false,"results":1,"models":1},{"id":"automationBench","slug":"automation-bench","name":"AutomationBench","description":"An agent benchmark for completing automation workflows in reproducible task environments.","category":"agentic","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"600 public automation tasks","counted":true,"results":32,"models":32},{"id":"automationBenchZapier106","slug":"automation-bench-zapier106","name":"AutomationBench, Zapier private evaluation set release 1.0.6","description":"An agent benchmark for completing business workflows across simulated applications and API endpoints.","category":"agentic","domain":null,"unit":"percent","publisher":"Daniel Shepard and Robin Salimans","url":"https://arxiv.org/abs/2604.18934","year":"2026","tasks":"600+ held-out workflows across 47 simulated applications and six business domains","counted":false,"results":1,"models":1},{"id":"morphologyToMolecule","slug":"morphology-to-molecule","name":"Axiom Bio morphology-to-molecule matching","description":"Matches multi-channel cellular imaging data to the molecule-dose perturbations that produced it.","category":"knowledge","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"Four multi-channel cell-imaging perturbation-matching tasks","counted":false,"results":5,"models":5},{"id":"b3AgentSecurity","slug":"b3-agent-security","name":"b3: Backbone Breaker Benchmark","description":"Lakera's open-source benchmark of how well the LLM inside an AI agent resists attacks such as system-prompt exfiltration, malicious code injection and tool-call manipulation. Higher means more attacks resisted.","category":"agentic","domain":null,"unit":"percent","publisher":"Lakera with the UK AI Security Institute","url":"https://www.lakera.ai/blog/the-backbone-breaker-benchmark","year":"2025","tasks":"Threat snapshots of attacks on agent backends","counted":false,"results":4,"models":4},{"id":"babyVision","slug":"baby-vision","name":"BabyVision","description":"A multimodal benchmark for fine-grained visual perception and grounded reasoning tasks.","category":"multimodal","domain":null,"unit":"percent","publisher":"Meta AI","url":"https://ai.meta.com/static-resource/muse-spark-1-1-evaluation-report","year":"2026","tasks":"Visual perception tasks","counted":true,"results":11,"models":11},{"id":"babyVisionPython","slug":"baby-vision-python","name":"BabyVision with Python","description":"A Python-assisted BabyVision evaluation for fine-grained visual perception and grounded reasoning.","category":"multimodal","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"Visual perception tasks with Python","counted":false,"results":7,"models":7},{"id":"bankerToolBench","slug":"banker-tool-bench","name":"BankerToolBench","description":null,"category":"agentic","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"Finance and banking tool-use tasks","counted":false,"results":3,"models":3},{"id":"benchCad21WithTools","slug":"bench-cad21-with-tools","name":"BenchCAD 2.1 Core, with tools","description":"BenchCAD 2.1 Core with tool access, in which a model writes CAD code that reproduces a target part.","category":"multimodal","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"BenchCAD 2.1 Core set","counted":false,"results":3,"models":3},{"id":"benchCad21","slug":"bench-cad21","name":"BenchCAD 2.1 Core, without tools","description":"Hardware-engineering benchmark in which a model writes CAD code that reproduces a target part, scored 0 to 100 on the Core set.","category":"multimodal","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"BenchCAD 2.1 Core set","counted":false,"results":3,"models":3},{"id":"benchCadVision2CodeWithTools","slug":"bench-cad-vision2-code-with-tools","name":"BenchCAD Vision2Code voxel IoU with tools","description":"Generates CadQuery code from multi-view renders with image inspection and code-execution tools.","category":"multimodal","domain":null,"unit":"percent","publisher":"Zhang et al. and Anthropic","url":"https://arxiv.org/abs/2605.10865","year":"2026","tasks":"1,000-file Vision2Code subset","counted":false,"results":6,"models":6},{"id":"benchCadVision2Code","slug":"bench-cad-vision2-code","name":"BenchCAD Vision2Code voxel IoU without tools","description":"Generates CadQuery code from multi-view renders and scores geometric similarity by voxel intersection-over-union.","category":"multimodal","domain":null,"unit":"percent","publisher":"Zhang et al. and Anthropic","url":"https://arxiv.org/abs/2605.10865","year":"2026","tasks":"1,000-file Vision2Code subset","counted":false,"results":5,"models":5},{"id":"protocolsUnderstanding","slug":"protocols-understanding","name":"Benchling Molecular Biology Protocols Understanding","description":"Extends online molecular-biology protocols in additional directions using document and web-search tools.","category":"knowledge","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"Molecular-biology protocol extension","counted":false,"results":3,"models":3},{"id":"bfclV3","slug":"bfcl-v3","name":"Berkeley Function Calling Leaderboard v3","description":"A function-calling benchmark for tool selection, schema adherence, and argument correctness, covering single-turn, parallel, irrelevance and multi-turn subsets.","category":"agentic","domain":null,"unit":"percent","publisher":"Shishir G. Patil et al.","url":"https://gorilla.cs.berkeley.edu/leaderboard.html","year":"2025","tasks":"Function-calling tasks","counted":false,"results":9,"models":9},{"id":"bfclV4","slug":"bfcl-v4","name":"Berkeley Function Calling Leaderboard v4","description":"A function-calling benchmark for tool selection, schema adherence, and argument correctness.","category":"agentic","domain":null,"unit":"percent","publisher":"Shishir G. Patil et al.","url":"https://gorilla.cs.berkeley.edu/leaderboard.html","year":"2025","tasks":"Function-calling tasks","counted":true,"results":26,"models":26},{"id":"bbh","slug":"bbh","name":"BIG-Bench Hard","description":"A suite of 23 challenging tasks from the BIG-Bench collaborative benchmark where prior language models failed to exceed average human performance, even with chain-of-thought prompting.","category":"reasoning","domain":null,"unit":"percent","publisher":"Mirac Suzgun et al.","url":"https://arxiv.org/abs/2210.09261","year":"2022","tasks":"23 tasks","counted":false,"results":8,"models":8},{"id":"bigCodeBench","slug":"big-code-bench","name":"BigCodeBench","description":"A code-generation benchmark reported in DeepSeek-V4 base-model evaluations.","category":"coding","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"Code generation tasks","counted":false,"results":1,"models":1},{"id":"bioBasicsBench","slug":"bio-basics-bench","name":"BioBasicsBench: everyday wet-lab science","description":"OpenAI and Ligo's benchmark of how well AI agents interpret raw wet-lab evidence, such as gel and blot images, colony plates, microscopy, sequencing files and instrument exports, spot experimental errors and recognize when the data cannot support a conclusion.","category":"reasoning","domain":null,"unit":"percent","publisher":"OpenAI, Ligo","url":"https://openai.com/index/measuring-how-ai-agents-handle-wet-lab-science/","year":"2026","tasks":"100 routine wet-lab analysis tasks (88 multimodal) across five domains","counted":false,"results":7,"models":7},{"id":"vals-ai:biomysterybench:overall","slug":"vals-ai-biomysterybench","name":"BioMysteryBench","description":"Anthropic's benchmark of whether AI agents can solve real-world bioinformatics mysteries from raw data","category":null,"domain":"Science","unit":"percent","publisher":"Vals AI","url":"https://www.vals.ai/benchmarks/biomysterybench","year":null,"tasks":null,"counted":false,"results":30,"models":30},{"id":"bioMysteryBench","slug":"bio-mystery-bench","name":"BioMysteryBench (overall, v11 release)","description":"Agentic investigation of biological datasets, reported as an overall score on the official v11 release.","category":"agentic","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"Biological data investigations","counted":false,"results":1,"models":1},{"id":"bioMysteryBenchHumanDifficult","slug":"bio-mystery-bench-human-difficult","name":"BioMysteryBench Human Difficult","description":"Computational biology challenges with objective answers that remained unsolved by independent human experts.","category":"knowledge","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"Human-difficult computational biology investigations","counted":false,"results":6,"models":6},{"id":"bioMysteryBenchHumanSolvable","slug":"bio-mystery-bench-human-solvable","name":"BioMysteryBench Human Solvable","description":"Computational biology challenges that independent human experts were able to solve.","category":"knowledge","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"Human-solvable computational biology investigations","counted":false,"results":5,"models":5},{"id":"blueprintBench2","slug":"blueprint-bench2","name":"Blueprint-Bench 2","description":"An agentic spatial reasoning benchmark reported as a normalized score.","category":"multimodal","domain":null,"unit":"percent","publisher":"Google DeepMind","url":"https://x.com/GoogleDeepMind","year":"2026","tasks":"Spatial reasoning from blueprints","counted":false,"results":4,"models":4},{"id":"brevityBench","slug":"brevity-bench","name":"Brevity Benchmark (NVIDIA)","description":"NVIDIA benchmark that scores conversational answers on quality and conciseness together.","category":"instruction","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"Conversational prompts (not yet published)","counted":false,"results":2,"models":2},{"id":"browseComp","slug":"browse-comp","name":"BrowseComp","description":"BrowseComp tests whether a browsing agent can find difficult, fact-seeking answers across the web. Its 1,266 problems use short answers that are easier to grade than an open-ended research report.","category":"agentic","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"Research questions requiring browsing","counted":true,"results":63,"models":63},{"id":"browseCompPlus","slug":"browse-comp-plus","name":"BrowseComp-Plus","description":"Evaluates research agents against a fixed document corpus, separately from live-web BrowseComp.","category":"agentic","domain":null,"unit":"percent","publisher":"Zijian Chen et al.","url":"https://arxiv.org/abs/2508.06600","year":"2025","tasks":null,"counted":false,"results":1,"models":1},{"id":"bug-hunt-bench","slug":"bug-hunt-bench","name":"Bug Hunt Bench","description":"A blind-graded coding-agent benchmark with 105 planted bugs across two production TypeScript repositories.","category":"coding","domain":null,"unit":"percent","publisher":"Pawel Huryn","url":"https://bughunt.productcompass.pm/method","year":"2026","tasks":"105 planted bugs across two production TypeScript repositories","counted":true,"results":25,"models":25},{"id":"kimiBusinessOperation","slug":"kimi-business-operation","name":"Business Operation (Internal)","description":"Moonshot AI's internal benchmark for long-horizon management and decision-making in a simulated business environment.","category":"agentic","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"Simulated business management","counted":false,"results":1,"models":1},{"id":"cEval","slug":"c-eval","name":"C-Eval","description":"A Chinese-language academic and professional benchmark spanning humanities, social science, STEM, and applied subjects.","category":"knowledge","domain":null,"unit":"percent","publisher":"C-Eval authors","url":"https://arxiv.org/abs/2305.08322","year":"2023","tasks":"Chinese academic and professional exams","counted":true,"results":12,"models":12},{"id":"vals-ai:case_law_v2:overall","slug":"vals-ai-case-law-v2","name":"CaseLaw v2","description":"Private question-answer benchmark over Canadian court-cases.","category":null,"domain":"Legal","unit":"percent","publisher":"Vals AI","url":"https://www.vals.ai/benchmarks/case_law_v2","year":null,"tasks":null,"counted":false,"results":53,"models":53},{"id":"ccOcr","slug":"cc-ocr","name":"CC-OCR","description":"An OCR-focused benchmark for reading and extracting text from visually complex documents and images.","category":"multimodal","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"Optical character recognition","counted":false,"results":7,"models":7},{"id":"chartographyWithTools","slug":"chartography-with-tools","name":"Chartography with image and code tools","description":"Professional chart understanding with a container, standard libraries, and image cropping.","category":"multimodal","domain":null,"unit":"percent","publisher":"Surge AI and Anthropic","url":"https://surgehq.ai/blog/chartography","year":"2026","tasks":"100 specialized chart types","counted":false,"results":9,"models":9},{"id":"chartography","slug":"chartography","name":"Chartography without tools","description":"Professional chart understanding across 100 specialized chart types with expert-set answer tolerances.","category":"multimodal","domain":null,"unit":"percent","publisher":"Surge AI and Anthropic","url":"https://surgehq.ai/blog/chartography","year":"2026","tasks":"100 specialized chart types","counted":false,"results":9,"models":9},{"id":"charxivOverall","slug":"charxiv-overall","name":"CharXiv Descriptive and Reasoning Combined","description":"A scientific chart understanding benchmark scored across both the descriptive and reasoning splits rather than the reasoning split alone.","category":"multimodal","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"Scientific chart description and reasoning","counted":false,"results":1,"models":1},{"id":"charxiv","slug":"charxiv","name":"CharXiv Reasoning","description":"A scientific chart reasoning benchmark that tests whether models can understand, interpret, and reason about complex scientific visualizations including plots, diagrams, and data charts.","category":"multimodal","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"Scientific chart reasoning","counted":true,"results":49,"models":49},{"id":"charxivNoTools","slug":"charxiv-no-tools","name":"CharXiv Reasoning without tools","description":"Tool-free variant of CharXiv that isolates raw visual reasoning ability without code execution or tool augmentation.","category":"multimodal","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"Scientific chart reasoning (tool-free)","counted":true,"results":17,"models":17},{"id":"epoch-ai:epoch-chess-puzzles:overall","slug":"epoch-ai-epoch-chess-puzzles","name":"Chess Puzzles","description":"Evaluated by Epoch AI on their own harness.","category":"reasoning","domain":null,"unit":"percent","publisher":"Epoch AI","url":"https://epoch.ai/benchmarks","year":null,"tasks":null,"counted":true,"results":90,"models":90},{"id":"cmmlu","slug":"cmmlu","name":"Chinese Massive Multitask Language Understanding","description":"A Chinese multitask academic benchmark reported in DeepSeek-V4 base-model evaluations.","category":"knowledge","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"Chinese academic QA","counted":false,"results":1,"models":1},{"id":"chineseSimpleQa","slug":"chinese-simple-qa","name":"Chinese-SimpleQA","description":"A Chinese short-form factuality benchmark reported by DeepSeek for V4 model evaluations.","category":"knowledge","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"Chinese factual questions","counted":false,"results":2,"models":2},{"id":"clawEval","slug":"claw-eval","name":"Claw-Eval","description":"A transparent real-world autonomous-agent benchmark with 300 human-verified tasks, 2,159 rubric items, and Pass^3 scoring across general, multi-turn, and native multimodal agent tasks.","category":"agentic","domain":null,"unit":"percent","publisher":"Bowen Ye et al.","url":"https://arxiv.org/abs/2604.06132","year":"2026","tasks":"300 tasks, 2,159 rubrics","counted":true,"results":42,"models":42},{"id":"vals-ai:code-migration:overall","slug":"vals-ai-code-migration","name":"Code Migration","description":"Can language models reimplement real-world programs in another language?","category":"coding","domain":null,"unit":"percent","publisher":"Vals AI","url":"https://www.vals.ai/benchmarks/code-migration","year":null,"tasks":null,"counted":true,"results":77,"models":77},{"id":"vals-ai:corp_fin_v2:overall","slug":"vals-ai-corp-fin-v2","name":"CorpFin v2","description":"A private benchmark evaluating understanding of long-context credit agreements","category":null,"domain":"Finance","unit":"percent","publisher":"Vals AI","url":"https://www.vals.ai/benchmarks/corp_fin_v2","year":null,"tasks":null,"counted":false,"results":131,"models":130},{"id":"corpusQa1m","slug":"corpus-qa1m","name":"CorpusQA 1M","description":"A million-token CorpusQA long-context question-answering benchmark reported in DeepSeek-V4 model evaluations.","category":"reasoning","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"Million-token corpus question answering","counted":false,"results":2,"models":2},{"id":"countBench","slug":"count-bench","name":"CountBench","description":"A visual counting benchmark that tests whether a model can count objects and entities reliably in complex scenes.","category":"multimodal","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"Visual counting tasks","counted":true,"results":10,"models":10},{"id":"coworkBench","slug":"cowork-bench","name":"CoWorkBench","description":"Qwen's internal benchmark for long-horizon professional work across computer science, finance, law, medicine, and other productivity domains.","category":"agentic","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"Long-horizon professional workflows","counted":false,"results":4,"models":4},{"id":"critpt","slug":"critpt","name":"Critical Physics Tasks","description":"An Artificial Analysis metric for research-level physics reasoning.","category":"reasoning","domain":null,"unit":"percent","publisher":"Artificial Analysis","url":"https://artificialanalysis.ai/evaluations/critpt","year":"2026","tasks":"Research-level physics questions","counted":true,"results":215,"models":215},{"id":"ctiRealm","slug":"cti-realm","name":"CTI-REALM","description":"A cybersecurity benchmark that measures whether an agent can turn raw threat-intelligence reports into working detection rules.","category":"agentic","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"Threat-intelligence-to-detection-rule workflows","counted":false,"results":1,"models":1},{"id":"vals-ai:cua_bench:overall","slug":"vals-ai-cua-bench","name":"CUA-bench","description":"Can an AI agent play six commercial video games with only a keyboard and a mouse? Half of them are held out.","category":null,"domain":"Other","unit":"percent","publisher":"Vals AI","url":"https://www.vals.ai/benchmarks/cua_bench","year":null,"tasks":null,"counted":false,"results":9,"models":9},{"id":"cursorBench31","slug":"cursor-bench31","name":"cursorBench31","description":null,"category":"coding","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":null,"counted":false,"results":7,"models":7},{"id":"cursorBench32","slug":"cursor-bench32","name":"cursorBench32","description":null,"category":"coding","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":null,"counted":true,"results":18,"models":18},{"id":"cursorBench40","slug":"cursor-bench40","name":"cursorBench40","description":null,"category":"coding","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":null,"counted":true,"results":21,"models":21},{"id":"cweBench","slug":"cwe-bench","name":"CWE-Bench","description":"An external benchmark for evaluating whether coding agents can produce correct patches for real-world software vulnerabilities.","category":"agentic","domain":null,"unit":"percent","publisher":"Collinear","url":"https://cwe-bench.com/#leaderboard","year":"2026","tasks":"Real-world vulnerability patching tasks","counted":false,"results":1,"models":1},{"id":"cweBenchV1","slug":"cwe-bench-v1","name":"CWE-bench v1","description":"Defensive vulnerability patching on 120 held-out tasks spanning 73 weakness types. Replaced by v2 on November 2, 2026.","category":"agentic","domain":null,"unit":"percent","publisher":"Collinear AI","url":"https://cwe-bench.com/","year":"2026","tasks":"Audit and patch held-out C/C++, Go, Java, TypeScript/JavaScript, Python, and Rust repositories","counted":true,"results":12,"models":12},{"id":"cweBenchV2","slug":"cwe-bench-v2","name":"CWE-bench v2","description":"Defensive vulnerability patching on 250 audit-and-patch tasks spanning 185 weakness types, 100 public and 150 held out.","category":"agentic","domain":null,"unit":"percent","publisher":"Collinear AI","url":"https://cwe-bench.com/","year":"2026","tasks":"Audit and patch C/C++, Go, Java, TypeScript/JavaScript, Python, and Rust repositories; 100 public and 150 held-out tasks","counted":true,"results":11,"models":11},{"id":"cybench","slug":"cybench","name":"Cybench","description":"A cybersecurity benchmark of professional Capture the Flag tasks for measuring autonomous cyber agent capability and risk.","category":"agentic","domain":null,"unit":"percent","publisher":"Andy K. Zhang et al.","url":"https://arxiv.org/abs/2408.08926","year":"2025","tasks":"40 professional CTF tasks","counted":false,"results":2,"models":2},{"id":"vals-ai:cyber:overall","slug":"vals-ai-cyber","name":"CyberBench v1.1","description":"Can autonomous agents craft PoC inputs that trigger OSS-Fuzz vulnerabilities—and produce source patches that fix them?","category":null,"domain":"Other","unit":"percent","publisher":"Vals AI","url":"https://www.vals.ai/benchmarks/cyber","year":null,"tasks":null,"counted":false,"results":48,"models":48},{"id":"cyberGym","slug":"cyber-gym","name":"CyberGym","description":"A cybersecurity task benchmark for evaluating defensive cyber workflows and vulnerability-oriented agent performance.","category":"agentic","domain":null,"unit":"percent","publisher":"Zhun Wang et al.","url":"https://www.cybergym.io/","year":"2026","tasks":"1,507 vulnerability analysis instances","counted":true,"results":29,"models":29},{"id":"draco","slug":"draco","name":"Data Research and Analysis with Complex Operations","description":"Agentic data-analysis tasks scored against per-task rubrics at a 980K-token budget.","category":"agentic","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"Agentic data research and analysis tasks","counted":false,"results":10,"models":10},{"id":"deckBench","slug":"deck-bench","name":"DECK-Bench (Internal)","description":"Moonshot AI's internal benchmark for presentation and deck-production workflows.","category":"agentic","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"Internal presentation workflows","counted":false,"results":1,"models":1},{"id":"deepPlanning","slug":"deep-planning","name":"DeepPlanning","description":"A long-horizon planning benchmark that tests whether agents can optimize under explicit time, budget, and feasibility constraints.","category":"agentic","domain":null,"unit":"percent","publisher":"DeepPlanning authors","url":"https://arxiv.org/abs/2601.18137","year":"2026","tasks":"Travel planning and constrained shopping","counted":false,"results":7,"models":7},{"id":"deepSearchQa","slug":"deep-search-qa","name":"DeepSearchQA","description":"An agentic browsing benchmark where models search the web, gather evidence, and answer list-style questions using browser tools.","category":"agentic","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"Agentic browsing and list-answer questions","counted":true,"results":24,"models":24},{"id":"dsBenchFullStack","slug":"ds-bench-full-stack","name":"DeepSeek DSBench FullStack","description":"DeepSeek's internal full-stack coding-agent benchmark.","category":"coding","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"Internal full-stack coding-agent tasks","counted":false,"results":2,"models":2},{"id":"dsBenchHard","slug":"ds-bench-hard","name":"DeepSeek DSBench Hard","description":"DeepSeek's internal hard coding-agent benchmark.","category":"coding","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"Internal hard coding-agent tasks","counted":false,"results":2,"models":2},{"id":"deepSwe","slug":"deep-swe","name":"DeepSWE","description":"A long-horizon software engineering benchmark from Datacurve for measuring frontier coding agents on original tasks drawn from active open-source repositories.","category":"agentic","domain":null,"unit":"percent","publisher":"Datacurve AI","url":"https://deepswe.datacurve.ai/blog","year":"2026","tasks":"113 software engineering tasks across 91 repositories and 5 languages","counted":true,"results":59,"models":59},{"id":"drop","slug":"drop","name":"Discrete Reasoning Over Paragraphs","description":"A reading-comprehension benchmark requiring discrete reasoning over paragraphs, reported in DeepSeek-V4 base-model evaluations.","category":"reasoning","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"Paragraph reasoning questions","counted":false,"results":6,"models":6},{"id":"dynaMath","slug":"dyna-math","name":"DynaMath","description":"A multimodal benchmark for dynamic mathematical reasoning over visual and structured inputs.","category":"multimodal","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"Dynamic visual math problems","counted":false,"results":7,"models":7},{"id":"epoch-ai:epoch-ebr-bench:overall","slug":"epoch-ai-epoch-ebr-bench","name":"EBR-bench","description":"Evaluated by Epoch AI on their own harness.","category":"reasoning","domain":null,"unit":"percent","publisher":"Epoch AI","url":"https://epoch.ai/benchmarks","year":null,"tasks":null,"counted":true,"results":31,"models":31},{"id":"eeBench","slug":"ee-bench","name":"EEBench V1 core corpus","description":"An electrical-engineering agent benchmark maintained by atopile.","category":"coding","domain":null,"unit":"percent","publisher":"atopile","url":"https://www.eebench.org/","year":"2026","tasks":"Electrical-engineering design tasks","counted":false,"results":1,"models":1},{"id":"vals-ai:emb:overall","slug":"vals-ai-emb","name":"EMB","description":"Evaluating agents on Excel-based financial modeling tasks","category":null,"domain":"Finance","unit":"percent","publisher":"Vals AI","url":"https://www.vals.ai/benchmarks/emb","year":null,"tasks":null,"counted":false,"results":75,"models":75},{"id":"enterpriseOpsGym","slug":"enterprise-ops-gym","name":"EnterpriseOps-Gym (provider runs)","description":null,"category":"agentic","domain":null,"unit":"percent","publisher":"ServiceNow","url":"https://arxiv.org/abs/2603.13594","year":"2026","tasks":"Enterprise workflow tasks","counted":false,"results":1,"models":1},{"id":"eqBench3","slug":"eq-bench3","name":"EQ-Bench 3","description":"Measures emotional intelligence in multi-turn roleplay scenarios, scored by an LLM judge against a rubric.","category":"instruction","domain":null,"unit":"percent","publisher":"EQ-Bench","url":"https://eqbench.com/eqbench3.html","year":"2025","tasks":"Multi-turn roleplay scenarios","counted":false,"results":2,"models":2},{"id":"erqa","slug":"erqa","name":"ERQA","description":"A grounded visual reasoning benchmark focused on evidence-based question answering over real images.","category":"multimodal","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"Evidence-based visual QA","counted":true,"results":18,"models":18},{"id":"euroeval:euroeval-dutch:overall","slug":"euroeval-euroeval-dutch","name":"EuroEval Dutch","description":"How well a model handles Dutch, as the unweighted mean of EuroEval's primary metric over its 9 Dutch datasets (dbrd, conll_nl, dutch_cola, squad_nl, wiki_lingua_nl, include_nl, multiloko_nl, winogrande_nl, duidelijke_taal). The primary metric differs by task — MCC for classification, knowledge and common-sense reasoning, micro-F1 without MISC for entity recognition, F1 for reading comprehension, chrF++ for summarization, METEOR for simplification — and is always the first of the two the board prints. Only models measured on every dataset are included, so each mean covers the same ground. Most rows are the validation split; some older open-weight rows are the test split, which EuroEval ranks in the same table.","category":"multilingual","domain":null,"unit":"percent","publisher":"EuroEval","url":"https://euroeval.com/leaderboards/dutch","year":null,"tasks":null,"counted":true,"results":175,"models":168},{"id":"euroeval:euroeval-french:overall","slug":"euroeval-euroeval-french","name":"EuroEval French","description":"How well a model handles French, as the unweighted mean of EuroEval's primary metric over its 8 French datasets (allocine, eltec, scala_fr, fquad, orange_sum, include_fr, multiloko_fr, hellaswag_fr). The primary metric differs by task — MCC for classification, knowledge and common-sense reasoning, micro-F1 without MISC for entity recognition, F1 for reading comprehension, chrF++ for summarization, METEOR for simplification — and is always the first of the two the board prints. Only models measured on every dataset are included, so each mean covers the same ground. Most rows are the validation split; some older open-weight rows are the test split, which EuroEval ranks in the same table.","category":"multilingual","domain":null,"unit":"percent","publisher":"EuroEval","url":"https://euroeval.com/leaderboards/french","year":null,"tasks":null,"counted":true,"results":178,"models":170},{"id":"euroeval:euroeval-german:overall","slug":"euroeval-euroeval-german","name":"EuroEval German","description":"How well a model handles German, as the unweighted mean of EuroEval's primary metric over its 8 German datasets (sb10k, germeval, scala_de, germanquad, mlsum_de, include_de, multiloko_de, winogrande_de). The primary metric differs by task — MCC for classification, knowledge and common-sense reasoning, micro-F1 without MISC for entity recognition, F1 for reading comprehension, chrF++ for summarization, METEOR for simplification — and is always the first of the two the board prints. Only models measured on every dataset are included, so each mean covers the same ground. Most rows are the validation split; some older open-weight rows are the test split, which EuroEval ranks in the same table.","category":"multilingual","domain":null,"unit":"percent","publisher":"EuroEval","url":"https://euroeval.com/leaderboards/german","year":null,"tasks":null,"counted":true,"results":191,"models":183},{"id":"euroeval:euroeval-italian:overall","slug":"euroeval-euroeval-italian","name":"EuroEval Italian","description":"How well a model handles Italian, as the unweighted mean of EuroEval's primary metric over its 8 Italian datasets (sentipolc16, multinerd_it, scala_it, squad_it, ilpost_sum, include_it, multiloko_it, winogrande_it). The primary metric differs by task — MCC for classification, knowledge and common-sense reasoning, micro-F1 without MISC for entity recognition, F1 for reading comprehension, chrF++ for summarization, METEOR for simplification — and is always the first of the two the board prints. Only models measured on every dataset are included, so each mean covers the same ground. Most rows are the validation split; some older open-weight rows are the test split, which EuroEval ranks in the same table.","category":"multilingual","domain":null,"unit":"percent","publisher":"EuroEval","url":"https://euroeval.com/leaderboards/italian","year":null,"tasks":null,"counted":true,"results":174,"models":166},{"id":"euroeval:euroeval-polish:overall","slug":"euroeval-euroeval-polish","name":"EuroEval Polish","description":"How well a model handles Polish, as the unweighted mean of EuroEval's primary metric over its 7 Polish datasets (polemo2, kpwr_ner, scala_pl, poquad, psc, llmzszl, winogrande_pl). The primary metric differs by task — MCC for classification, knowledge and common-sense reasoning, micro-F1 without MISC for entity recognition, F1 for reading comprehension, chrF++ for summarization, METEOR for simplification — and is always the first of the two the board prints. Only models measured on every dataset are included, so each mean covers the same ground. Most rows are the validation split; some older open-weight rows are the test split, which EuroEval ranks in the same table.","category":"multilingual","domain":null,"unit":"percent","publisher":"EuroEval","url":"https://euroeval.com/leaderboards/polish","year":null,"tasks":null,"counted":true,"results":209,"models":200},{"id":"euroeval:euroeval-portuguese:overall","slug":"euroeval-euroeval-portuguese","name":"EuroEval Portuguese","description":"How well a model handles Portuguese, as the unweighted mean of EuroEval's primary metric over its 8 Portuguese datasets (sst2_pt, harem, scala_pt, multi_wiki_qa_pt, publico, alba_mcq_pt, cultura_viva_pt, winogrande_pt). The primary metric differs by task — MCC for classification, knowledge and common-sense reasoning, micro-F1 without MISC for entity recognition, F1 for reading comprehension, chrF++ for summarization, METEOR for simplification — and is always the first of the two the board prints. Only models measured on every dataset are included, so each mean covers the same ground. Most rows are the validation split; some older open-weight rows are the test split, which EuroEval ranks in the same table.","category":"multilingual","domain":null,"unit":"percent","publisher":"EuroEval","url":"https://euroeval.com/leaderboards/portuguese","year":null,"tasks":null,"counted":true,"results":182,"models":174},{"id":"euroeval:euroeval-spanish:overall","slug":"euroeval-euroeval-spanish","name":"EuroEval Spanish","description":"How well a model handles Spanish, as the unweighted mean of EuroEval's primary metric over its 8 Spanish datasets (sentiment_headlines_es, conll_es, scala_es, mlqa_es, mlsum_es, include_es, multiloko_es, winogrande_es). The primary metric differs by task — MCC for classification, knowledge and common-sense reasoning, micro-F1 without MISC for entity recognition, F1 for reading comprehension, chrF++ for summarization, METEOR for simplification — and is always the first of the two the board prints. Only models measured on every dataset are included, so each mean covers the same ground. Most rows are the validation split; some older open-weight rows are the test split, which EuroEval ranks in the same table.","category":"multilingual","domain":null,"unit":"percent","publisher":"EuroEval","url":"https://euroeval.com/leaderboards/spanish","year":null,"tasks":null,"counted":true,"results":176,"models":168},{"id":"euroeval:euroeval-swedish:overall","slug":"euroeval-euroeval-swedish","name":"EuroEval Swedish","description":"How well a model handles Swedish, as the unweighted mean of EuroEval's primary metric over its 8 Swedish datasets (swerec, suc3, scala_sv, multi_wiki_qa_sv, swedn, skolprov, swedish_facts, winogrande_sv). The primary metric differs by task — MCC for classification, knowledge and common-sense reasoning, micro-F1 without MISC for entity recognition, F1 for reading comprehension, chrF++ for summarization, METEOR for simplification — and is always the first of the two the board prints. Only models measured on every dataset are included, so each mean covers the same ground. Most rows are the validation split; some older open-weight rows are the test split, which EuroEval ranks in the same table.","category":"multilingual","domain":null,"unit":"percent","publisher":"EuroEval","url":"https://euroeval.com/leaderboards/swedish","year":null,"tasks":null,"counted":true,"results":195,"models":187},{"id":"humaneval","slug":"humaneval","name":"Evaluating Large Language Models Trained on Code","description":"HumanEval tests whether generated Python functions pass unit tests on 164 handwritten problems. These scores describe function-level correctness; they do not measure repository work or a coding agent.","category":"coding","domain":null,"unit":"percent","publisher":"Mark Chen et al.","url":"https://arxiv.org/abs/2107.03374","year":"2021","tasks":"164 problems","counted":true,"results":22,"models":22},{"id":"exploitGym","slug":"exploit-gym","name":"ExploitGym","description":"A controlled benchmark for evaluating whether AI agents can extend vulnerability-triggering inputs into working exploits.","category":"agentic","domain":null,"unit":"percent","publisher":"Zhun Wang et al.","url":"https://arxiv.org/abs/2605.11086","year":"2026","tasks":"898 exploitation tasks","counted":true,"results":23,"models":23},{"id":"extractBenchPro","slug":"extract-bench-pro","name":"ExtractBench Pro: long-tail document extraction challenges (LlamaIndex)","description":"A harder companion to ExtractBench: 100 synthetic documents and 2,622 pages across 86 document types that test document conventions, house vocabulary, counting rules, drawn relations and charts, scored by perfect-document rate.","category":"multimodal","domain":null,"unit":"percent","publisher":"LlamaIndex","url":"https://www.extractbench.ai/pro","year":"2026","tasks":"100 synthetic documents, 2,622 pages, 86 document types, six challenge groups","counted":false,"results":7,"models":7},{"id":"extractBenchLlamaIndex","slug":"extract-bench-llama-index","name":"ExtractBench: schema-guided enterprise document extraction (LlamaIndex leaderboard)","description":"Scores schema-guided extraction from 370 enterprise documents and 4,869 pages on value F1, record completeness, word and page grounding and cost, split by short, medium and long documents.","category":"multimodal","domain":null,"unit":"percent","publisher":"Boyang Zhang et al.","url":"https://arxiv.org/abs/2607.29677","year":"2026","tasks":"370 documents, 4,869 pages, 8 business domains, 67 document types, each with a JSON Schema","counted":true,"results":26,"models":26},{"id":"vals-ai:fabv2:overall","slug":"vals-ai-fabv2","name":"Finance Agent (v2)","description":"Evaluating agents on core financial analyst tasks using the FAB v2 harness","category":null,"domain":"Finance","unit":"percent","publisher":"Vals AI","url":"https://www.vals.ai/benchmarks/fabv2","year":null,"tasks":null,"counted":false,"results":79,"models":79},{"id":"vals-ai:finance_agent:overall","slug":"vals-ai-finance-agent","name":"Finance Agent v1.1","description":"Evaluating agents on core financial analyst tasks","category":null,"domain":"Finance","unit":"percent","publisher":"Vals AI","url":"https://www.vals.ai/benchmarks/finance_agent","year":null,"tasks":null,"counted":false,"results":51,"models":51},{"id":"kimiFinancialModeling","slug":"kimi-financial-modeling","name":"Financial Modeling (Internal)","description":"Moonshot AI's internal benchmark for formula linkage, financial logic, and final workbook quality in real-world modeling tasks.","category":"agentic","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"Spreadsheet financial models","counted":false,"results":1,"models":1},{"id":"kimiFinancialResearch","slug":"kimi-financial-research","name":"Financial Research (Internal)","description":"Moonshot AI's internal benchmark for auditable financial research under evidence and cutoff constraints.","category":"agentic","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"Source-grounded research reports","counted":false,"results":1,"models":1},{"id":"finch","slug":"finch","name":"Finch (FinWorkBench): Finance & Accounting across Spreadsheet-Centric Enterprise Workflows","description":"A benchmark of real finance and accounting workflows built around spreadsheets: creating, editing and analysing workbooks the way an analyst or accountant would.","category":"agentic","domain":null,"unit":"percent","publisher":"Finch authors (ACL 2026 Findings)","url":"https://arxiv.org/abs/2512.13168","year":"2026","tasks":"Spreadsheet-centric finance and accounting workflows","counted":false,"results":4,"models":4},{"id":"frontierFinance","slug":"frontier-finance","name":"Frontier Finance","description":"An investment-workflow benchmark covering complete tasks from screening to profitability analysis.","category":"agentic","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"Investment research workflows","counted":false,"results":1,"models":1},{"id":"frontierCode11Extended","slug":"frontier-code11-extended","name":"FrontierCode 1.1 Extended","description":"Cognition's 150-task Extended subset of the FrontierCode 1.1 software-engineering benchmark.","category":"coding","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"150 private software-engineering tasks","counted":false,"results":8,"models":8},{"id":"frontierCode","slug":"frontier-code","name":"FrontierCode 1.1 Main","description":"Cognition's 100-task software-engineering benchmark for whether coding agents produce mergeable, production-quality pull requests, scored for correctness, tests, scope, style, and maintainability through maintainer-authored rubrics.","category":"coding","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"100 private Main tasks (150 in Extended)","counted":true,"results":17,"models":17},{"id":"frontierCode2","slug":"frontier-code2","name":"FrontierCode 2.0","description":"Cognition's second-generation coding-agent benchmark, scoring whether agents produce mergeable pull requests on larger, less fully specified tasks, with added emphasis on thoroughness such as fixing every instance of a bug.","category":"coding","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"132 private software-engineering tasks","counted":true,"results":27,"models":27},{"id":"aaFrontierCode2","slug":"aa-frontier-code2","name":"FrontierCode 2.0, Artificial Analysis evaluation","description":"Artificial Analysis's own run of Cognition's FrontierCode 2.0 coding-agent benchmark, one of the nine evaluations in its Intelligence Index v5, judged on whether each pull request is mergeable.","category":"coding","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"132 private software-engineering tasks","counted":false,"results":3,"models":3},{"id":"epoch-ai:epoch-frontiermath-2025-02-28-private:overall","slug":"epoch-ai-epoch-frontiermath-2025-02-28-private","name":"FrontierMath-2025-02-28-Private","description":"Evaluated by Epoch AI on their own harness.","category":"math","domain":null,"unit":"percent","publisher":"Epoch AI","url":"https://epoch.ai/benchmarks","year":null,"tasks":null,"counted":true,"results":69,"models":69},{"id":"epoch-ai:epoch-frontiermath-2025-02-28-public:overall","slug":"epoch-ai-epoch-frontiermath-2025-02-28-public","name":"FrontierMath-2025-02-28-Public","description":"Evaluated by Epoch AI on their own harness.","category":"math","domain":null,"unit":"percent","publisher":"Epoch AI","url":"https://epoch.ai/benchmarks","year":null,"tasks":null,"counted":false,"results":64,"models":64},{"id":"epoch-ai:epoch-frontiermath-erdos:overall","slug":"epoch-ai-epoch-frontiermath-erdos","name":"FrontierMath-Erdos","description":"Evaluated by Epoch AI on their own harness.","category":"math","domain":null,"unit":"percent","publisher":"Epoch AI","url":"https://epoch.ai/benchmarks","year":null,"tasks":null,"counted":false,"results":15,"models":15},{"id":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-private:overall","slug":"epoch-ai-epoch-frontiermath-tier-4-2025-07-01-private","name":"FrontierMath-Tier-4-2025-07-01-Private","description":"Evaluated by Epoch AI on their own harness.","category":"math","domain":null,"unit":"percent","publisher":"Epoch AI","url":"https://epoch.ai/benchmarks","year":null,"tasks":null,"counted":true,"results":55,"models":55},{"id":"epoch-ai:epoch-frontiermath-tier-4-2025-07-01-public:overall","slug":"epoch-ai-epoch-frontiermath-tier-4-2025-07-01-public","name":"FrontierMath-Tier-4-2025-07-01-Public","description":"Evaluated by Epoch AI on their own harness.","category":"math","domain":null,"unit":"percent","publisher":"Epoch AI","url":"https://epoch.ai/benchmarks","year":null,"tasks":null,"counted":false,"results":36,"models":36},{"id":"epoch-ai:epoch-frontiermath-tier-4-v2-private:overall","slug":"epoch-ai-epoch-frontiermath-tier-4-v2-private","name":"FrontierMath-Tier-4-v2-Private","description":"Evaluated by Epoch AI on their own harness.","category":"math","domain":null,"unit":"percent","publisher":"Epoch AI","url":"https://epoch.ai/benchmarks","year":null,"tasks":null,"counted":true,"results":72,"models":72},{"id":"epoch-ai:epoch-frontiermath-tiers-1-3-v2-private:overall","slug":"epoch-ai-epoch-frontiermath-tiers-1-3-v2-private","name":"FrontierMath-Tiers-1-3-v2-Private","description":"Evaluated by Epoch AI on their own harness.","category":"math","domain":null,"unit":"percent","publisher":"Epoch AI","url":"https://epoch.ai/benchmarks","year":null,"tasks":null,"counted":true,"results":90,"models":90},{"id":"frontierScience","slug":"frontier-science","name":"FrontierScience","description":"A benchmark for research-level scientific reasoning, designed to separate frontier models on difficult science tasks that mix domain knowledge with deep reasoning.","category":"knowledge","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"Research-level science tasks","counted":false,"results":2,"models":2},{"id":"frontierScienceResearch","slug":"frontier-science-research","name":"FrontierScience Research","description":"A research-focused FrontierScience evaluation variant for scientific investigation and problem solving.","category":"knowledge","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"Scientific research problems","counted":false,"results":5,"models":5},{"id":"frontierSwe","slug":"frontier-swe","name":"FrontierSWE","description":"An ultra-long-horizon software-engineering benchmark with open-ended implementation, performance, and research tasks designed to challenge frontier coding agents.","category":"coding","domain":null,"unit":"percent","publisher":"Evan Chu et al.","url":"https://www.frontierswe.com/blog","year":"2026","tasks":"17 ultra-long-horizon engineering and research tasks","counted":false,"results":3,"models":3},{"id":"frontierSweV2","slug":"frontier-swe-v2","name":"FrontierSWE v2","description":"A 34-task expansion of FrontierSWE for ultra-long-horizon engineering and research work that remains far from saturation.","category":"coding","domain":null,"unit":"percent","publisher":"Proximal","url":"https://www.frontierswe.com/blog/v2","year":"2026","tasks":"34 ultra-long-horizon engineering and research tasks","counted":true,"results":22,"models":22},{"id":"epoch-ai:epoch-furniture-assembly:overall","slug":"epoch-ai-epoch-furniture-assembly","name":"Furniture Assembly","description":"Evaluated by Epoch AI on their own harness.","category":"reasoning","domain":null,"unit":"percent","publisher":"Epoch AI","url":"https://epoch.ai/benchmarks","year":null,"tasks":null,"counted":true,"results":37,"models":37},{"id":"garage","slug":"garage","name":"GaRAGe","description":"Amazon benchmark of grounded answers in retrieval-augmented generation, with human labels for which supplied passages are relevant.","category":"knowledge","domain":null,"unit":"percent","publisher":"Amazon Science","url":"https://github.com/amazon-science/GaRAGe","year":"2025","tasks":"Questions with retrieved passages","counted":false,"results":2,"models":2},{"id":"gdpPdfWithTools","slug":"gdp-pdf-with-tools","name":"GDP.pdf mean criteria pass rate with tools","description":"Professional document understanding with a container, standard libraries, and image cropping.","category":"multimodal","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"100 professional document prompts","counted":false,"results":2,"models":2},{"id":"gdpPdf","slug":"gdp-pdf","name":"GDP.pdf mean criteria pass rate without tools","description":"Professional document understanding over 100 real-world PDFs from ten domains.","category":"multimodal","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"100 professional document prompts","counted":false,"results":3,"models":3},{"id":"gdpPdfGeminiJudge","slug":"gdp-pdf-gemini-judge","name":"GDP.pdf strict pass rate, page images plus text, Gemini judge","description":"Professional document understanding over real-world PDFs, graded per rubric criterion by Gemini-3.5-Flash.","category":"multimodal","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"Professional document prompts","counted":false,"results":1,"models":1},{"id":"gdpPdfPythonGeminiJudge","slug":"gdp-pdf-python-gemini-judge","name":"GDP.pdf task-macro rubric pass rate with Python, Gemini judge","description":"Professional document understanding where the model receives each original PDF in a sandbox with an IPython tool.","category":"multimodal","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"Professional document prompts with Python","counted":false,"results":1,"models":1},{"id":"gdpvalRubrics","slug":"gdpval-rubrics","name":"GDPval rubrics","description":null,"category":"agentic","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"Economically valuable work tasks","counted":false,"results":2,"models":2},{"id":"gdpvalAaNormalized","slug":"gdpval-aa-normalized","name":"GDPval-AA normalized","description":"An Artificial Analysis normalized score for economically valuable tasks.","category":"agentic","domain":null,"unit":"percent","publisher":"Artificial Analysis","url":"https://artificialanalysis.ai/evaluations/gdpval-aa","year":"2026","tasks":"Economically valuable tasks","counted":true,"results":134,"models":134},{"id":"geneBenchPro","slug":"gene-bench-pro","name":"GeneBench-Pro","description":"A multistage statistical-reasoning benchmark for genomics and biological-data analysis agents.","category":"reasoning","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"129 genomics statistical-analysis workflows","counted":false,"results":2,"models":2},{"id":"gaia","slug":"gaia","name":"General AI Assistants","description":"GAIA evaluates AI models on real-world tasks that are conceptually simple for humans but require multi-step reasoning, web browsing, tool use, and multimodal understanding for AI. Tasks span three difficulty levels and test practical assistant capabilities rather than academic knowledge.","category":"agentic","domain":null,"unit":"percent","publisher":"","url":null,"year":"2024","tasks":466,"counted":false,"results":2,"models":2},{"id":"gertLabs","slug":"gert-labs","name":"Gert Labs Composite Game Benchmark","description":"A game-environment benchmark that evaluates AI models in novel games covering strategic planning, resource management, spatial reasoning, cooperation, and theory of mind.","category":"agentic","domain":null,"unit":"percent","publisher":"Gert Labs","url":"https://gertlabs.com/rankings","year":"2026","tasks":"Novel game environments","counted":true,"results":52,"models":52},{"id":"gmmlu","slug":"gmmlu","name":"Global MMLU","description":"MMLU-style knowledge evaluation across 42 high- and low-resource languages.","category":"multilingual","domain":null,"unit":"percent","publisher":"Singh et al.","url":"https://arxiv.org/abs/2412.03304","year":"2024","tasks":"Knowledge questions across 42 languages","counted":false,"results":5,"models":5},{"id":"gpqaDiamond","slug":"gpqa-diamond","name":"GPQA Diamond · BenchLM","description":"The hardest subset of GPQA featuring the most challenging graduate-level science questions. Sometimes reported separately from the standard GPQA benchmark.","category":"knowledge","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"Expert-level science questions","counted":true,"results":84,"models":84},{"id":"epoch-ai:epoch-gpqa-diamond:overall","slug":"epoch-ai-epoch-gpqa-diamond","name":"GPQA diamond · Epoch AI","description":"Evaluated by Epoch AI on their own harness.","category":"knowledge","domain":null,"unit":"percent","publisher":"Epoch AI","url":"https://epoch.ai/benchmarks","year":null,"tasks":null,"counted":true,"results":217,"models":217},{"id":"vals-ai:gpqa:overall","slug":"vals-ai-gpqa","name":"GPQA Diamond · Vals AI","description":"Graduate-level Google-Proof Q&A benchmark evaluating models on questions that require deep reasoning.","category":"knowledge","domain":null,"unit":"percent","publisher":"Vals AI","url":"https://www.vals.ai/benchmarks/gpqa","year":null,"tasks":null,"counted":true,"results":135,"models":134},{"id":"gsm8k","slug":"gsm8k","name":"Grade School Math 8K","description":"GSM8K tests multi-step arithmetic in grade-school word problems. OpenAI introduced the dataset in 2021; a high score on this fixed test is one piece of evidence about mathematical reasoning.","category":"math","domain":null,"unit":"percent","publisher":"Karl Cobbe and colleagues (OpenAI)","url":"https://arxiv.org/abs/2110.14168","year":"2021","tasks":"Grade-school math word problems","counted":false,"results":7,"models":7},{"id":"gpqa","slug":"gpqa","name":"Graduate-Level Google-Proof Q&A","description":"A challenging dataset of 448 multiple-choice questions written by domain experts in biology, physics, and chemistry. Designed to be difficult even for skilled non-experts with access to Google.","category":"knowledge","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"448 questions","counted":true,"results":141,"models":141},{"id":"graphwalksBfs128k","slug":"graphwalks-bfs128k","name":"Graphwalks BFS 0K-128K","description":"Long-context graph traversal benchmark using breadth-first search tasks.","category":"reasoning","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"Graph traversal tasks","counted":false,"results":2,"models":2},{"id":"graphwalksBfs256k1m","slug":"graphwalks-bfs256k1m","name":"GraphWalks BFS 256K–1M","description":"Breadth-first-search graph traversal on 200 problems with context lengths from 256K to 1M tokens.","category":"reasoning","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"Breadth-first-search graph traversal on 200 problems with context lengths from 256K to 1M tokens.","counted":false,"results":5,"models":5},{"id":"handbookMd","slug":"handbook-md","name":"HANDBOOK.md","description":"An agent benchmark run through its official OpenHands-based harness.","category":"agentic","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"Agent tasks (v1 release)","counted":false,"results":1,"models":1},{"id":"hmmtFeb2025","slug":"hmmt-feb2025","name":"Harvard-MIT Mathematics Tournament February 2025","description":null,"category":"math","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"Competition math problems","counted":true,"results":15,"models":15},{"id":"hmmtFeb2026","slug":"hmmt-feb2026","name":"Harvard-MIT Mathematics Tournament February 2026","description":"A February 2026 HMMT slice used in newer frontier-model math comparisons.","category":"math","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"Competition math problems","counted":true,"results":27,"models":27},{"id":"hmmtNov2025","slug":"hmmt-nov2025","name":"Harvard-MIT Mathematics Tournament November 2025","description":"A November 2025 HMMT slice for high-end mathematical reasoning comparisons.","category":"math","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"Competition math problems","counted":true,"results":12,"models":12},{"id":"aaHarveyLabV11","slug":"aa-harvey-lab-v11","name":"Harvey LAB-AA v1.1","description":"Legal-agent task success requiring every rubric criterion to pass and no material hallucination, averaged across three judges.","category":"agentic","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"120 private legal tasks across 24 practice areas","counted":false,"results":8,"models":8},{"id":"harveyLegalAgentPublic","slug":"harvey-legal-agent-public","name":"Harvey's Legal Agent Benchmark, public 1,251-task set, all-pass","description":"Legal-agent tasks from the public 1,251-task Harvey dataset, scored by the share of tasks that pass every rubric criterion.","category":"agentic","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"1,251 public legal tasks","counted":false,"results":1,"models":1},{"id":"healthBenchHard","slug":"health-bench-hard","name":"HealthBench Hard","description":"A harder subset of OpenAI's HealthBench for evaluating open-ended medical and health reasoning with rubric-based grading.","category":"knowledge","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"1,000 health prompts","counted":true,"results":13,"models":13},{"id":"healthBenchLengthAdjusted","slug":"health-bench-length-adjusted","name":"HealthBench length-adjusted score","description":"HealthBench score after applying a verbosity penalty to model responses.","category":"knowledge","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"5,000 multi-turn patient conversations","counted":false,"results":8,"models":8},{"id":"healthBenchProfessional","slug":"health-bench-professional","name":"HealthBench Professional","description":"An open benchmark for clinician-facing model responses across care consult, writing and documentation, and medical research tasks.","category":"knowledge","domain":null,"unit":"percent","publisher":"Rebecca Soskin Hicks et al.","url":"https://arxiv.org/abs/2604.27470","year":"2026","tasks":"Clinician chat tasks","counted":true,"results":18,"models":18},{"id":"healthBenchProfessionalRaw","slug":"health-bench-professional-raw","name":"HealthBench Professional raw score","description":"Raw score on physician-authored clinical consult, documentation, and research conversations.","category":"knowledge","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"525 physician-authored conversations","counted":false,"results":8,"models":8},{"id":"healthBench","slug":"health-bench","name":"HealthBench raw score","description":"Raw score on realistic multi-turn healthcare conversations graded against expert-written rubrics.","category":"knowledge","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"5,000 multi-turn patient conversations","counted":false,"results":8,"models":8},{"id":"hermesBench","slug":"hermes-bench","name":"Hermes Bench: 150 agent tasks graded on the files and state the agent leaves behind","description":"Tests agents on 150 tasks across 25 categories, including Hermes skills, research, diagrams and art, memory, tool use and safety. Each agent starts in a workspace with real files, some tasks add follow-up turns, and the grader checks the files and state left behind.","category":"agentic","domain":null,"unit":"percent","publisher":"Nous Research","url":"https://portal.nousresearch.com/bench","year":"2026","tasks":"150 tasks across 25 categories","counted":false,"results":16,"models":16},{"id":"hermesIndex","slug":"hermes-index","name":"Hermes Index: agentic capability and cost per task inside Hermes Agent","description":"An index of agentic capability from Nous Research. It averages pass@1 scores on four suites, Hermes Bench, TerminalBench 4, TerminalBench Science and SkillsBench, with every run inside the Hermes Agent harness, and reports the mean cost per task beside the score.","category":"agentic","domain":null,"unit":"percent","publisher":"Nous Research","url":"https://portal.nousresearch.com/bench","year":"2026","tasks":"Four suites: Hermes Bench (150 tasks), TerminalBench 4 without its 4 GPU tasks, TerminalBench Science and SkillsBench","counted":true,"results":16,"models":16},{"id":"vals-ai:hlab:overall","slug":"vals-ai-hlab","name":"HLAB-Vals-1.1","description":"Tests an agent's ability to complete legal work using documents, spreadsheets, presentations, and file-system tools","category":null,"domain":"Legal","unit":"percent","publisher":"Vals AI","url":"https://www.vals.ai/benchmarks/hlab","year":null,"tasks":null,"counted":false,"results":35,"models":35},{"id":"hleVerified","slug":"hle-verified","name":"HLE-Verified","description":"A verified and revised Humanity's Last Exam set that removes uncertain items and repairs fixable questions before evaluating frontier expert reasoning.","category":"knowledge","domain":null,"unit":"percent","publisher":"Weiqi Zhai et al.","url":"https://arxiv.org/abs/2602.13964","year":"2026","tasks":"1,811 verified or revised expert questions","counted":false,"results":6,"models":6},{"id":"hle","slug":"hle","name":"Humanity's Last Exam","description":null,"category":"knowledge","domain":null,"unit":"percent","publisher":"Center for AI Safety et al.","url":"https://lastexam.ai/","year":"2025","tasks":"Expert-level questions","counted":true,"results":77,"models":77},{"id":"hleWithTools","slug":"hle-with-tools","name":"Humanity's Last Exam with tools","description":"Tool-augmented Humanity's Last Exam scores reported in DeepSeek-V4 thinking-mode evaluations.","category":"agentic","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"Expert questions with tool use","counted":true,"results":36,"models":36},{"id":"hleNoTools","slug":"hle-no-tools","name":"Humanity's Last Exam without tools","description":"Tool-free variant of Humanity's Last Exam that isolates a model's raw frontier reasoning.","category":"knowledge","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"Expert-level questions","counted":true,"results":53,"models":53},{"id":"ifStruct","slug":"if-struct","name":"IFStruct","description":"Liquid AI benchmark of following structured-output instructions, such as producing output that matches a requested schema or format.","category":"instruction","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"Structured-output instructions","counted":false,"results":1,"models":1},{"id":"ihEval","slug":"ih-eval","name":"IHEval: instruction hierarchy evaluation","description":"Tests whether a model follows the priority order of system messages, user messages, conversation history and tool outputs, including when their instructions conflict.","category":"instruction","domain":null,"unit":"percent","publisher":"IHEval authors","url":"https://github.com/ytyz1307zzh/IHEval","year":"2025","tasks":"Aligned and conflicting instruction sets","counted":false,"results":2,"models":2},{"id":"imoAnswerBench","slug":"imo-answer-bench","name":"IMOAnswerBench","description":"A challenging mathematical reasoning benchmark reported in DeepSeek-V4 model evaluations.","category":"math","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"Advanced mathematical answer generation","counted":true,"results":10,"models":10},{"id":"include","slug":"include","name":"INCLUDE","description":null,"category":"multilingual","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"Cross-lingual understanding","counted":true,"results":12,"models":12},{"id":"ifBench","slug":"if-bench","name":"Instruction Following Benchmark","description":"IFBench evaluates precise instruction-following generalization on 58 challenging, verifiable out-of-domain constraints. Unlike IFEval which tests familiar constraint types, IFBench specifically measures how well models follow novel instructions they haven't been optimized for, exposing overfitting to common instruction patterns.","category":"instruction","domain":null,"unit":"percent","publisher":"","url":null,"year":"2025","tasks":58,"counted":true,"results":57,"models":57},{"id":"ifeval","slug":"ifeval","name":"Instruction-Following Eval","description":"A benchmark of 541 prompts built from 25 verifiable instruction types. It tests whether a model follows checkable constraints such as keyword, length, casing, and response-format requirements.","category":"instruction","domain":null,"unit":"percent","publisher":"Jeffrey Zhou et al.","url":"https://arxiv.org/abs/2311.07911","year":"2023","tasks":"541 prompts across 25 instruction types","counted":true,"results":70,"models":70},{"id":"imo2026","slug":"imo2026","name":"International Mathematical Olympiad 2026","description":"Proof-based olympiad performance on all six IMO 2026 problems.","category":"math","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"6 proof-based problems","counted":false,"results":2,"models":2},{"id":"ipho2025Theory","slug":"ipho2025-theory","name":"International Physics Olympiad 2025 (Theory)","description":"The three official theory problems from the 2025 International Physics Olympiad, scored with blinded human evaluation.","category":"math","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"3 olympiad theory problems","counted":false,"results":1,"models":1},{"id":"inverseIfEval","slug":"inverse-if-eval","name":"InverseIFEval","description":"Tests whether a model follows counterintuitive instructions that run against conventions it learned in training.","category":"instruction","domain":null,"unit":"percent","publisher":"InverseIFEval authors","url":"https://arxiv.org/abs/2509.04292","year":"2025","tasks":"Counterintuitive instructions","counted":false,"results":2,"models":2},{"id":"vals-ai:ioi:overall","slug":"vals-ai-ioi","name":"IOI","description":"Based on the International Olympiad in Informatics","category":"coding","domain":null,"unit":"percent","publisher":"Vals AI","url":"https://www.vals.ai/benchmarks/ioi","year":null,"tasks":null,"counted":true,"results":45,"models":45},{"id":"vals-ai:ioi-v1:overall","slug":"vals-ai-ioi-v1","name":"IOI v1","description":"Based on the International Olympiad in Informatics","category":"coding","domain":null,"unit":"percent","publisher":"Vals AI","url":"https://www.vals.ai/benchmarks/ioi-v1","year":null,"tasks":null,"counted":true,"results":61,"models":60},{"id":"jevBench14","slug":"jev-bench14","name":"jevBench14","description":null,"category":"reasoning","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":null,"counted":true,"results":89,"models":89},{"id":"jevBench15","slug":"jev-bench15","name":"jevBench15","description":null,"category":"reasoning","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":null,"counted":true,"results":104,"models":104},{"id":"jobBench","slug":"job-bench","name":"JobBench","description":"An occupational agent benchmark for professional workflows that workers say they most want delegated to AI.","category":"agentic","domain":null,"unit":"percent","publisher":"Yuetai Li et al.","url":"https://arxiv.org/abs/2605.26329","year":"2026","tasks":"130 tasks across 35 occupations","counted":true,"results":36,"models":36},{"id":"kernelBenchHard","slug":"kernel-bench-hard","name":"KernelBench Hard","description":null,"category":"coding","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"Hard GPU kernel coding tasks","counted":false,"results":1,"models":1},{"id":"kimiClaw247","slug":"kimi-claw247","name":"Kimi Claw 24/7 Bench","description":"A Moonshot AI internal long-horizon agent benchmark for persistent professional coworking tasks.","category":"agentic","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"17 professional scenarios, 610 evaluation points","counted":false,"results":1,"models":1},{"id":"kimiCodeBench3LongHorizon","slug":"kimi-code-bench3-long-horizon","name":"Kimi Code Bench 3.0 Long-horizon (Internal)","description":"Moonshot AI's internal long-horizon coding-agent benchmark, the successor to Kimi Code Bench 2.0.","category":"coding","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"Internal long-horizon coding-agent tasks","counted":false,"results":1,"models":1},{"id":"kimiCodeBenchV2","slug":"kimi-code-bench-v2","name":"Kimi Code Bench v2","description":"A Moonshot AI internal coding-agent benchmark for realistic software-engineering tasks across mainstream programming languages and production technology stacks.","category":"coding","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"Realistic coding-agent tasks","counted":false,"results":3,"models":3},{"id":"kimiHalluBenchHardTools","slug":"kimi-hallu-bench-hard-tools","name":"Kimi Hallucination Benchmark Hard with tools (Internal)","description":"The hard split of Moonshot AI's internal factuality benchmark, run with tools and scored as the claim-level hallucination rate.","category":"agentic","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"Hard internal work-task prompts with tools","counted":false,"results":1,"models":1},{"id":"kimiHalluBench","slug":"kimi-hallu-bench","name":"Kimi Hallucination Benchmark without tools (Internal)","description":"Moonshot AI's internal factuality benchmark built from challenging, de-identified prompts drawn from practical work tasks, scored as the claim-level hallucination rate.","category":"knowledge","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"Internal work-task prompts without tools","counted":false,"results":1,"models":1},{"id":"labBench2","slug":"lab-bench2","name":"LABBench2: An Improved Benchmark for AI Systems Performing Biology Research","description":"A benchmark of realistic biology-research tasks involving literature, figures, tables, databases, and bioinformatics files.","category":"knowledge","domain":null,"unit":"percent","publisher":"Jon M. Laurent et al.","url":"https://arxiv.org/abs/2604.09554","year":"2026","tasks":"Nearly 1,900 biology-research tasks","counted":false,"results":7,"models":7},{"id":"singleCellBench","slug":"single-cell-bench","name":"LatchBio SingleCellBench","description":"Single-cell RNA sequencing analysis tasks spanning common bioinformatics workflows.","category":"knowledge","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"195 single-cell RNA sequencing problems","counted":false,"results":5,"models":5},{"id":"spatialBenchVerified","slug":"spatial-bench-verified","name":"LatchBio SpatialBench Verified","description":"Analysis of spatial transcriptomics data across externally validated biological problems.","category":"knowledge","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"115 externally validated spatial transcriptomics problems","counted":false,"results":3,"models":3},{"id":"legalAgentBenchAllPass","slug":"legal-agent-bench-all-pass","name":"Legal Agent Benchmark all-pass rate — Anthropic harness","description":"Strict task success requiring every expert-written legal-work rubric criterion to pass.","category":"agentic","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"1,235 legal-agent tasks","counted":false,"results":1,"models":1},{"id":"legalAgentBenchHeldoutAllPass","slug":"legal-agent-bench-heldout-all-pass","name":"Legal Agent Benchmark all-pass rate — Harvey held-out set","description":"Harvey AI's strict held-out task success rate requiring every rubric criterion to pass.","category":"agentic","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"Harvey-held-out legal-agent tasks","counted":false,"results":3,"models":3},{"id":"legalAgentBenchCriterionPass","slug":"legal-agent-bench-criterion-pass","name":"Legal Agent Benchmark mean criterion-pass rate — Anthropic harness","description":"Mean fraction of expert-written rubric criteria passed across legal-agent tasks.","category":"agentic","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"1,235 legal-agent tasks","counted":false,"results":1,"models":1},{"id":"legalAgentBenchHeldoutCriterionPass","slug":"legal-agent-bench-heldout-criterion-pass","name":"Legal Agent Benchmark mean criterion-pass rate — Harvey held-out set","description":"Harvey AI's mean criterion-level score on its held-out legal-agent evaluation.","category":"agentic","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"Harvey-held-out legal-agent tasks","counted":false,"results":3,"models":3},{"id":"vals-ai:legal_research:overall","slug":"vals-ai-legal-research","name":"Legal Research Bench","description":"Evaluating agents on legal research tasks across diverse areas of US law","category":null,"domain":"Legal","unit":"percent","publisher":"Vals AI","url":"https://www.vals.ai/benchmarks/legal_research","year":null,"tasks":null,"counted":false,"results":90,"models":90},{"id":"vals-ai:legal_bench:overall","slug":"vals-ai-legal-bench","name":"LegalBench","description":"Evaluating language models on a wide range of open source legal reasoning tasks.","category":null,"domain":"Legal","unit":"percent","publisher":"Vals AI","url":"https://www.vals.ai/benchmarks/legal_bench","year":null,"tasks":null,"counted":false,"results":149,"models":148},{"id":"lifeVal","slug":"life-val","name":"LifeVal: everyday life tasks","description":"OpenAI's benchmark of practical everyday tasks people want a personal assistant to handle, such as finding childcare, accessing government services, budgeting, shopping and caregiving.","category":"agentic","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"169 everyday life tasks (caregiving, health, education, household administration, personal finance, shopping, government services)","counted":false,"results":5,"models":5},{"id":"liquidExtractJsonValidity","slug":"liquid-extract-json-validity","name":"Liquid image-to-JSON extraction JSON validity","description":null,"category":"multimodal","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"Image-to-JSON extraction","counted":false,"results":2,"models":2},{"id":"liquidExtractSchemaF1","slug":"liquid-extract-schema-f1","name":"Liquid image-to-JSON extraction schema consistency F1","description":null,"category":"multimodal","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"Image-to-JSON extraction","counted":false,"results":2,"models":2},{"id":"liquidExtractVlmJudge","slug":"liquid-extract-vlm-judge","name":"Liquid image-to-JSON extraction VLM judge score","description":null,"category":"multimodal","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"Image-to-JSON extraction","counted":false,"results":2,"models":2},{"id":"livebench:livebench-agentic-coding:overall","slug":"livebench-livebench-agentic-coding","name":"LiveBench Agentic Coding","description":"Mean score over agentic tasks in live JavaScript, TypeScript and Python repositories, where the model changes a codebase rather than writing one function. Not in LiveBench 2.0.","category":"agentic","domain":null,"unit":"percent","publisher":"LiveBench","url":"https://livebench.ai/#/release/2026-06-25?cats=Agentic%20Coding","year":null,"tasks":null,"counted":true,"results":66,"models":66},{"id":"livebench:livebench-browser:overall","slug":"livebench-livebench-browser","name":"LiveBench Browser Use","description":"Multi-step tasks in working web apps, done in a real browser by screenshot. Score is the share of tasks passed.","category":"agentic","domain":null,"unit":"percent","publisher":"LiveBench","url":"https://livebench.ai/#/?cats=Browser","year":null,"tasks":null,"counted":true,"results":49,"models":49},{"id":"livebench:livebench-coding:overall","slug":"livebench-livebench-coding","name":"LiveBench Coding","description":"Mean score over writing a solution to a fresh competition problem and completing one whose body was removed. Not in LiveBench 2.0.","category":"coding","domain":null,"unit":"percent","publisher":"LiveBench","url":"https://livebench.ai/#/release/2026-06-25?cats=Coding","year":null,"tasks":null,"counted":true,"results":66,"models":66},{"id":"livebench:livebench-coding-2:overall","slug":"livebench-livebench-coding-2","name":"LiveBench Coding 2.0","description":"Fix real bugs from merged pull requests in a large frozen codebase. Hidden tests grade the fix; a regression scores 0.","category":"coding","domain":null,"unit":"percent","publisher":"LiveBench","url":"https://livebench.ai/#/?cats=Coding","year":null,"tasks":null,"counted":true,"results":52,"models":52},{"id":"livebench:livebench-data-analysis:overall","slug":"livebench-livebench-data-analysis","name":"LiveBench Data Analysis","description":"Short questions on small tables: runs of growth, column matching, format changes. Questions from the 2026-06-25 release; LiveBench 2.0 keeps them.","category":"reasoning","domain":null,"unit":"percent","publisher":"LiveBench","url":"https://livebench.ai/#/?cats=Data%20Analysis","year":null,"tasks":null,"counted":true,"results":80,"models":80},{"id":"livebench:livebench-design:overall","slug":"livebench-livebench-design","name":"LiveBench Design","description":"Make or restyle Word documents, PDFs and slide decks to a brief. Visual and structural checks grade the file.","category":"agentic","domain":null,"unit":"percent","publisher":"LiveBench","url":"https://livebench.ai/#/?cats=Design","year":null,"tasks":null,"counted":true,"results":52,"models":52},{"id":"livebench:livebench-instruction-following:overall","slug":"livebench-livebench-instruction-following","name":"LiveBench Instruction Following","description":"Mean score over paraphrasing, simplifying, summarising and story generation, each marked on whether the stated constraints were actually met. Not in LiveBench 2.0.","category":"instruction","domain":null,"unit":"percent","publisher":"LiveBench","url":"https://livebench.ai/#/release/2026-06-25?cats=IF","year":null,"tasks":null,"counted":true,"results":66,"models":66},{"id":"livebench:livebench-language:overall","slug":"livebench-livebench-language","name":"LiveBench Language","description":"Mean score over Connections word puzzles, unscrambling film plot summaries and repairing typos. English only. Not in LiveBench 2.0.","category":"knowledge","domain":null,"unit":"percent","publisher":"LiveBench","url":"https://livebench.ai/#/release/2026-06-25?cats=Language","year":null,"tasks":null,"counted":true,"results":66,"models":66},{"id":"livebench:livebench-long-horizon:overall","slug":"livebench-livebench-long-horizon","name":"LiveBench Long Agentic","description":"Long agentic work: extend business apps over many turns, run a codebase through simulated months, build and tune systems. Graded by scripts.","category":"agentic","domain":null,"unit":"percent","publisher":"LiveBench","url":"https://livebench.ai/#/?cats=Long%20Agentic","year":null,"tasks":null,"counted":true,"results":51,"models":51},{"id":"livebench:livebench-math:overall","slug":"livebench-livebench-math","name":"LiveBench Mathematics","description":"Mean score over synthetic AMPS problems, integrals, recent competition questions and olympiad problem rearrangement. Not in LiveBench 2.0.","category":"math","domain":null,"unit":"percent","publisher":"LiveBench","url":"https://livebench.ai/#/release/2026-06-25?cats=Mathematics","year":null,"tasks":null,"counted":true,"results":66,"models":66},{"id":"livebench:livebench-reasoning:overall","slug":"livebench-livebench-reasoning","name":"LiveBench Reasoning","description":"Short logic, spatial and theory-of-mind questions with one correct answer. Questions from the 2026-06-25 release; LiveBench 2.0 keeps them.","category":"reasoning","domain":null,"unit":"percent","publisher":"LiveBench","url":"https://livebench.ai/#/?cats=Reasoning","year":null,"tasks":null,"counted":true,"results":80,"models":80},{"id":"livebench:livebench-research:overall","slug":"livebench-livebench-research","name":"LiveBench Research","description":"Research a brief and write a report with sources. A rubric for each task grades the report.","category":"agentic","domain":null,"unit":"percent","publisher":"LiveBench","url":"https://livebench.ai/#/?cats=Research","year":null,"tasks":null,"counted":true,"results":52,"models":52},{"id":"liveClaw","slug":"live-claw","name":"LiveClaw","description":"Personal-assistant agent tasks run end to end inside the OpenClaw agent harness.","category":"agentic","domain":null,"unit":"percent","publisher":"Liquid AI (reporting)","url":"https://www.liquid.ai/blog/lfm3","year":"2026","tasks":"Assistant agent tasks","counted":false,"results":1,"models":1},{"id":"vals-ai:lcb:overall","slug":"vals-ai-lcb","name":"LiveCodeBench","description":"Our Implementation of the LiveCodeBench benchmark","category":"coding","domain":null,"unit":"percent","publisher":"Vals AI","url":"https://www.vals.ai/benchmarks/lcb","year":null,"tasks":null,"counted":true,"results":142,"models":141},{"id":"liveCodeBenchPass1Cot","slug":"live-code-bench-pass1-cot","name":"LiveCodeBench Pass@1 with Chain-of-Thought","description":null,"category":"coding","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"DeepSeek-V4 report evaluation window","counted":false,"results":2,"models":2},{"id":"liveCodeBenchPro","slug":"live-code-bench-pro","name":"LiveCodeBench Pro","description":"A harder competitive-programming benchmark family built from Codeforces, ICPC, and IOI problems, with quarter-specific public leaderboards and difficulty-aware reporting.","category":"coding","domain":null,"unit":"percent","publisher":"LiveCodeBench Pro authors","url":"https://arxiv.org/abs/2506.11928","year":"2025","tasks":"Quarter-specific contest programming sets","counted":false,"results":9,"models":9},{"id":"liveCodeBenchV5","slug":"live-code-bench-v5","name":"LiveCodeBench v5","description":null,"category":"coding","domain":null,"unit":"percent","publisher":"LiveCodeBench maintainers","url":"https://github.com/LiveCodeBench/LiveCodeBench","year":"2025","tasks":"July 2024 to May 2025 release window","counted":false,"results":9,"models":9},{"id":"liveCodeBenchV6","slug":"live-code-bench-v6","name":"LiveCodeBench v6","description":null,"category":"coding","domain":null,"unit":"percent","publisher":"LiveCodeBench maintainers","url":"https://github.com/LiveCodeBench/LiveCodeBench","year":"2026","tasks":"Fresh programming problems","counted":true,"results":59,"models":59},{"id":"liveCodeBench","slug":"live-code-bench","name":"LiveCodeBench: Holistic and Contamination Free Evaluation of Large Language Models for Code","description":"A continuously updated coding benchmark built from newly collected LeetCode, AtCoder, and Codeforces problems. Fresh problem windows reduce one contamination path, but results still need a release and setup check.","category":"coding","domain":null,"unit":"percent","publisher":"Naman Jain et al.","url":"https://arxiv.org/abs/2403.07974","year":"2024","tasks":"Continuously updated contest problems","counted":true,"results":12,"models":12},{"id":"locaBench256k","slug":"loca-bench256k","name":"LOCA-Bench 256K","description":"Tests long-context agents as tool environments grow to a 256K-token environment description.","category":"agentic","domain":null,"unit":"percent","publisher":"Weihao Zeng et al.","url":"https://arxiv.org/abs/2602.07962","year":"2026","tasks":null,"counted":false,"results":1,"models":1},{"id":"longContextTranslation","slug":"long-context-translation","name":"Long-context translation evaluation (Cohere)","description":"Cohere's long-document translation evaluation: two book chapters translated in a single call, scored per paragraph with xCOMET-XL.","category":"multilingual","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"Two book chapters per call","counted":false,"results":1,"models":1},{"id":"longBenchV2","slug":"long-bench-v2","name":"LongBench v2","description":"A long-context benchmark that measures whether models can actually use extended context windows for reasoning and retrieval.","category":"reasoning","domain":null,"unit":"percent","publisher":"LongBench v2 authors","url":"https://arxiv.org/abs/2412.15204","year":"2025","tasks":"Long-context tasks","counted":true,"results":25,"models":25},{"id":"longMemEval","slug":"long-mem-eval","name":"LongMemEval","description":"Tests whether a chat assistant recalls and reasons over information spread across long, multi-session conversation histories.","category":"reasoning","domain":null,"unit":"percent","publisher":"LongMemEval authors","url":"https://github.com/xiaowu0162/LongMemEval","year":"2024","tasks":"Questions over multi-session chat histories","counted":false,"results":2,"models":2},{"id":"lvBench","slug":"lv-bench","name":"LVBench","description":"A long-video understanding benchmark for retrieving and reasoning over information distributed across extended video inputs.","category":"multimodal","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"Long-form video question answering","counted":false,"results":7,"models":7},{"id":"mmmu","slug":"mmmu","name":"Massive Multi-discipline Multimodal Understanding","description":"A broad multimodal reasoning benchmark spanning charts, diagrams, tables, and academic visual question answering.","category":"multimodal","domain":null,"unit":"percent","publisher":"MMMU authors","url":"https://arxiv.org/abs/2401.05508","year":"2024","tasks":"Multimodal academic reasoning","counted":true,"results":28,"models":28},{"id":"mmmuPro","slug":"mmmu-pro","name":"Massive Multi-discipline Multimodal Understanding Pro","description":"A harder multimodal benchmark for frontier models that combines text with images, diagrams, charts, and academic visual reasoning tasks.","category":"multimodal","domain":null,"unit":"percent","publisher":"MMMU-Pro authors","url":"https://arxiv.org/abs/2409.02813","year":"2024","tasks":"Multimodal academic reasoning","counted":true,"results":53,"models":53},{"id":"mmlu","slug":"mmlu","name":"Massive Multitask Language Understanding","description":"A comprehensive multiple-choice question answering test covering 57 tasks including elementary mathematics, US history, computer science, law, and more. Tests knowledge across diverse academic subjects from high school to professional level.","category":"knowledge","domain":null,"unit":"percent","publisher":"Dan Hendrycks et al.","url":"https://arxiv.org/abs/2009.03300","year":"2020","tasks":"57 subjects","counted":true,"results":34,"models":34},{"id":"mmluPro","slug":"mmlu-pro","name":"Massive Multitask Language Understanding Professional","description":"An enhanced version of MMLU with 10 answer choices instead of 4, featuring more reasoning-focused questions that better differentiate frontier models.","category":"knowledge","domain":null,"unit":"percent","publisher":"Yubo Wang et al.","url":"https://arxiv.org/abs/2406.01574","year":"2024","tasks":"Multiple subjects","counted":true,"results":75,"models":75},{"id":"vals-ai:math500:overall","slug":"vals-ai-math500","name":"MATH 500","description":"Academic math benchmark on probability, algebra, and trigonometry","category":"math","domain":null,"unit":"percent","publisher":"Vals AI","url":"https://www.vals.ai/benchmarks/math500","year":null,"tasks":null,"counted":true,"results":59,"models":59},{"id":"epoch-ai:epoch-math-level-5:overall","slug":"epoch-ai-epoch-math-level-5","name":"MATH level 5","description":"Evaluated by Epoch AI on their own harness.","category":"math","domain":null,"unit":"percent","publisher":"Epoch AI","url":"https://epoch.ai/benchmarks","year":null,"tasks":null,"counted":true,"results":88,"models":88},{"id":"math500","slug":"math500","name":"MATH-500 Problem Set","description":"A curated subset of 500 problems from the MATH dataset, covering algebra, counting and probability, geometry, intermediate algebra, number theory, prealgebra, and precalculus.","category":"math","domain":null,"unit":"percent","publisher":"Dan Hendrycks et al.","url":"https://arxiv.org/abs/2103.03874","year":"2021","tasks":"500 problems","counted":true,"results":27,"models":27},{"id":"mathVision","slug":"math-vision","name":"MathVision","description":"A visual mathematics benchmark that tests whether a model can solve math problems grounded in diagrams, equations, figures, and other visual inputs.","category":"multimodal","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"Visually grounded math problems","counted":true,"results":24,"models":24},{"id":"mathVisionPython","slug":"math-vision-python","name":"MathVision with Python","description":"A tool-augmented MathVision variant that permits Python during visual mathematics reasoning.","category":"multimodal","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"Visual mathematics problems with Python","counted":false,"results":6,"models":6},{"id":"maxife","slug":"maxife","name":"MAXIFE","description":"A multilingual instruction-following and understanding benchmark row published in Qwen's launch comparisons.","category":"multilingual","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"Multilingual instruction following","counted":false,"results":5,"models":5},{"id":"mazeBench","slug":"maze-bench","name":"MazeBench: visual spatial reasoning in a 3D open world","description":"Tests whether a coding agent can explore a 256-room 3D puzzle world and collect its 100 hidden gems, using box-pushing puzzles, camera rotation, and long-horizon planning.","category":"agentic","domain":null,"unit":"percent","publisher":"Jonathan Pappas and David Pappas","url":"https://mazebench.com/blog?post=maze-bench-results","year":"2026","tasks":"100 hidden gems across 256 rooms in one open world","counted":true,"results":48,"models":48},{"id":"mcpAtlas","slug":"mcp-atlas","name":"MCP Atlas","description":"A benchmark for tool-calling over Model Context Protocol integrations and external tools.","category":"agentic","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"Tool-integrated agent tasks","counted":true,"results":52,"models":52},{"id":"mcpAtlasClaimCoverage","slug":"mcp-atlas-claim-coverage","name":"MCP-Atlas mean claim coverage","description":"Average coverage of required claims in answers produced during real-world MCP tool-use workflows.","category":"agentic","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"Production-like multi-server MCP workflows","counted":false,"results":2,"models":2},{"id":"mcpTasks","slug":"mcp-tasks","name":"MCP-Tasks","description":"A Model Context Protocol task benchmark used in Qwen's launch tables to measure practical execution over MCP-style tools and integrations.","category":"agentic","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"MCP-integrated tool tasks","counted":false,"results":5,"models":5},{"id":"mcpMarkVerified","slug":"mcp-mark-verified","name":"MCPMark-Verified","description":"A human-verified edition of MCPMark for MCP tool use across Notion, GitHub, Filesystem, Postgres, and Playwright server environments.","category":"agentic","domain":null,"unit":"percent","publisher":"MCPMark","url":"https://mcpmark.ai/","year":"2026","tasks":"MCP tool-use tasks across five server environments","counted":false,"results":1,"models":1},{"id":"simpleQa","slug":"simple-qa","name":"Measuring Short-Form Factuality in Large Language Models","description":"A benchmark that evaluates the ability of language models to answer short, fact-seeking questions accurately. Focuses on factual correctness rather than reasoning complexity.","category":"knowledge","domain":null,"unit":"percent","publisher":"Jason Wei et al.","url":"https://arxiv.org/abs/2411.04368","year":"2024","tasks":"Factual questions","counted":true,"results":14,"models":14},{"id":"vals-ai:medcode:overall","slug":"vals-ai-medcode","name":"MedCode","description":"Can models support the medical billing process?","category":null,"domain":"Healthcare","unit":"percent","publisher":"Vals AI","url":"https://www.vals.ai/benchmarks/medcode","year":null,"tasks":null,"counted":false,"results":106,"models":106},{"id":"aaMlcr","slug":"aa-mlcr","name":"Medical Long Context Reasoning (MLCR-AA)","description":"An open benchmark from Wisedocs, run by Artificial Analysis, measuring how well models reason over long, fragmented medical records with multi-document synthesis.","category":"reasoning","domain":null,"unit":"percent","publisher":"Wisedocs and Artificial Analysis","url":"https://artificialanalysis.ai/evaluations/mlcr-aa","year":"2026","tasks":"Long, fragmented medical-record reasoning","counted":true,"results":22,"models":22},{"id":"vals-ai:medqa:overall","slug":"vals-ai-medqa","name":"MedQA","description":"Evaluating language model bias in medical questions.","category":null,"domain":"Healthcare","unit":"percent","publisher":"Vals AI","url":"https://www.vals.ai/benchmarks/medqa","year":null,"tasks":null,"counted":false,"results":95,"models":94},{"id":"vals-ai:medscribe:overall","slug":"vals-ai-medscribe","name":"MedScribe","description":"Can models support doctors with their administrative work?","category":null,"domain":"Healthcare","unit":"percent","publisher":"Vals AI","url":"https://www.vals.ai/benchmarks/medscribe","year":null,"tasks":null,"counted":false,"results":109,"models":109},{"id":"medXpertQaMm","slug":"med-xpert-qa-mm","name":"MedXpertQA Multimodal","description":"A multimodal medical multiple-choice benchmark covering clinical images such as X-rays, histology, and dermatology.","category":"multimodal","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"2,000 multimodal medical questions","counted":false,"results":8,"models":8},{"id":"medXpertQaText","slug":"med-xpert-qa-text","name":"MedXpertQA Text","description":"A medical multiple-choice benchmark spanning many specialties with 10 answer options per question.","category":"knowledge","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"2,450 medical multiple-choice questions","counted":false,"results":6,"models":6},{"id":"vals-ai:mgsm:overall","slug":"vals-ai-mgsm","name":"MGSM","description":"A multilingual benchmark for mathematical questions.","category":"multilingual","domain":null,"unit":"percent","publisher":"Vals AI","url":"https://www.vals.ai/benchmarks/mgsm","year":null,"tasks":null,"counted":false,"results":74,"models":73},{"id":"epoch-ai:epoch-mirrorcode:overall","slug":"epoch-ai-epoch-mirrorcode","name":"MirrorCode","description":"Evaluated by Epoch AI on their own harness.","category":"coding","domain":null,"unit":"percent","publisher":"Epoch AI","url":"https://epoch.ai/benchmarks","year":null,"tasks":null,"counted":true,"results":39,"models":39},{"id":"mkqa11","slug":"mkqa11","name":"MKQA-11 multilingual retrieval","description":null,"category":"multilingual","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"Cross-lingual open-domain QA retrieval","counted":false,"results":1,"models":1},{"id":"mleBenchLite","slug":"mle-bench-lite","name":"MLE-Bench Lite","description":"A lightweight machine-learning competition benchmark that measures whether models can iteratively train, evaluate, and improve ML systems in low-resource settings.","category":"agentic","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"Low-resource ML competitions","counted":false,"results":4,"models":4},{"id":"mlsBenchLite","slug":"mls-bench-lite","name":"MLS-Bench Lite","description":"A 30-task subset of MLS-Bench that evaluates whether AI systems can invent generalizable and scalable machine-learning methods.","category":"coding","domain":null,"unit":"percent","publisher":"MLS-Bench","url":"https://mls-bench.com/","year":"2026","tasks":"30 machine-learning research tasks","counted":false,"results":4,"models":4},{"id":"mlvuAvg","slug":"mlvu-avg","name":"MLVU mean average","description":"A multi-task video understanding benchmark averaged across MLVU categories.","category":"multimodal","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"General video understanding","counted":false,"results":10,"models":10},{"id":"mmClawBench","slug":"mm-claw-bench","name":"MM-ClawBench","description":"An OpenClaw-derived agent benchmark covering practical work and life tasks such as office document delivery, research, planning, and code maintenance.","category":"agentic","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"OpenClaw-style real-world tasks","counted":false,"results":2,"models":2},{"id":"mmAnswerBench","slug":"mm-answer-bench","name":"MMAnswerBench","description":"A multimodal mathematical reasoning benchmark that tests whether models can answer visually grounded math questions correctly.","category":"math","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"Multimodal math questions","counted":false,"results":10,"models":10},{"id":"mmLongBenchDoc","slug":"mm-long-bench-doc","name":"MMLongBench-Doc","description":"A long-document multimodal benchmark for grounded reasoning over extended document contexts.","category":"multimodal","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"Long document understanding","counted":false,"results":6,"models":6},{"id":"vals-ai:mmlu_pro:overall","slug":"vals-ai-mmlu-pro","name":"MMLU Pro","description":"Academic multiple-choice benchmark covering 14 subjects including STEM, humanities, and social sciences.","category":"knowledge","domain":null,"unit":"percent","publisher":"Vals AI","url":"https://www.vals.ai/benchmarks/mmlu_pro","year":null,"tasks":null,"counted":true,"results":135,"models":134},{"id":"mmluProArcee","slug":"mmlu-pro-arcee","name":"MMLU-Pro first-party comparison snapshot","description":null,"category":"knowledge","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"Professional academic QA","counted":true,"results":6,"models":6},{"id":"mmluProX","slug":"mmlu-pro-x","name":"MMLU-ProX","description":"A multilingual extension of professional-level academic evaluation across many languages.","category":"multilingual","domain":null,"unit":"percent","publisher":"MMLU-ProX authors","url":"https://arxiv.org/abs/2503.10497","year":"2025","tasks":"Multilingual professional QA","counted":true,"results":17,"models":17},{"id":"mmluRedux","slug":"mmlu-redux","name":"MMLU-Redux","description":"A harder refresh of MMLU intended to keep broad knowledge evaluation useful after the original benchmark became too easy for frontier models.","category":"knowledge","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"Broad academic QA","counted":true,"results":19,"models":19},{"id":"mmmlu","slug":"mmmlu","name":"MMMLU","description":null,"category":"knowledge","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"Multilingual academic QA","counted":true,"results":15,"models":15},{"id":"vals-ai:mmmu:overall","slug":"vals-ai-mmmu","name":"MMMU Pro","description":"Multimodal Multi-task Benchmark","category":"multimodal","domain":null,"unit":"percent","publisher":"Vals AI","url":"https://www.vals.ai/benchmarks/mmmu","year":null,"tasks":null,"counted":true,"results":92,"models":92},{"id":"mmmuProPython","slug":"mmmu-pro-python","name":"MMMU-Pro with Python","description":"Tool-augmented MMMU-Pro variant that allows Python assistance during multimodal reasoning.","category":"multimodal","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"Multimodal academic reasoning","counted":false,"results":10,"models":10},{"id":"mmSearchPlus","slug":"mm-search-plus","name":"MMSearch-Plus","description":"A harder MMSearch variant for multimodal retrieval and grounded tool-use workflows.","category":"multimodal","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"Hard multimodal search tasks","counted":false,"results":1,"models":1},{"id":"mobileWorld","slug":"mobile-world","name":"MobileWorld","description":"A mobile-use agent benchmark for completing interactive tasks in smartphone environments.","category":"agentic","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"Interactive mobile-device workflows","counted":false,"results":1,"models":1},{"id":"protocolsTroubleshooting","slug":"protocols-troubleshooting","name":"Molecular Biology Protocols Troubleshooting","description":"Detects and fixes errors in molecular-biology protocols using document, code, and web-search tools.","category":"knowledge","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"Molecular-biology protocol troubleshooting","counted":false,"results":5,"models":5},{"id":"vals-ai:mortgage_tax:overall","slug":"vals-ai-mortgage-tax","name":"MortgageTax","description":"Evaluating reading and understanding tax certificates as images","category":null,"domain":"Finance","unit":"percent","publisher":"Vals AI","url":"https://www.vals.ai/benchmarks/mortgage_tax","year":null,"tasks":null,"counted":false,"results":97,"models":97},{"id":"mrcr1m","slug":"mrcr1m","name":"MRCR 1M","description":"A million-token MRCR long-context retrieval benchmark reported in DeepSeek-V4 model evaluations.","category":"reasoning","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"Million-token retrieval","counted":false,"results":4,"models":4},{"id":"mrcrv2","slug":"mrcrv2","name":"MRCRv2","description":"A long-context benchmark for memory, retrieval, and multi-round coherence over large contexts.","category":"reasoning","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"Long-context retrieval","counted":true,"results":14,"models":14},{"id":"mStar","slug":"m-star","name":"MStar","description":null,"category":"multimodal","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"Real-image visual QA","counted":false,"results":1,"models":1},{"id":"multiAgentBrowseCompPrerelease","slug":"multi-agent-browse-comp-prerelease","name":"Multi-Agent BrowseComp — 10-agent team prerelease configuration","description":"BrowseComp accuracy from ten collaborating Opus 5 agents on a pre-release model and unreleased effort configuration.","category":"agentic","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"BrowseComp web-research tasks","counted":false,"results":1,"models":1},{"id":"multiIf","slug":"multi-if","name":"Multi-IF","description":"Meta benchmark of instruction following over multi-turn conversations in eight languages.","category":"instruction","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"Multi-turn, multilingual instructions","counted":false,"results":1,"models":1},{"id":"multiSweBench","slug":"multi-swe-bench","name":"Multi-SWE Bench","description":"A multi-language software-engineering benchmark that measures repository-level bug fixing and implementation across more than one programming ecosystem.","category":"coding","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"Multi-language repo tasks","counted":false,"results":1,"models":1},{"id":"milu","slug":"milu","name":"Multi-task Indic Language Understanding Benchmark","description":"Culturally grounded knowledge comprehension across ten Indic languages and English.","category":"multilingual","domain":null,"unit":"percent","publisher":"Verma et al.","url":"https://arxiv.org/abs/2411.02538","year":"2024","tasks":"Knowledge tasks across 11 languages","counted":false,"results":5,"models":5},{"id":"multiChallenge","slug":"multi-challenge","name":"MultiChallenge: provider-reported runs","description":"Tests instruction retention, memory of user information, versioned editing, and coherence over multi-turn conversations.","category":"instruction","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":null,"counted":false,"results":3,"models":3},{"id":"mgsm","slug":"mgsm","name":"Multilingual Grade School Math","description":"A multilingual benchmark that translates 250 grade school math problems from GSM8K into 10 typologically diverse languages: Bengali, German, Spanish, French, Japanese, Russian, Swahili, Telugu, Thai, and Chinese.","category":"multilingual","domain":null,"unit":"percent","publisher":"Freda Shi et al.","url":"https://arxiv.org/abs/2210.03057","year":"2022","tasks":"250 problems × 11 languages","counted":true,"results":13,"models":13},{"id":"mka","slug":"mka","name":"Multimodal Knowledge Agent Benchmark","description":"Google DeepMind's benchmark of professional knowledge work where the key information sits in visual files such as architectural drawings, engineering schematics and medical scans. The agent has to deliver a finished report, spreadsheet, slide deck or marked-up drawing. Artificial Analysis runs it on a private 200-task set.","category":"agentic","domain":null,"unit":"percent","publisher":"Google DeepMind, in partnership with Artificial Analysis","url":"https://artificialanalysis.ai/evaluations/mka","year":"2026","tasks":"200 multimodal professional tasks across 10 industry verticals","counted":true,"results":19,"models":19},{"id":"mmvu","slug":"mmvu","name":"Multimodal Multi-disciplinary Video Understanding","description":"A benchmark for evaluating multimodal models on video understanding tasks across multiple disciplines, emphasizing temporal reasoning and comprehension over video content.","category":"multimodal","domain":null,"unit":"percent","publisher":"MMVU benchmark maintainers","url":"https://www.kimi.com/blog/kimi-k2-5.html","year":"2026","tasks":"Video understanding","counted":false,"results":8,"models":8},{"id":"epoch-ai:epoch-mystery-game-puzzles:overall","slug":"epoch-ai-epoch-mystery-game-puzzles","name":"Mystery Game Puzzles","description":"Evaluated by Epoch AI on their own harness.","category":"reasoning","domain":null,"unit":"percent","publisher":"Epoch AI","url":"https://epoch.ai/benchmarks","year":null,"tasks":null,"counted":true,"results":80,"models":80},{"id":"vals-ai:mysterymechanism:overall","slug":"vals-ai-mysterymechanism","name":"MysteryMechanism","description":"Can agents rediscover sealed mathematical mechanisms through bounded experiments?","category":null,"domain":"Science","unit":"percent","publisher":"Vals AI","url":"https://www.vals.ai/benchmarks/mysterymechanism","year":null,"tasks":null,"counted":false,"results":30,"models":30},{"id":"nanoBeirMultilingual","slug":"nano-beir-multilingual","name":"NanoBEIR Multilingual Extended","description":null,"category":"multilingual","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"Multilingual document retrieval","counted":false,"results":1,"models":1},{"id":"nl2Repo","slug":"nl2-repo","name":"NL2Repo","description":"A repository-understanding benchmark that measures whether models can map natural-language requests onto the right code locations and system changes.","category":"coding","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"Natural language to repository tasks","counted":true,"results":36,"models":36},{"id":"nova63","slug":"nova63","name":"NOVA-63","description":"A broad multilingual benchmark row from Qwen's launch comparisons intended to measure cross-lingual capability beyond a single language family.","category":"multilingual","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"Broad multilingual evaluation","counted":false,"results":7,"models":7},{"id":"ocrBenchV2","slug":"ocr-bench-v2","name":"OCRBench V2","description":"A native OCR benchmark for reading text from images across multilingual scripts, low-quality scans, handwriting, structured layouts, charts, and screenshots.","category":"multimodal","domain":null,"unit":"percent","publisher":"OCRBench authors","url":"https://arxiv.org/abs/2501.00321","year":"2025","tasks":"Image OCR tasks","counted":false,"results":6,"models":6},{"id":"odinw13","slug":"odinw13","name":"ODINW13","description":"A visual detection and grounding benchmark slice used to compare zero-shot object understanding across diverse domains.","category":"multimodal","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"Out-of-distribution object understanding","counted":false,"results":4,"models":4},{"id":"epoch-ai:epoch-oeis-open:overall","slug":"epoch-ai-epoch-oeis-open","name":"OEIS Open","description":"Evaluated by Epoch AI on their own harness.","category":"math","domain":null,"unit":"percent","publisher":"Epoch AI","url":"https://epoch.ai/benchmarks","year":null,"tasks":null,"counted":false,"results":3,"models":3},{"id":"epoch-ai:epoch-oeis-open-lite:overall","slug":"epoch-ai-epoch-oeis-open-lite","name":"OEIS Open Lite","description":"Evaluated by Epoch AI on their own harness.","category":"math","domain":null,"unit":"percent","publisher":"Epoch AI","url":"https://epoch.ai/benchmarks","year":null,"tasks":null,"counted":false,"results":5,"models":5},{"id":"officeQa","slug":"office-qa","name":"OfficeQA","description":"Grounded numerical reasoning over a corpus of historical U.S. Treasury Bulletin documents.","category":"multimodal","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"Historical Treasury Bulletin questions","counted":false,"results":5,"models":5},{"id":"officeQaPro","slug":"office-qa-pro","name":"OfficeQA Pro","description":"A benchmark for grounded reasoning over office-style documents, spreadsheets, charts, and business artifacts.","category":"multimodal","domain":null,"unit":"percent","publisher":"OfficeQA Pro authors","url":"https://arxiv.org/abs/2603.08655","year":"2026","tasks":"Document and spreadsheet tasks","counted":true,"results":16,"models":16},{"id":"officeQaProV2","slug":"office-qa-pro-v2","name":"OfficeQA Pro v2","description":"The second version of OfficeQA Pro: grounded reasoning over real financial and office documents through retrieval, cross-document verification, and multi-step numerical reasoning.","category":"multimodal","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"Document and spreadsheet tasks","counted":false,"results":2,"models":2},{"id":"olmOcr","slug":"olm-ocr","name":"olmOCR-Bench","description":"An end-to-end document understanding benchmark over long, layout-rich PDFs with tables, equations, headers, footnotes, and multi-column flows.","category":"multimodal","domain":null,"unit":"percent","publisher":"Allen Institute for AI","url":"https://github.com/allenai/olmocr/tree/main/olmocr/bench","year":"2025","tasks":"Layout-rich PDF understanding","counted":false,"results":1,"models":1},{"id":"omniDocBench","slug":"omni-doc-bench","name":"OmniDocBench","description":"A document-understanding benchmark for parsing and reasoning over complex document layouts.","category":"multimodal","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"Complex document-understanding tasks","counted":false,"results":1,"models":1},{"id":"omniDocBench15","slug":"omni-doc-bench15","name":"OmniDocBench 1.5","description":"A document understanding benchmark used in frontier-model comparison tables to measure extraction and grounded reasoning quality on complex documents.","category":"multimodal","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"Document understanding tasks","counted":false,"results":11,"models":11},{"id":"omniDocBench16","slug":"omni-doc-bench16","name":"OmniDocBench v1.6","description":"A document parsing benchmark whose overall score composites edit distance, CDM and TEDS sub-scores across diverse PDF layouts.","category":"multimodal","domain":null,"unit":"percent","publisher":"Linke Ouyang et al.","url":"https://arxiv.org/abs/2412.07626","year":"2024","tasks":"Document parsing and extraction","counted":false,"results":1,"models":1},{"id":"mrcrv2_128_256","slug":"mrcrv2-128-256","name":"OpenAI MRCR v2 8-needle 128K-256K","description":"MRCR v2 slice focused on very long contexts at 128K-256K lengths.","category":"reasoning","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"8-needle retrieval tasks","counted":false,"results":2,"models":2},{"id":"mrcrv2_256_512","slug":"mrcrv2-256-512","name":"OpenAI MRCR v2 8-needle 256K-512K","description":"MRCR v2 slice focused on retrieval across 256K-512K-token contexts.","category":"reasoning","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"100 eight-needle retrieval examples","counted":false,"results":2,"models":2},{"id":"mrcrv2_512_1m","slug":"mrcrv2-512-1m","name":"OpenAI MRCR v2 8-needle 512K-1M","description":"MRCR v2 slice focused on retrieval across 512K-1M-token contexts.","category":"reasoning","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"100 eight-needle retrieval examples","counted":false,"results":2,"models":2},{"id":"mrcrv2_64_128","slug":"mrcrv2-64-128","name":"OpenAI MRCR v2 8-needle 64K-128K","description":"MRCR v2 slice focused on long-context retrieval at 64K-128K lengths.","category":"reasoning","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"8-needle retrieval tasks","counted":false,"results":2,"models":2},{"id":"openHarmonyBench","slug":"open-harmony-bench","name":"OpenHarmony Bench v1.0","description":"An app-level coding benchmark that asks DevEco Code configurations to implement observable behavior in buildable OpenHarmony ArkTS applications.","category":"coding","domain":null,"unit":"percent","publisher":"OpenHarmony Bench authors","url":"https://arxiv.org/abs/2608.16022","year":"2026","tasks":"153 app-development and bug-fix tasks","counted":true,"results":13,"models":13},{"id":"osWorld","slug":"os-world","name":"OSWorld","description":"A computer-use benchmark for GUI task completion across the broader OSWorld task suite.","category":"agentic","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"Computer-use tasks","counted":false,"results":2,"models":2},{"id":"osWorld2","slug":"os-world2","name":"OSWorld 2.0","description":"OSWorld 2.0 evaluates long-horizon computer-use tasks. Published results use different task releases, metrics, and run settings, so scores appear by comparable protocol and unmatched rows stay unranked.","category":"agentic","domain":null,"unit":"percent","publisher":"Mengqi Yuan et al.","url":"https://arxiv.org/abs/2606.29537","year":"2026","tasks":"108 long-horizon computer-use workflows","counted":true,"results":25,"models":25},{"id":"osWorld21","slug":"os-world21","name":"OSWorld 2.1 (all 108 tasks)","description":"OSWorld 2.1 is the current release of OSWorld 2.0: the same 108 long-horizon computer-use workflows with the maintainers' later fixes to task files, assets, mocked websites, and evaluators. This page stores partial scores on the full task set.","category":"agentic","domain":null,"unit":"percent","publisher":"Mengqi Yuan et al.","url":"https://github.com/xlang-ai/OSWorld-V2/releases","year":"2026","tasks":"108 long-horizon computer-use workflows","counted":true,"results":11,"models":11},{"id":"osWorld21Offline","slug":"os-world21-offline","name":"OSWorld 2.1 offline subset (82 tasks)","description":"The official offline subset of OSWorld 2.1: 82 of the 108 long-horizon computer-use workflows, run with no internet access in the agent's virtual machine.","category":"agentic","domain":null,"unit":"percent","publisher":"Mengqi Yuan et al.","url":"https://github.com/xlang-ai/OSWorld-V2/releases","year":"2026","tasks":"82 offline computer-use workflows","counted":true,"results":10,"models":10},{"id":"osWorldVerified","slug":"os-world-verified","name":"OSWorld-Verified","description":"OSWorld-Verified is the July 2025 repaired release of OSWorld's real-computer evaluation. It measures whether a model-agent system can finish desktop and web tasks from configured starting states, with success checked by execution-based evaluators.","category":"agentic","domain":null,"unit":"percent","publisher":"Tianbao Xie et al.","url":"https://os-world.github.io/","year":"2025","tasks":"369 real-world computer tasks (361 when eight Google Drive tasks are excluded)","counted":true,"results":37,"models":37},{"id":"epoch-ai:epoch-otis-mock-aime-2024-2025:overall","slug":"epoch-ai-epoch-otis-mock-aime-2024-2025","name":"OTIS Mock AIME 2024-2025","description":"Evaluated by Epoch AI on their own harness.","category":"math","domain":null,"unit":"percent","publisher":"Epoch AI","url":"https://epoch.ai/benchmarks","year":null,"tasks":null,"counted":true,"results":200,"models":200},{"id":"paperBench","slug":"paper-bench","name":"PaperBench","description":"A research-reproduction benchmark that asks agents to recreate the contributions of AI papers from the paper alone.","category":"coding","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"AI research-paper reproduction","counted":false,"results":2,"models":2},{"id":"parseBenchLlamaIndex","slug":"parse-bench-llama-index","name":"ParseBench: document parsing for AI agents (LlamaIndex leaderboard)","description":"Scores document parsers and vision-language models on 2,078 human-verified enterprise pages across tables, charts, content faithfulness, semantic formatting and visual grounding, with the listed cost per page.","category":"multimodal","domain":null,"unit":"percent","publisher":"Boyang Zhang et al.","url":"https://arxiv.org/abs/2604.08538","year":"2026","tasks":"2,078 human-verified enterprise pages from insurance, finance and government documents","counted":true,"results":39,"models":39},{"id":"perceptionBench","slug":"perception-bench","name":"PerceptionBench (Internal)","description":"Moonshot AI's internal benchmark for atomic visual perception capabilities.","category":"multimodal","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"Internal atomic visual-perception tasks","counted":false,"results":5,"models":5},{"id":"perceptionBenchTools","slug":"perception-bench-tools","name":"PerceptionBench with tools","description":"PerceptionBench fine-grained visual perception tasks run with tool use, as reported by Meta.","category":"multimodal","domain":null,"unit":"percent","publisher":"Meta (reported run)","url":"https://research.meta.ai/blog/muse-ember","year":"2026","tasks":"Fine-grained visual-perception tasks with tools","counted":false,"results":3,"models":3},{"id":"physicianBench","slug":"physician-bench","name":"PhysicianBench","description":"A public benchmark of 100 physician tasks carried out inside an electronic health record system.","category":"knowledge","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"100 EHR physician tasks","counted":false,"results":5,"models":5},{"id":"pinchBench","slug":"pinch-bench","name":"PinchBench","description":"An OpenClaw agent benchmark from Kilo that measures successful task completion across standardized real-world agent workflows.","category":"agentic","domain":null,"unit":"percent","publisher":"Kilo Code","url":"https://pinchbench.com/about","year":"2026","tasks":"23 OpenClaw agent tasks","counted":true,"results":10,"models":10},{"id":"polyMath","slug":"poly-math","name":"PolyMath","description":"A multilingual mathematical reasoning benchmark that tests whether math performance transfers across languages rather than only in English.","category":"multilingual","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"Multilingual math problems","counted":false,"results":6,"models":6},{"id":"postTrainBench","slug":"post-train-bench","name":"PostTrain Bench","description":"A software-engineering benchmark for post-training infrastructure and implementation tasks, evaluated through the official Harbor implementation.","category":"coding","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"Post-training software-engineering tasks","counted":false,"results":3,"models":3},{"id":"postTrainBench11","slug":"post-train-bench11","name":"PostTrainBench v1.1","description":"Post-training four base language models across seven weighted benchmarks, with ten hours and one H100 per run.","category":"coding","domain":null,"unit":"percent","publisher":"PostTrainBench team","url":"https://posttrainbench.com/?version=v1.1","year":"2026","tasks":"Post-training Qwen3 1.7B and 4B, SmolLM3 3B, and Gemma3 4B","counted":true,"results":15,"models":15},{"id":"postTrainBench12","slug":"post-train-bench12","name":"PostTrainBench v1.2","description":"Version 1.2 of PostTrainBench, which asks an agent to post-train a base model and scores the resulting model.","category":"coding","domain":null,"unit":"percent","publisher":"PostTrainBench authors","url":"https://posttrainbench.com/","year":"2026","tasks":"Agent post-training tasks","counted":false,"results":3,"models":3},{"id":"vals-ai:programbench:overall","slug":"vals-ai-programbench","name":"ProgramBench","description":"Can language models rebuild programs from scratch?","category":"coding","domain":null,"unit":"percent","publisher":"Vals AI","url":"https://www.vals.ai/benchmarks/programbench","year":null,"tasks":null,"counted":false,"results":65,"models":64},{"id":"programBenchEpisode1","slug":"program-bench-episode1","name":"ProgramBench hidden-test pass rate after episode 1","description":"Program-reconstruction hidden-test pass rate after the first of five sequential long-context episodes.","category":"coding","domain":null,"unit":"percent","publisher":"Yang et al.","url":"https://arxiv.org/abs/2605.03546","year":"2026","tasks":"166 golden program-reconstruction tasks","counted":false,"results":1,"models":1},{"id":"programBench","slug":"program-bench","name":"ProgramBench: Can Language Models Rebuild Programs From Scratch?","description":"A cleanroom software-engineering benchmark where agents receive only a compiled executable and documentation, then must architect and implement a complete codebase that reproduces the original program's behavior.","category":"coding","domain":null,"unit":"percent","publisher":"John Yang et al.","url":"https://programbench.com/static/paper.pdf","year":"2026","tasks":"200 program reconstruction tasks","counted":true,"results":19,"models":19},{"id":"vals-ai:proof_bench:overall","slug":"vals-ai-proof-bench","name":"ProofBench v1.1","description":"Automated theorem proving benchmark","category":"math","domain":null,"unit":"percent","publisher":"Vals AI","url":"https://www.vals.ai/benchmarks/proof_bench","year":null,"tasks":null,"counted":true,"results":54,"models":54},{"id":"proteinDomainIdentification","slug":"protein-domain-identification","name":"Protein domain identification","description":"Anthropic's protein-understanding test: from an amino acid sequence alone, choose the correct Pfam domain family among similar distractors, without tools.","category":"knowledge","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"Multiple-choice Pfam family questions","counted":false,"results":3,"models":3},{"id":"proteinMutationAttribution","slug":"protein-mutation-attribution","name":"Protein mutation attribution","description":"Anthropic's life-sciences evaluation of whether a model can attribute the effects of protein mutations.","category":"knowledge","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"Protein mutation questions","counted":false,"results":3,"models":3},{"id":"proteinGymHard","slug":"protein-gym-hard","name":"ProteinGym Hard","description":"Predicts mutation effects by ranking mutant protein sequences against wild type and comparing against laboratory measurements.","category":"knowledge","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"Hard protein mutation-effect ranking tasks","counted":false,"results":1,"models":1},{"id":"vals-ai:public-benefits-bench-v1:overall","slug":"vals-ai-public-benefits-bench-v1","name":"Public Benefits Bench v1","description":"Can AI help people navigate SNAP benefits?","category":null,"domain":"Public benefits","unit":"percent","publisher":"Vals AI","url":"https://www.vals.ai/benchmarks/public-benefits-bench-v1","year":null,"tasks":null,"counted":false,"results":13,"models":13},{"id":"vals-ai:public-benefits-bench:overall","slug":"vals-ai-public-benefits-bench","name":"Public Benefits Bench v1.1","description":"Can AI help people navigate SNAP benefits?","category":null,"domain":"Public benefits","unit":"percent","publisher":"Vals AI","url":"https://www.vals.ai/benchmarks/public-benefits-bench","year":null,"tasks":null,"counted":false,"results":50,"models":50},{"id":"qwenClawBench","slug":"qwen-claw-bench","name":"QwenClawBench","description":"Qwen's internal OpenClaw-style benchmark for measuring broad real-world agent performance across practical productivity and research tasks.","category":"agentic","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"Real-world agent workflows","counted":true,"results":10,"models":10},{"id":"ragTruthProvider","slug":"rag-truth-provider","name":"RAGTruth: provider-reported F1","description":"Corpus of retrieval-augmented responses annotated word by word for unsupported or contradicted content.","category":"knowledge","domain":null,"unit":"percent","publisher":"RAGTruth authors","url":"https://github.com/ParticleMedia/RAGTruth","year":"2024","tasks":"Retrieval-augmented question answering, summarization and data-to-text","counted":false,"results":2,"models":2},{"id":"reactNativeEvals","slug":"react-native-evals","name":"React Native Evals","description":"An open benchmark for AI coding agents on real-world React Native implementation tasks, emphasizing working app behavior, recommended architecture choices, and strict constraint adherence.","category":"coding","domain":null,"unit":"percent","publisher":"Callstack","url":"https://rn-evals.vercel.app/","year":"2026","tasks":"React Native app implementation tasks","counted":true,"results":16,"models":16},{"id":"realWorldQa","slug":"real-world-qa","name":"RealWorldQA","description":"A grounded visual QA benchmark focused on answering practical questions about real-world images and scenes.","category":"multimodal","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"Real-world visual question answering","counted":true,"results":18,"models":18},{"id":"refcocoAvg","slug":"refcoco-avg","name":"RefCOCO average","description":"A referring-expression grounding benchmark averaged across RefCOCO variants to test whether a model can localize described objects correctly.","category":"multimodal","domain":null,"unit":"percent","publisher":"RefCOCO dataset authors","url":"https://github.com/lichengunc/refer","year":"2026","tasks":"Referring-expression grounding","counted":true,"results":10,"models":10},{"id":"cyberHarmfulRefusal","slug":"cyber-harmful-refusal","name":"Refusal of harmful cyber requests (JailbreakBench, StrongREJECT, AgentHarm)","description":"Average refusal rate on the cyber-related prompts of JailbreakBench, StrongREJECT and AgentHarm, as measured by Mistral. Higher means the model refuses more malicious cyber requests.","category":"agentic","domain":null,"unit":"percent","publisher":"Mistral AI (composite of JailbreakBench, StrongREJECT and AgentHarm)","url":"https://mistral.ai/news/mistral-large-4-oss/","year":"2026","tasks":"Cyber-related harmful prompts from three safety benchmarks","counted":false,"results":4,"models":4},{"id":"relyToolBench","slug":"rely-tool-bench","name":"RelyToolBench","description":"Tool-hallucination benchmark that checks whether a model selects and calls tools that fit the request.","category":"agentic","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"Tool-use requests","counted":false,"results":2,"models":2},{"id":"rliAutomationRate","slug":"rli-automation-rate","name":"Remote Labor Index (RLI)","description":"Scale AI and the Center for AI Safety's benchmark of 240 real paid freelance projects; the automation rate is the share of projects whose deliverable a client would accept.","category":"agentic","domain":null,"unit":"percent","publisher":"Center for AI Safety; Scale AI","url":"https://scale.com/leaderboard/rli","year":"2025","tasks":"240 freelance projects","counted":false,"results":4,"models":4},{"id":"researchClawBench","slug":"research-claw-bench","name":"ResearchClawBench","description":"An end-to-end autonomous scientific research benchmark with 40 tasks across 10 scientific domains, where agents receive related literature and raw data, then attempt to rediscover the hidden target paper.","category":"agentic","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"40 tasks across 10 scientific domains","counted":false,"results":19,"models":19},{"id":"riemannBenchWithTools","slug":"riemann-bench-with-tools","name":"RiemannBench with tools","description":"Research-level mathematics problems with unique programmatically verified closed-form answers and tool access.","category":"math","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"25 private research-level mathematics problems","counted":false,"results":1,"models":1},{"id":"riemannBench","slug":"riemann-bench","name":"RiemannBench without tools","description":"Research-level mathematics problems with unique programmatically verified closed-form answers.","category":"math","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"25 private research-level mathematics problems","counted":false,"results":1,"models":1},{"id":"roboDojoSim","slug":"robo-dojo-sim","name":"RoboDojo Sim","description":"A simulated robotics benchmark that measures the average success rate of a model controlling robots across simulated manipulation tasks.","category":"agentic","domain":null,"unit":"percent","publisher":"Anthropic (reported run)","url":"https://www.anthropic.com/claude-fable-and-mythos-5-2","year":"2026","tasks":"Simulated robot-control tasks (count not published)","counted":false,"results":5,"models":5},{"id":"roleBench","slug":"role-bench","name":"RoleBench (RoleLLM)","description":"Character role-play benchmark from the RoleLLM project that tests whether a model speaks in a given character's style.","category":"instruction","domain":null,"unit":"percent","publisher":"RoleLLM authors","url":"https://github.com/InteractiveNLP-Team/RoleLLM-public","year":"2023","tasks":"Character role-play prompts","counted":false,"results":2,"models":2},{"id":"roleMrc","slug":"role-mrc","name":"RoleMRC","description":"Role-playing benchmark that checks whether a model keeps to a defined role identity while following instructions.","category":"instruction","domain":null,"unit":"percent","publisher":"RoleMRC authors","url":"https://github.com/LuJunru/RoleMRC","year":"2025","tasks":"Role-play and in-role instruction prompts","counted":false,"results":2,"models":2},{"id":"ruler128k","slug":"ruler128k","name":"RULER at 128K context","description":"Synthetic long-context benchmark of retrieval, multi-hop tracing, aggregation and question answering, scored at a 128K-token context.","category":"reasoning","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"Synthetic long-context tasks","counted":false,"results":1,"models":1},{"id":"ruler64k","slug":"ruler64k","name":"RULER at a 64K-token context","description":"Synthetic long-context benchmark of retrieval, multi-hop tracing, aggregation and question answering, scored at a 64K-token context.","category":"reasoning","domain":null,"unit":"percent","publisher":"RULER authors","url":"https://github.com/NVIDIA/RULER","year":"2024","tasks":"Synthetic long-context tasks","counted":false,"results":2,"models":2},{"id":"vals-ai:sage:overall","slug":"vals-ai-sage","name":"SAGE","description":"Student Assessment with Generative Evaluation","category":null,"domain":"Education","unit":"percent","publisher":"Vals AI","url":"https://www.vals.ai/benchmarks/sage","year":null,"tasks":null,"counted":false,"results":29,"models":29},{"id":"scarfBench","slug":"scarf-bench","name":"ScarfBench: Cross-Framework Application Migration in Enterprise Java","description":"IBM Research's benchmark for coding agents that migrate complete enterprise Java applications between Jakarta EE, Quarkus and Spring while keeping them working.","category":"coding","domain":null,"unit":"percent","publisher":"Advait Pavuluri et al.","url":"https://arxiv.org/abs/2605.06754","year":"2026","tasks":"204 framework migration tasks across 34 enterprise Java applications","counted":true,"results":11,"models":11},{"id":"sciCode","slug":"sci-code","name":"Scientific Code Benchmark","description":"SciCode evaluates language models on generating code for realistic scientific research problems across 16 subfields of physics, math, chemistry, biology, and material science. Problems decompose into 338 subproblems requiring domain knowledge recall, scientific reasoning, and precise code synthesis. Based on real scripts from published research.","category":"coding","domain":null,"unit":"percent","publisher":"","url":null,"year":"2024","tasks":80,"counted":true,"results":35,"models":35},{"id":"screenSpotPro","slug":"screen-spot-pro","name":"ScreenSpot Pro","description":"A GUI-grounding benchmark for 1,581 instructions in full-screen, high-resolution professional interfaces. It tests where a target is, not whether an agent can finish the surrounding workflow.","category":"multimodal","domain":null,"unit":"percent","publisher":"Kaixin Li et al.","url":"https://arxiv.org/abs/2504.07981","year":"2025","tasks":"1,581 grounding instructions","counted":true,"results":20,"models":20},{"id":"epoch-ai:epoch-simpleqa-verified:overall","slug":"epoch-ai-epoch-simpleqa-verified","name":"SimpleQA Verified","description":"Evaluated by Epoch AI on their own harness.","category":"knowledge","domain":null,"unit":"percent","publisher":"Epoch AI","url":"https://epoch.ai/benchmarks","year":null,"tasks":null,"counted":true,"results":91,"models":91},{"id":"simpleVqa","slug":"simple-vqa","name":"SimpleVQA","description":"A visual question answering benchmark focused on straightforward image-grounded understanding.","category":"multimodal","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"Visual QA tasks","counted":true,"results":18,"models":18},{"id":"vals-ai:skillsbench:overall","slug":"vals-ai-skillsbench","name":"SkillsBench","description":"How important are skills for agents?","category":"coding","domain":null,"unit":"percent","publisher":"Vals AI","url":"https://www.vals.ai/benchmarks/skillsbench","year":null,"tasks":null,"counted":true,"results":35,"models":35},{"id":"sweVerified","slug":"swe-verified","name":"Software Engineering Benchmark Verified","description":"SWE-bench Verified tests patches for 500 human-filtered GitHub issues. A score measures how many tasks an evaluated model-and-agent setup resolved; the model name alone does not identify that setup.","category":"coding","domain":null,"unit":"percent","publisher":"Carlos E. Jimenez et al.","url":"https://arxiv.org/abs/2310.06770","year":"2024","tasks":"500 verified issues","counted":true,"results":114,"models":114},{"id":"spider2Lite","slug":"spider2-lite","name":"Spider 2.0-Lite","description":"A text-to-SQL benchmark over realistic warehouse-scale schemas, reported by Interfaze for model comparison.","category":"coding","domain":null,"unit":"percent","publisher":"Spider 2.0 authors","url":"https://github.com/xlang-ai/Spider2","year":"2024","tasks":"Text-to-SQL queries","counted":false,"results":1,"models":1},{"id":"epoch-ai:epoch-spireclimb:overall","slug":"epoch-ai-epoch-spireclimb","name":"SpireClimb","description":"Evaluated by Epoch AI on their own harness.","category":null,"domain":"Other","unit":"percent","publisher":"Epoch AI","url":"https://epoch.ai/benchmarks","year":null,"tasks":null,"counted":false,"results":9,"models":9},{"id":"epoch-ai:epoch-spireclimb-slay-the-spire-1-ascension-20-heart:overall","slug":"epoch-ai-epoch-spireclimb-slay-the-spire-1-ascension-20-heart","name":"SpireClimb (Slay the Spire 1, Ascension 20 Heart)","description":"Evaluated by Epoch AI on their own harness.","category":null,"domain":"Other","unit":"percent","publisher":"Epoch AI","url":"https://epoch.ai/benchmarks","year":null,"tasks":null,"counted":false,"results":9,"models":9},{"id":"epoch-ai:epoch-spireclimb-slay-the-spire-2-ascension-10:overall","slug":"epoch-ai-epoch-spireclimb-slay-the-spire-2-ascension-10","name":"SpireClimb (Slay the Spire 2, Ascension 10)","description":"Evaluated by Epoch AI on their own harness.","category":null,"domain":"Other","unit":"percent","publisher":"Epoch AI","url":"https://epoch.ai/benchmarks","year":null,"tasks":null,"counted":false,"results":9,"models":9},{"id":"spreadsheetBench2","slug":"spreadsheet-bench2","name":"SpreadsheetBench 2","description":"A spreadsheet-focused benchmark for agentic analysis and editing workflows.","category":"agentic","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"Spreadsheet analysis and editing tasks","counted":false,"results":4,"models":4},{"id":"vals-ai:srebench:overall","slug":"vals-ai-srebench","name":"SRE Bench","description":"Can AI agents reverse engineer real-world, contamination-free binaries?","category":null,"domain":"Other","unit":"percent","publisher":"Vals AI","url":"https://www.vals.ai/benchmarks/srebench","year":null,"tasks":null,"counted":false,"results":36,"models":36},{"id":"stemBench120","slug":"stem-bench120","name":"STEMBench120: six-domain scientific research progress","description":"Scores how far a model advances 120 open research tasks across biology, chemistry, physics, astronomy, Earth science and engineering on a 0 to 4 progress scale, crediting supported partial work and honest statements of what remains unresolved.","category":"reasoning","domain":null,"unit":"percent","publisher":"Ulam","url":"https://www.ulam.ai/stembench","year":"2026","tasks":"120 research tasks, 20 in each of six STEM domains","counted":false,"results":9,"models":9},{"id":"structEvalT","slug":"struct-eval-t","name":"StructEval text-format subset","description":"Measures whether a model produces valid structured output in text formats such as JSON, YAML and CSV.","category":"instruction","domain":null,"unit":"percent","publisher":"StructEval authors","url":"https://structeval.github.io/","year":"2025","tasks":"Structured-output generation and conversion","counted":false,"results":2,"models":2},{"id":"sobValueAcc","slug":"sob-value-acc","name":"Structured Output Benchmark Value Accuracy","description":"A structured-output benchmark from Interfaze measuring whether extracted JSON leaf values exactly match verified ground truth.","category":"instruction","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"Structured output extraction","counted":false,"results":1,"models":1},{"id":"superGpqa","slug":"super-gpqa","name":"SuperGPQA: Scaling LLM Evaluation Across 285 Graduate Disciplines","description":"An expanded version of GPQA that evaluates graduate-level knowledge and reasoning capabilities across 285 disciplines, providing comprehensive coverage of academic domains.","category":"knowledge","domain":null,"unit":"percent","publisher":"Xiaoxuan Du et al.","url":"https://arxiv.org/abs/2502.14739","year":"2025","tasks":"285 disciplines","counted":true,"results":22,"models":22},{"id":"svgBench","slug":"svg-bench","name":"SVG-Bench","description":null,"category":"coding","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"SVG generation and editing tasks","counted":false,"results":1,"models":1},{"id":"sweRefactorBench","slug":"swe-refactor-bench","name":"SWE Refactor Bench: Can Coding Agents Complete a Long-Horizon, Whole-Repository Stack Migration?","description":"Tests whether coding agents can complete long-horizon, whole-repository stack migrations while preserving the original program's behavior.","category":"agentic","domain":null,"unit":"percent","publisher":"Deyao Hong et al.","url":"https://arxiv.org/abs/2608.23564","year":"2026","tasks":"20 whole-repository stack migrations","counted":false,"results":1,"models":1},{"id":"sweAtlasCodebaseQna","slug":"swe-atlas-codebase-qna","name":"SWE-Atlas Codebase QnA","description":"A code-comprehension benchmark covering production repositories across Go, Python, C, and TypeScript.","category":"coding","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"124 codebase questions across 11 repositories","counted":false,"results":8,"models":8},{"id":"vals-ai:swebench:overall","slug":"vals-ai-swebench","name":"SWE-bench","description":"Solving production software engineering tasks","category":"coding","domain":null,"unit":"percent","publisher":"Vals AI","url":"https://www.vals.ai/benchmarks/swebench","year":null,"tasks":null,"counted":true,"results":87,"models":87},{"id":"swebench-experiments:swe-bench-test:overall","slug":"swebench-experiments-swe-bench-test","name":"SWE-bench (full test split)","description":"Resolved-issue rate on the full 2,294-issue SWE-bench test split, which the leaderboard site does not publish a board for.","category":"coding","domain":null,"unit":"percent","publisher":"SWE-bench team","url":"https://github.com/SWE-bench/experiments/tree/main/evaluation/test","year":null,"tasks":null,"counted":false,"results":7,"models":7},{"id":"swebench-experiments:swe-bench-lite:overall","slug":"swebench-experiments-swe-bench-lite","name":"SWE-bench Lite · SWE-bench/experiments","description":"Resolved-issue rate on the 300-issue Lite split, from the submission records behind the leaderboard.","category":"coding","domain":null,"unit":"percent","publisher":"SWE-bench team","url":"https://github.com/SWE-bench/experiments/tree/main/evaluation/lite","year":null,"tasks":null,"counted":true,"results":15,"models":13},{"id":"swe-bench:swe-bench-lite:overall","slug":"swe-bench-swe-bench-lite","name":"SWE-bench Lite · swebench.com","description":"Resolved-issue rate on real GitHub issues, as published by the SWE-bench team.","category":"coding","domain":null,"unit":"percent","publisher":"SWE-bench team","url":"https://www.swebench.com/","year":null,"tasks":null,"counted":true,"results":15,"models":15},{"id":"swebench-experiments:swe-bench-multilingual:overall","slug":"swebench-experiments-swe-bench-multilingual","name":"SWE-bench Multilingual · SWE-bench/experiments","description":"Resolved-issue rate on issues across nine programming languages, from the submission records behind the leaderboard.","category":"coding","domain":null,"unit":"percent","publisher":"SWE-bench team","url":"https://github.com/SWE-bench/experiments/tree/main/evaluation/multilingual","year":null,"tasks":null,"counted":false,"results":14,"models":14},{"id":"swe-bench:swe-bench-multilingual:overall","slug":"swe-bench-swe-bench-multilingual","name":"SWE-bench Multilingual · swebench.com","description":"Resolved-issue rate on real GitHub issues, as published by the SWE-bench team.","category":"multilingual","domain":null,"unit":"percent","publisher":"SWE-bench team","url":"https://www.swebench.com/","year":null,"tasks":null,"counted":false,"results":12,"models":12},{"id":"swebench-experiments:swe-bench-multimodal:overall","slug":"swebench-experiments-swe-bench-multimodal","name":"SWE-bench Multimodal · SWE-bench/experiments","description":"Resolved-issue rate on JavaScript issues whose reports carry images, from the submission records behind the leaderboard.","category":"multimodal","domain":null,"unit":"percent","publisher":"SWE-bench team","url":"https://github.com/SWE-bench/experiments/tree/main/evaluation/multimodal","year":null,"tasks":null,"counted":false,"results":7,"models":7},{"id":"swe-bench:swe-bench-multimodal:overall","slug":"swe-bench-swe-bench-multimodal","name":"SWE-bench Multimodal · swebench.com","description":"Resolved-issue rate on real GitHub issues, as published by the SWE-bench team.","category":"multimodal","domain":null,"unit":"percent","publisher":"SWE-bench team","url":"https://www.swebench.com/","year":null,"tasks":null,"counted":false,"results":8,"models":8},{"id":"swePro","slug":"swe-pro","name":"SWE-bench Pro","description":"A long-horizon repository benchmark built to test realistic software engineering work. Its scores need a task-quality and setup check before they support a coding-agent decision.","category":"coding","domain":null,"unit":"percent","publisher":"Xiang Deng et al.","url":"https://arxiv.org/abs/2509.16941","year":"2025","tasks":"1,865 repository problems","counted":true,"results":83,"models":83},{"id":"epoch-ai:epoch-swe-bench-verified:overall","slug":"epoch-ai-epoch-swe-bench-verified","name":"SWE-Bench verified · Epoch AI","description":"Evaluated by Epoch AI on their own harness.","category":"coding","domain":null,"unit":"percent","publisher":"Epoch AI","url":"https://epoch.ai/benchmarks","year":null,"tasks":null,"counted":true,"results":32,"models":32},{"id":"swebench-experiments:swe-bench-verified:overall","slug":"swebench-experiments-swe-bench-verified","name":"SWE-bench Verified · SWE-bench/experiments","description":"Resolved-issue rate on the 500 human-validated SWE-bench issues, from the submission records behind the leaderboard.","category":"coding","domain":null,"unit":"percent","publisher":"SWE-bench team","url":"https://github.com/SWE-bench/experiments/tree/main/evaluation/verified","year":null,"tasks":null,"counted":true,"results":65,"models":65},{"id":"swe-bench:swe-bench-verified:overall","slug":"swe-bench-swe-bench-verified","name":"SWE-bench Verified · swebench.com","description":"Resolved-issue rate on real GitHub issues, as published by the SWE-bench team.","category":"coding","domain":null,"unit":"percent","publisher":"SWE-bench team","url":"https://www.swebench.com/","year":null,"tasks":null,"counted":true,"results":46,"models":46},{"id":"sweVerifiedArcee","slug":"swe-verified-arcee","name":"SWE-bench Verified (mini-swe-agent-v2)","description":null,"category":"coding","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"Repository task completion","counted":true,"results":5,"models":5},{"id":"sweMarathon","slug":"swe-marathon","name":"SWE-Marathon","description":"A long-horizon software engineering benchmark from Abundant AI with multi-hour tasks spanning library reproductions, full-stack product clones, and ML engineering.","category":"coding","domain":null,"unit":"percent","publisher":"Abundant AI and BenchFlow","url":"https://www.swe-marathon.org/","year":"2026","tasks":"20 multi-hour software engineering tasks","counted":false,"results":9,"models":9},{"id":"sweRebench","slug":"swe-rebench","name":"SWE-Rebench","description":"A continuously updated software engineering benchmark by Nebius using fresh GitHub issues to avoid contamination. Models are evaluated 5 times per problem under a fixed ReAct scaffolding; the Resolved Rate (best pass@1) is reported.","category":"coding","domain":null,"unit":"percent","publisher":"Nebius","url":"https://swe-rebench.com","year":"2026","tasks":"Fresh GitHub issues (rolling window)","counted":false,"results":13,"models":13},{"id":"sysBench","slug":"sys-bench","name":"SysBench: system message following","description":"Tests how well a model follows the rules in a system message across single and multi-turn conversations.","category":"instruction","domain":null,"unit":"percent","publisher":"SysBench authors","url":"https://github.com/PKU-Baichuan-MLSystemLab/SysBench","year":"2024","tasks":"System-message constraints over conversations","counted":false,"results":2,"models":2},{"id":"vals-ai:tax_agent_bench:overall","slug":"vals-ai-tax-agent-bench","name":"Tax Agent Bench","description":"Evaluating agents on research-grade US tax questions using the Tax Agent Bench harness","category":null,"domain":"Finance","unit":"percent","publisher":"Vals AI","url":"https://www.vals.ai/benchmarks/tax_agent_bench","year":null,"tasks":null,"counted":false,"results":82,"models":82},{"id":"vals-ai:tax_eval_v2:overall","slug":"vals-ai-tax-eval-v2","name":"TaxEval v2","description":"A Vals-created set of questions and responses to tax questions","category":null,"domain":"Finance","unit":"percent","publisher":"Vals AI","url":"https://www.vals.ai/benchmarks/tax_eval_v2","year":null,"tasks":null,"counted":false,"results":143,"models":142},{"id":"vals-ai:terminal-bench:overall","slug":"vals-ai-terminal-bench","name":"Terminal-Bench 1.0","description":"State-of-the-art set of difficult terminal-based tasks","category":"agentic","domain":null,"unit":"percent","publisher":"Vals AI","url":"https://www.vals.ai/benchmarks/terminal-bench","year":null,"tasks":null,"counted":true,"results":47,"models":46},{"id":"terminalBench2","slug":"terminal-bench2","name":"Terminal-Bench 2.0 · BenchLM","description":"A benchmark for agentic software engineering tasks executed in real terminal environments. Models must inspect files, run commands, edit code, and recover from errors over multi-step workflows.","category":"agentic","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"Terminal-based software tasks","counted":true,"results":41,"models":41},{"id":"vals-ai:terminal-bench-2:overall","slug":"vals-ai-terminal-bench-2","name":"Terminal-Bench 2.0 · Vals AI","description":"State-of-the-art set of difficult terminal-based tasks","category":"agentic","domain":null,"unit":"percent","publisher":"Vals AI","url":"https://www.vals.ai/benchmarks/terminal-bench-2","year":null,"tasks":null,"counted":true,"results":66,"models":66},{"id":"terminalBench21","slug":"terminal-bench21","name":"Terminal-Bench 2.1 · BenchLM","description":null,"category":"agentic","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"Terminal-based software-agent tasks","counted":true,"results":86,"models":86},{"id":"vals-ai:terminal-bench-2-1:overall","slug":"vals-ai-terminal-bench-2-1","name":"Terminal-Bench 2.1 · Vals AI","description":"State-of-the-art set of difficult terminal-based tasks","category":"agentic","domain":null,"unit":"percent","publisher":"Vals AI","url":"https://www.vals.ai/benchmarks/terminal-bench-2-1","year":null,"tasks":null,"counted":true,"results":75,"models":75},{"id":"frontierBench","slug":"frontier-bench","name":"Terminal-Bench 3.0","description":"A continuously maintained benchmark for difficult computer work, including coding, deep learning, finance, engineering, math, and science tasks.","category":"agentic","domain":null,"unit":"percent","publisher":"Ryan Marten et al.","url":"https://www.frontierbench.ai/","year":"2026","tasks":"74 professional computer-work tasks across 7 domains","counted":true,"results":13,"models":13},{"id":"vals-ai:terminal-bench-4:overall","slug":"vals-ai-terminal-bench-4","name":"Terminal-Bench 4.0","description":"Frontier-difficulty terminal tasks across software, science, ML, operations, hardware, security, and media","category":"agentic","domain":null,"unit":"percent","publisher":"Vals AI","url":"https://www.vals.ai/benchmarks/terminal-bench-4","year":null,"tasks":null,"counted":true,"results":61,"models":61},{"id":"terminal-bench:terminal-bench-4-0-0:overall","slug":"terminal-bench-terminal-bench-4-0-0","name":"Terminal-Bench 4.0.0","description":"Agentic terminal task suite run by the Terminal-Bench team.","category":"agentic","domain":null,"unit":"percent","publisher":"Terminal-Bench","url":"https://www.tbench.ai/leaderboard","year":null,"tasks":null,"counted":true,"results":23,"models":23},{"id":"terminal-bench:terminal-bench-4-1-0:overall","slug":"terminal-bench-terminal-bench-4-1-0","name":"Terminal-Bench 4.1.0","description":"Agentic terminal task suite run by the Terminal-Bench team.","category":"agentic","domain":null,"unit":"percent","publisher":"Terminal-Bench","url":"https://www.tbench.ai/leaderboard","year":null,"tasks":null,"counted":true,"results":23,"models":23},{"id":"terminalBenchHard","slug":"terminal-bench-hard","name":"Terminal-Bench Hard","description":null,"category":"agentic","domain":null,"unit":"percent","publisher":"Artificial Analysis","url":"https://artificialanalysis.ai/evaluations/terminalbench-hard","year":"2026","tasks":"Agentic coding and terminal tasks","counted":true,"results":23,"models":23},{"id":"vals-ai:terminal-bench-science:overall","slug":"vals-ai-terminal-bench-science","name":"Terminal-Bench-Science","description":"Research workflow tasks contributed by practicing scientists across five domains","category":null,"domain":"Science","unit":"percent","publisher":"Vals AI","url":"https://www.vals.ai/benchmarks/terminal-bench-science","year":null,"tasks":null,"counted":false,"results":44,"models":44},{"id":"terminalBenchScience","slug":"terminal-bench-science","name":"Terminal-Bench-Science 0.1","description":"A benchmark of AI agents completing expert-curated research workflows across the life, physical, Earth, mathematical, and engineering sciences.","category":"agentic","domain":null,"unit":"percent","publisher":"Terminal-Bench-Science Team","url":"https://www.terminal-bench-science.ai/","year":"2026","tasks":"70 expert-curated scientific research workflows","counted":true,"results":11,"models":11},{"id":"terminalBenchScience6xVerifier","slug":"terminal-bench-science6x-verifier","name":"Terminal-Bench-Science 0.1 with sixfold verifier timeout","description":"Google-run scientific workflow evaluation using a sixfold verifier timeout to address verification timeouts.","category":"agentic","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"70 expert-curated scientific research workflows","counted":false,"results":1,"models":1},{"id":"terminalBench3","slug":"terminal-bench3","name":"terminalBench3","description":null,"category":"agentic","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":null,"counted":false,"results":2,"models":2},{"id":"musr","slug":"musr","name":"Testing the Limits of Chain-of-thought with Multistep Soft Reasoning","description":"A dataset for evaluating language models on multistep soft reasoning tasks specified in natural language narratives. Tests the ability to perform complex, structured reasoning.","category":"reasoning","domain":null,"unit":"percent","publisher":"Zayne Sprague et al.","url":"https://arxiv.org/abs/2310.16049","year":"2023","tasks":"Multi-step reasoning","counted":false,"results":6,"models":6},{"id":"vals-ai:time_horizon_index:overall","slug":"vals-ai-time-horizon-index","name":"Time Horizon Index: KSP","description":"Can an AI agent build and run a space program in Kerbal Space Program?","category":null,"domain":"Other","unit":"percent","publisher":"Vals AI","url":"https://www.vals.ai/benchmarks/time_horizon_index","year":null,"tasks":null,"counted":false,"results":13,"models":13},{"id":"toolathlon","slug":"toolathlon","name":"Toolathlon","description":"A tool-use benchmark focused on selecting, sequencing, and completing tasks with external tools.","category":"agentic","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"Multi-tool workflows","counted":true,"results":22,"models":22},{"id":"toolathlonVerifiedAvgTurns","slug":"toolathlon-verified-avg-turns","name":"Toolathlon Verified average assistant turns","description":"Average assistant turns per Toolathlon Verified trajectory.","category":"agentic","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"108 verified real-world tool-use tasks","counted":false,"results":4,"models":4},{"id":"toolathlonVerifiedPass3All","slug":"toolathlon-verified-pass3-all","name":"Toolathlon Verified Pass cubed","description":"Fraction of Toolathlon Verified tasks solved in all three independent trials.","category":"agentic","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"108 verified real-world tool-use tasks","counted":false,"results":4,"models":4},{"id":"toolathlonVerifiedPass3","slug":"toolathlon-verified-pass3","name":"Toolathlon Verified Pass@3","description":"Fraction of Toolathlon Verified tasks solved in at least one of three trials.","category":"agentic","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"108 verified real-world tool-use tasks","counted":false,"results":4,"models":4},{"id":"toolathlonVerified","slug":"toolathlon-verified","name":"Toolathlon-Verified","description":"A verified tool-use benchmark variant for completing multi-step workflows with external tools.","category":"agentic","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"Verified multi-tool workflows","counted":true,"results":23,"models":23},{"id":"toolSandbox","slug":"tool-sandbox","name":"ToolSandbox","description":"Apple benchmark of stateful, conversational tool use, where tool calls change state and a simulated user takes part.","category":"agentic","domain":null,"unit":"percent","publisher":"Apple","url":"https://github.com/apple-aiml-research/ToolSandbox","year":"2024","tasks":"Stateful tool-use conversations","counted":false,"results":3,"models":3},{"id":"usamo2026","slug":"usamo2026","name":"United States of America Mathematical Olympiad 2026","description":"The premier US mathematical olympiad competition, featuring proof-based problems that require deep mathematical insight and rigorous argumentation at the highest competition level.","category":"math","domain":null,"unit":"percent","publisher":"Mathematical Association of America","url":"https://www.maa.org/math-competitions/usamo","year":"2026","tasks":"6 proof-based problems","counted":false,"results":3,"models":3},{"id":"vStar","slug":"v-star","name":"V*","description":"A vision-centric benchmark for high-level multimodal reasoning and perception quality.","category":"multimodal","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"Frontier multimodal reasoning tasks","counted":true,"results":14,"models":14},{"id":"vals-ai:web_search:overall","slug":"vals-ai-web-search","name":"Vals Web Search Index","description":"Comparing native provider search against independent web-search tools on legal-research and finance-analysis tasks","category":null,"domain":"Other","unit":"percent","publisher":"Vals AI","url":"https://www.vals.ai/benchmarks/web_search","year":null,"tasks":null,"counted":false,"results":12,"models":12},{"id":"vals-ai:vcb-1-100:overall","slug":"vals-ai-vcb-1-100","name":"Vibe Code Bench 1-100","description":"Can models extend a working web application across a long sequence of dependent requests?","category":"coding","domain":null,"unit":"percent","publisher":"Vals AI","url":"https://www.vals.ai/benchmarks/vcb-1-100","year":null,"tasks":null,"counted":true,"results":26,"models":26},{"id":"vals-ai:vibe-code:overall","slug":"vals-ai-vibe-code","name":"Vibe Code Bench v1.1","description":"Can models build web applications from scratch?","category":"coding","domain":null,"unit":"percent","publisher":"Vals AI","url":"https://www.vals.ai/benchmarks/vibe-code","year":null,"tasks":null,"counted":true,"results":123,"models":123},{"id":"vibeV2","slug":"vibe-v2","name":"VIBE V2","description":null,"category":"coding","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"End-to-end coding-agent tasks","counted":false,"results":1,"models":1},{"id":"vibePro","slug":"vibe-pro","name":"VIBE-Pro","description":"A repo-level code generation and full-project delivery benchmark spanning web, mobile, and simulation-style implementation tasks.","category":"coding","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"Full project delivery tasks","counted":false,"results":1,"models":1},{"id":"videoMme","slug":"video-mme","name":"Video-MME","description":"A comprehensive benchmark for multimodal large language models on video understanding, covering temporal reasoning, perception, and question answering over videos.","category":"multimodal","domain":null,"unit":"percent","publisher":"Video-MME benchmark team","url":"https://mme-benchmark.github.io/","year":"2024","tasks":"Video understanding","counted":false,"results":4,"models":4},{"id":"videoMmeWithSub","slug":"video-mme-with-sub","name":"Video-MME with subtitle","description":"A video understanding benchmark that allows subtitle access when answering multimodal questions about videos.","category":"multimodal","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"Video understanding","counted":false,"results":12,"models":12},{"id":"videoMmeNoSub","slug":"video-mme-no-sub","name":"Video-MME without subtitle","description":"A stricter Video-MME setting that removes subtitle help and tests video understanding from visual and audio context alone.","category":"multimodal","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"Video understanding","counted":false,"results":8,"models":8},{"id":"videoMmmu","slug":"video-mmmu","name":"VideoMMMU","description":"A video extension of MMMU-style multimodal reasoning over expert questions grounded in temporal media.","category":"multimodal","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"Video-grounded expert reasoning","counted":false,"results":13,"models":13},{"id":"vision2Web","slug":"vision2-web","name":"Vision2Web","description":"A benchmark for converting visual references into functional web implementations.","category":"multimodal","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"Screenshot-to-web tasks","counted":false,"results":5,"models":5},{"id":"vitaBench","slug":"vita-bench","name":"VITA-Bench","description":"An interactive real-world agent benchmark grounded in practical consumer-service tasks such as delivery, in-store consumption, and online travel workflows.","category":"agentic","domain":null,"unit":"percent","publisher":"Meituan LongCat Team","url":"https://vitabench.github.io/","year":"2025","tasks":"Interactive consumer-service agent tasks","counted":true,"results":13,"models":13},{"id":"vulcanCiiV1","slug":"vulcan-cii-v1","name":"VulcanBench Coding Intelligence Index v1","description":null,"category":"coding","domain":null,"unit":"percent","publisher":"VulcanBench contributors","url":"https://github.com/morganlinton/VulcanBench/blob/main/docs/results/cii-v1-2026-08/README.md","year":"2026","tasks":"38 validated post-cutoff repository tasks","counted":false,"results":3,"models":3},{"id":"vulcanBench","slug":"vulcan-bench","name":"VulcanBench v3","description":"An open software-engineering benchmark built from real merged post-cutoff pull requests across Python, Rust, TypeScript, JavaScript, and Go repositories.","category":"coding","domain":null,"unit":"percent","publisher":"VulcanBench contributors","url":"https://github.com/morganlinton/VulcanBench/tree/main","year":"2026","tasks":"23 post-cutoff repository tasks in the v3 report","counted":true,"results":14,"models":14},{"id":"wandr","slug":"wandr","name":"WANDR (Wide ANd Deep Research)","description":"Perplexity's wide-search research benchmark: 500 tasks in which a model must find tens to hundreds of entities, collect facts about each, and return them as cited structured records.","category":"agentic","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"500 wide-search research tasks","counted":false,"results":3,"models":3},{"id":"weMath","slug":"we-math","name":"We-Math","description":"A multimodal math benchmark for visually grounded mathematical reasoning and answer generation.","category":"multimodal","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"Visually grounded math problems","counted":false,"results":6,"models":6},{"id":"webArena","slug":"web-arena","name":"WebArena Web Agent Benchmark","description":"WebArena tests whether a browser-agent system can complete 812 long-horizon tasks inside self-hosted replicas of functional websites. It checks the requested end state, so a result reflects the model, agent scaffold, browser interface, action budget, and evaluator together—not the base model alone.","category":"agentic","domain":null,"unit":"percent","publisher":"Shuyan Zhou et al.","url":"https://arxiv.org/abs/2307.13854","year":"2024","tasks":"812 long-horizon browser tasks","counted":false,"results":2,"models":2},{"id":"webArenaVerified","slug":"web-arena-verified","name":"WebArena-Verified Browser Agent Benchmark","description":"WebArena-Verified is an audited release of the WebArena browser-agent benchmark. It rechecks task descriptions, reference answers, and evaluators, and replaces nondeterministic judging with deterministic checks where possible.","category":"agentic","domain":null,"unit":"percent","publisher":"Amine El Hattami et al.","url":"https://openreview.net/forum?id=94tlGxmqkN","year":"2025","tasks":"812 verified tasks; separate 258-task Hard subset","counted":false,"results":5,"models":5},{"id":"webVoyager","slug":"web-voyager","name":"WebVoyager","description":"A browser-agent benchmark for completing multi-step workflows on live websites.","category":"agentic","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"Live website workflows","counted":false,"results":2,"models":2},{"id":"when2CallMultiTurn","slug":"when2-call-multi-turn","name":"When2Call Multi-Turn (NVIDIA)","description":"NVIDIA multi-turn version of When2Call that tests whether a model calls a tool, asks a follow-up question, or abstains.","category":"agentic","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"Multi-turn tool-use decisions (not yet published)","counted":false,"results":2,"models":2},{"id":"wideResearch","slug":"wide-research","name":"WideResearch","description":"A broad research-agent benchmark for open-ended information gathering, synthesis, and answer construction across wide search spaces.","category":"agentic","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"Open-ended research tasks","counted":true,"results":16,"models":16},{"id":"wmt26","slug":"wmt26","name":"WMT26 General Machine Translation (all languages)","description":"Machine-translation quality on the WMT26 general translation task, averaged across all evaluated languages and scored 0-100 by an LLM judge.","category":"multilingual","domain":null,"unit":"percent","publisher":"WMT organizers","url":"https://www2.statmt.org/wmt26/translation-task.html","year":"2026","tasks":"General-domain translation across 50+ languages (32 high-resource plus 18 additional)","counted":false,"results":1,"models":1},{"id":"worldVqaForceAnswer","slug":"world-vqa-force-answer","name":"WorldVQA ForceAnswer","description":"A forced-answer WorldVQA variant for atomic visual world knowledge.","category":"multimodal","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"Atomic visual world-knowledge questions","counted":false,"results":1,"models":1},{"id":"zeroBench","slug":"zero-bench","name":"ZeroBench","description":"A multi-step visual reasoning benchmark with pass@5 reporting and optional tool use.","category":"multimodal","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"100 visual reasoning questions","counted":true,"results":10,"models":10},{"id":"zeroBenchPython","slug":"zero-bench-python","name":"ZeroBench_main with Python","description":"A Python-assisted ZeroBench_main evaluation reported as pass@5.","category":"multimodal","domain":null,"unit":"percent","publisher":"","url":null,"year":null,"tasks":"Visual reasoning questions with Python","counted":false,"results":5,"models":5},{"id":"tau2-bench:airline:overall","slug":"tau2-bench-airline","name":"τ²-bench Airline","description":"Share of tasks an agent completes on one attempt, against a simulated user who holds information it must ask for.","category":"agentic","domain":null,"unit":"percent","publisher":"Sierra Research","url":"https://github.com/sierra-research/tau2-bench","year":null,"tasks":null,"counted":true,"results":22,"models":22},{"id":"tau2Airline","slug":"tau2-airline","name":"τ²-Bench Airline Domain","description":"τ²-bench Airline tests conversational agents on airline customer-service tasks governed by domain policy and database-changing tools.","category":"agentic","domain":null,"unit":"percent","publisher":"Victor Barres et al.","url":"https://arxiv.org/abs/2506.07982","year":"2025","tasks":"Airline customer-service tasks","counted":false,"results":5,"models":5},{"id":"tau2-bench:banking-knowledge:overall","slug":"tau2-bench-banking-knowledge","name":"τ²-bench Banking","description":"Share of tasks an agent completes on one attempt, against a simulated user who holds information it must ask for.","category":"agentic","domain":null,"unit":"percent","publisher":"Sierra Research","url":"https://github.com/sierra-research/tau2-bench","year":null,"tasks":null,"counted":true,"results":28,"models":28},{"id":"tau2-bench:retail:overall","slug":"tau2-bench-retail","name":"τ²-bench Retail","description":"Share of tasks an agent completes on one attempt, against a simulated user who holds information it must ask for.","category":"agentic","domain":null,"unit":"percent","publisher":"Sierra Research","url":"https://github.com/sierra-research/tau2-bench","year":null,"tasks":null,"counted":true,"results":23,"models":23},{"id":"tau2-bench:telecom:overall","slug":"tau2-bench-telecom","name":"τ²-bench Telecom","description":"Share of tasks an agent completes on one attempt, against a simulated user who holds information it must ask for.","category":"agentic","domain":null,"unit":"percent","publisher":"Sierra Research","url":"https://github.com/sierra-research/tau2-bench","year":null,"tasks":null,"counted":true,"results":18,"models":18},{"id":"tau2Bench","slug":"tau2-bench","name":"τ²-Bench Tool-Agent-User Evaluation","description":null,"category":"agentic","domain":null,"unit":"percent","publisher":"Victor Barres et al.","url":"https://arxiv.org/abs/2506.07982","year":"2025","tasks":"Airline, retail, and telecom customer-service task sets","counted":true,"results":150,"models":150},{"id":"tau3Bench","slug":"tau3-bench","name":"τ³-Bench Tool-Agent-User Evaluation","description":"τ³-bench is the current evolution of Sierra's tool-agent-user framework, adding corrected task releases and newer knowledge and voice evaluation modes alongside airline, retail, and telecom.","category":"agentic","domain":null,"unit":"percent","publisher":"Sierra Research","url":"https://github.com/sierra-research/tau2-bench","year":"2026","tasks":"Corrected customer-service tasks plus knowledge and voice evaluation modes","counted":true,"results":32,"models":32}],"providers":[{"slug":"anthropic","name":"Anthropic","models":86,"ranked":34,"top":{"name":"Claude Fable 5.2","slug":"claude-fable-5-2","score":85.61873265280894},"latest":"2026-11-09"},{"slug":"google","name":"Google","models":110,"ranked":26,"top":{"name":"Gemini 4 Carbon","slug":"gemini-4-carbon","score":82.6671329264836},"latest":"2026-11-11"},{"slug":"openai","name":"OpenAI","models":116,"ranked":38,"top":{"name":"GPT-6 Astra","slug":"gpt-6-astra","score":78.03632542729953},"latest":"2026-11-05"},{"slug":"meta","name":"Meta","models":49,"ranked":16,"top":{"name":"Muse Wonder Full","slug":"muse-wonder-full","score":77.28621216839132},"latest":"2026-11-09"},{"slug":"moonshot-ai","name":"Moonshot AI","models":15,"ranked":10,"top":{"name":"Kimi K3.1 Pro","slug":"kimi-3-1-pro","score":75.70562584326319},"latest":"2026-10-23"},{"slug":"tencent","name":"Tencent","models":14,"ranked":3,"top":{"name":"Hy4","slug":"hy4","score":72.03866231653724},"latest":"2026-11-04"},{"slug":"xiaomi","name":"Xiaomi","models":8,"ranked":7,"top":{"name":"MiMo-V2.6-Pro","slug":"mimo-v2-6-pro","score":70.15290697170158},"latest":"2026-09-22"},{"slug":"minimax","name":"MiniMax","models":12,"ranked":5,"top":{"name":"MiniMax M3.1 Flash","slug":"minimax-m3-1-flash","score":69.27835804080617},"latest":"2026-10-19"},{"slug":"xai","name":"xAI","models":33,"ranked":16,"top":{"name":"Grok 4.7","slug":"grok-4-7","score":69.13287977697333},"latest":"2026-09-21"},{"slug":"deepseek","name":"DeepSeek","models":35,"ranked":13,"top":{"name":"DeepSeek V4.1 Flash 1017","slug":"deepseek-v4-1-flash","score":68.50210603700033},"latest":"2026-10-17"},{"slug":"stepfun","name":"StepFun","models":5,"ranked":2,"top":{"name":"Step 5 Preview","slug":"step-5-preview","score":68.39429489260567},"latest":"2026-09-20"},{"slug":"inclusionai","name":"InclusionAI","models":15,"ranked":4,"top":{"name":"Ling 3.1 Flash","slug":"ling-3-1-flash","score":67.20803735717801},"latest":"2026-10-19"},{"slug":"thinking-machines-lab","name":"Thinking Machines Lab","models":3,"ranked":3,"top":{"name":"Inkling 2 Small","slug":"inkling-2-small","score":66.35512760452673},"latest":"2026-11-05"},{"slug":"sakana","name":"Sakana","models":8,"ranked":2,"top":{"name":"Sakana Fugu-Ultra","slug":"sakana-fugu-ultra-sakana-fugu-ultra","score":66.33022038858476},"latest":"2026-09-11T05:43:03.000Z"},{"slug":"z-ai","name":"Z.AI","models":25,"ranked":11,"top":{"name":"GLM-5.3","slug":"glm-5-3","score":65.83764831309777},"latest":"2026-11-02T13:29:14.000Z"},{"slug":"alibaba","name":"Alibaba","models":122,"ranked":24,"top":{"name":"Qwen3.8 Max","slug":"qwen3-8-max","score":65.56026544686976},"latest":"2026-09-23T19:20:28.000Z"},{"slug":"dots-studio","name":"Dots Studio","models":1,"ranked":1,"top":{"name":"dots3-note Preview","slug":"dots3-note-preview","score":63.39430292254704},"latest":"2026-08-14"},{"slug":"mistral","name":"Mistral","models":63,"ranked":11,"top":{"name":"Mistral Large 4","slug":"mistral-large-4","score":60.68193312186661},"latest":"2026-10-06"},{"slug":"internscience","name":"InternScience","models":11,"ranked":2,"top":{"name":"Sage-A1.5","slug":"sage-a1-5","score":60.52518469856023},"latest":"2026-10-21"},{"slug":"reflection-ai","name":"Reflection AI","models":3,"ranked":3,"top":{"name":"Beam 1.0 501B-A23B","slug":"reflection-beam","score":60.18237998368361},"latest":"2026-10-16"},{"slug":"bytedance","name":"ByteDance","models":16,"ranked":2,"top":{"name":"Seed 2.1 Pro","slug":"seed-2-1-pro","score":60.084466822352105},"latest":"2026-08-12T16:05:01.000Z"},{"slug":"apodex","name":"Apodex","models":2,"ranked":2,"top":{"name":"Apodex 1.1","slug":"apodex-1-1","score":58.886760364543846},"latest":"2026-08-24"},{"slug":"microsoft","name":"Microsoft","models":16,"ranked":3,"top":{"name":"MAI-Thinking-1","slug":"mai-thinking-1","score":53.54391829305899},"latest":"2026-10-12"},{"slug":"upstage","name":"Upstage","models":5,"ranked":3,"top":{"name":"Solar Open 2","slug":"solar-open-2","score":53.300528899497365},"latest":"2026-09-23T10:45:58.000Z"},{"slug":"inception","name":"Inception","models":6,"ranked":2,"top":{"name":"Mercury 3","slug":"mercury-3","score":52.32067017661652},"latest":"2026-10-19"},{"slug":"ricursive-intelligence","name":"Ricursive Intelligence","models":1,"ranked":1,"top":{"name":"Ripple-65B Preview","slug":"ripple-65b-preview","score":52.30073851637254},"latest":"2026-11-04"},{"slug":"prism-ml","name":"Prism ML","models":9,"ranked":1,"top":{"name":"Ternary Bonsai 2 27B","slug":"ternary-bonsai-2-27b","score":52.18291238717265},"latest":"2026-09-17"},{"slug":"sk-telecom","name":"SK Telecom","models":2,"ranked":1,"top":{"name":"A.X K2","slug":"a-x-k2","score":51.90031776096021},"latest":"2026-07-29"},{"slug":"nvidia","name":"NVIDIA","models":41,"ranked":9,"top":{"name":"Nemotron 3.5 Super VL 120B A12B","slug":"nemotron-3-5-super-vl-120b-a12b","score":51.04274101079333},"latest":"2026-10-23"},{"slug":"lg-ai-research","name":"LG AI Research","models":4,"ranked":3,"top":{"name":"K-EXAONE 2.0","slug":"k-exaone-2-0","score":48.82157240861064},"latest":"2026-07-31"},{"slug":"sparkllm","name":"SparkLLM","models":1,"ranked":1,"top":{"name":"Spark-X2.6-4B","slug":"spark-x2-6-4b","score":48.6794404062999},"latest":"2026-10-19"},{"slug":"arcee-ai","name":"Arcee AI","models":2,"ranked":2,"top":{"name":"Trinity-Large-Thinking","slug":"trinity-large-thinking","score":43.48151899345726},"latest":"2026-03-10"},{"slug":"ibm","name":"IBM","models":8,"ranked":6,"top":{"name":"Granite 4.2 30B","slug":"granite-4-2-30b","score":41.62665107491319},"latest":"2026-08-25"},{"slug":"liquidai","name":"LiquidAI","models":29,"ranked":4,"top":{"name":"LFM3-1.7B","slug":"lfm3-1-7b","score":41.4626143181411},"latest":"2026-10-27"},{"slug":"cohere","name":"Cohere","models":9,"ranked":3,"top":{"name":"Command A+","slug":"command-a-plus","score":40.24425156984971},"latest":"2026-09-10"},{"slug":"meituan","name":"Meituan","models":3,"ranked":1,"top":{"name":"LongCat-Flash-Lite-Sparse","slug":"longcat-flash-lite-sparse","score":39.2928618062931},"latest":"2026-11-02"},{"slug":"zyphra","name":"Zyphra","models":4,"ranked":1,"top":{"name":"ZAYA1-8B","slug":"zaya1-8b","score":38.20921005565147},"latest":"2026-06-12"},{"slug":"openbmb","name":"OpenBMB","models":3,"ranked":2,"top":{"name":"MiniCPM5-2B","slug":"minicpm5-2b","score":37.10047147141234},"latest":"2026-09-06"},{"slug":"sarvam","name":"Sarvam","models":2,"ranked":2,"top":{"name":"Sarvam 105B","slug":"sarvam-105b","score":36.71167484674234},"latest":"2026-03-06"},{"slug":"celeris","name":"Celeris","models":3,"ranked":1,"top":{"name":"Celeris-1","slug":"celeris-1","score":33.71657713389645},"latest":"2026-11-05"},{"slug":"amazon","name":"Amazon","models":9,"ranked":1,"top":{"name":"Nova Pro","slug":"nova-pro","score":25.593668316441114},"latest":"2025-12-02"},{"slug":"h-company","name":"H Company","models":14,"ranked":0,"top":null,"latest":"2026-06-01"},{"slug":"ifm","name":"IFM","models":11,"ranked":0,"top":null,"latest":"2026-10-15"},{"slug":"allen-institute-for-ai","name":"Allen Institute for AI","models":10,"ranked":0,"top":null,"latest":null},{"slug":"perplexity","name":"Perplexity","models":9,"ranked":0,"top":null,"latest":"2026-10-16"},{"slug":"fastino","name":"Fastino","models":7,"ranked":0,"top":null,"latest":"2026-10-22"},{"slug":"fixie-ai","name":"Fixie AI","models":7,"ranked":0,"top":null,"latest":null},{"slug":"aion-labs","name":"Aion Labs","models":6,"ranked":0,"top":null,"latest":"2026-09-23T13:42:42.000Z"},{"slug":"scaledown","name":"ScaleDown","models":6,"ranked":0,"top":null,"latest":"2026-10-13T12:09:20.000Z"},{"slug":"ai21-labs","name":"AI21 Labs","models":5,"ranked":0,"top":null,"latest":null},{"slug":"kakao","name":"Kakao","models":5,"ranked":0,"top":null,"latest":"2026-07-24"},{"slug":"pleias","name":"PleIAs","models":5,"ranked":0,"top":null,"latest":null},{"slug":"bad-theory-labs","name":"Bad Theory Labs","models":4,"ranked":0,"top":null,"latest":"2026-08-05"},{"slug":"flymy-ai","name":"FlyMy.AI","models":4,"ranked":0,"top":null,"latest":null},{"slug":"jared-palmer","name":"Jared Palmer","models":4,"ranked":0,"top":null,"latest":null},{"slug":"kirp","name":"kirp","models":4,"ranked":0,"top":null,"latest":null},{"slug":"mapika","name":"Mapika","models":4,"ranked":0,"top":null,"latest":null},{"slug":"nace-ai","name":"Nace.AI","models":4,"ranked":0,"top":null,"latest":"2026-10-28"},{"slug":"nous-research","name":"Nous Research","models":4,"ranked":0,"top":null,"latest":"2025-08-26T19:11:03.000Z"},{"slug":"ornith-ai","name":"Ornith AI","models":4,"ranked":0,"top":null,"latest":"2026-08-18"},{"slug":"poolside","name":"Poolside","models":4,"ranked":0,"top":null,"latest":"2026-07-21"},{"slug":"autotrust","name":"autotrust","models":3,"ranked":0,"top":null,"latest":null},{"slug":"baidu","name":"Baidu","models":3,"ranked":0,"top":null,"latest":"2026-11-11"},{"slug":"cloudflare","name":"Cloudflare","models":3,"ranked":0,"top":null,"latest":"2026-10-09"},{"slug":"cognition","name":"Cognition","models":3,"ranked":0,"top":null,"latest":"2026-11-04"},{"slug":"convai-innovations","name":"Convai Innovations","models":3,"ranked":0,"top":null,"latest":null},{"slug":"cursor","name":"Cursor","models":3,"ranked":0,"top":null,"latest":"2026-05-18"},{"slug":"deepreinforce-ai","name":"DeepReinforce AI","models":3,"ranked":0,"top":null,"latest":"2026-06-01"},{"slug":"denis-pplx","name":"denis-pplx","models":3,"ranked":0,"top":null,"latest":null},{"slug":"featherless-ai","name":"Featherless AI","models":3,"ranked":0,"top":null,"latest":null},{"slug":"glenn2","name":"glenn2","models":3,"ranked":0,"top":null,"latest":"2026-08-06"},{"slug":"hotchpotch","name":"hotchpotch","models":3,"ranked":0,"top":null,"latest":null},{"slug":"igor-l-ninfer-contributors","name":"Igor L. / NInfer contributors","models":3,"ranked":0,"top":null,"latest":null},{"slug":"internlm","name":"internlm","models":3,"ranked":0,"top":null,"latest":null},{"slug":"jetbrains","name":"JetBrains","models":3,"ranked":0,"top":null,"latest":"2026-10-08"},{"slug":"moondream","name":"Moondream","models":3,"ranked":0,"top":null,"latest":"2026-10-15T22:13:05.000Z"},{"slug":"nex-agi","name":"Nex AGI","models":3,"ranked":0,"top":null,"latest":"2026-11-10T03:46:19.000Z"},{"slug":"sao10k","name":"Sao10K","models":3,"ranked":0,"top":null,"latest":"2024-12-18T15:32:08.000Z"},{"slug":"thedrummer","name":"TheDrummer","models":3,"ranked":0,"top":null,"latest":"2025-09-27T00:11:18.000Z"},{"slug":"01-ai","name":"01.AI","models":2,"ranked":0,"top":null,"latest":null},{"slug":"aleph-alpha","name":"Aleph Alpha","models":2,"ranked":0,"top":null,"latest":"2024-08-26"},{"slug":"alibiserikbay","name":"alibiserikbay","models":2,"ranked":0,"top":null,"latest":null},{"slug":"allebee","name":"allebee","models":2,"ranked":0,"top":null,"latest":null},{"slug":"baichuan","name":"Baichuan","models":2,"ranked":0,"top":null,"latest":"2025-02-24"},{"slug":"baseten","name":"Baseten","models":2,"ranked":0,"top":null,"latest":"2026-10-21"},{"slug":"bespokelabs","name":"bespokelabs","models":2,"ranked":0,"top":null,"latest":null},{"slug":"blockbrain","name":"blockbrain","models":2,"ranked":0,"top":null,"latest":null},{"slug":"cactus-compute","name":"Cactus Compute","models":2,"ranked":0,"top":null,"latest":null},{"slug":"contrastive-lm","name":"Contrastive-LM","models":2,"ranked":0,"top":null,"latest":null},{"slug":"databricks","name":"Databricks","models":2,"ranked":0,"top":null,"latest":null},{"slug":"davidburhans","name":"davidburhans","models":2,"ranked":0,"top":null,"latest":null},{"slug":"dvl-lab","name":"DVL Lab","models":2,"ranked":0,"top":null,"latest":"2024-12-12"},{"slug":"eljiwo","name":"eljiwo","models":2,"ranked":0,"top":null,"latest":null},{"slug":"fermion-research","name":"Fermion Research","models":2,"ranked":0,"top":null,"latest":"2026-09-01"},{"slug":"gpt-omni","name":"gpt-omni","models":2,"ranked":0,"top":null,"latest":"2024-10-15"},{"slug":"hemant-heman10x","name":"Hemant (heman10x)","models":2,"ranked":0,"top":null,"latest":null},{"slug":"hopitai","name":"HopitAI","models":2,"ranked":0,"top":null,"latest":null},{"slug":"inference-net","name":"Inference.Net","models":2,"ranked":0,"top":null,"latest":"2026-09-12T01:35:49.000Z"},{"slug":"jaredpalmer","name":"jaredpalmer","models":2,"ranked":0,"top":null,"latest":null},{"slug":"juspay","name":"Juspay","models":2,"ranked":0,"top":null,"latest":null},{"slug":"kshetrajna12","name":"kshetrajna12","models":2,"ranked":0,"top":null,"latest":null},{"slug":"lighton","name":"LightOn","models":2,"ranked":0,"top":null,"latest":"2026-02-08"},{"slug":"maisa","name":"Maisa","models":2,"ranked":0,"top":null,"latest":null},{"slug":"mohit67890","name":"mohit67890","models":2,"ranked":0,"top":null,"latest":null},{"slug":"morph","name":"Morph","models":2,"ranked":0,"top":null,"latest":"2025-07-07T17:54:18.000Z"},{"slug":"naver-cloud","name":"Naver Cloud","models":2,"ranked":0,"top":null,"latest":"2025-12-28"},{"slug":"newfull5-dhtocks","name":"newfull5 (dhtocks)","models":2,"ranked":0,"top":null,"latest":null},{"slug":"perceptron","name":"Perceptron","models":2,"ranked":0,"top":null,"latest":"2026-09-25T16:11:01.000Z"},{"slug":"reka-ai","name":"Reka AI","models":2,"ranked":0,"top":null,"latest":"2026-03-20T17:16:05.000Z"},{"slug":"relace","name":"Relace","models":2,"ranked":0,"top":null,"latest":"2025-12-08T17:06:00.000Z"},{"slug":"salmonn-authors","name":"SALMONN authors","models":2,"ranked":0,"top":null,"latest":null},{"slug":"sthanika-ai","name":"sthanika-ai","models":2,"ranked":0,"top":null,"latest":null},{"slug":"surogate","name":"Surogate","models":2,"ranked":0,"top":null,"latest":null},{"slug":"tii","name":"TII","models":2,"ranked":0,"top":null,"latest":null},{"slug":"togethercomputer","name":"togethercomputer","models":2,"ranked":0,"top":null,"latest":null},{"slug":"typesafe-ai","name":"TypeSafe AI","models":2,"ranked":0,"top":null,"latest":"2026-10-14"},{"slug":"vita-mllm","name":"VITA-MLLM","models":2,"ranked":0,"top":null,"latest":"2025-01-03"},{"slug":"zefan-cai-zefan-cai","name":"Zefan Cai (@Zefan_Cai)","models":2,"ranked":0,"top":null,"latest":null},{"slug":"zoowork","name":"ZooWork","models":2,"ranked":0,"top":null,"latest":null},{"slug":"abhishek-rai-abhishek085","name":"Abhishek Rai (abhishek085)","models":1,"ranked":0,"top":null,"latest":null},{"slug":"academic","name":"Academic","models":1,"ranked":0,"top":null,"latest":null},{"slug":"aditya-ishesatoshi","name":"Aditya (isHeSatoshi)","models":1,"ranked":0,"top":null,"latest":null},{"slug":"ai-singapore","name":"AI Singapore","models":1,"ranked":0,"top":null,"latest":"2024-12-13"},{"slug":"akhilaaa3","name":"akhilaaa3","models":1,"ranked":0,"top":null,"latest":null},{"slug":"altslate-labs","name":"AltSlate Labs","models":1,"ranked":0,"top":null,"latest":null},{"slug":"amirbraham","name":"AmirBraham","models":1,"ranked":0,"top":null,"latest":null},{"slug":"amith-chandrappa-amithgc","name":"Amith Chandrappa (amithgc)","models":1,"ranked":0,"top":null,"latest":null},{"slug":"andyzhang232","name":"andyzhang232","models":1,"ranked":0,"top":null,"latest":null},{"slug":"anthracite-org","name":"Anthracite Org","models":1,"ranked":0,"top":null,"latest":"2024-10-22T00:00:00.000Z"},{"slug":"autoloops","name":"Autoloops","models":1,"ranked":0,"top":null,"latest":null},{"slug":"bespoke-labs","name":"Bespoke Labs","models":1,"ranked":0,"top":null,"latest":null},{"slug":"block","name":"Block","models":1,"ranked":0,"top":null,"latest":null},{"slug":"blsp-authors","name":"BLSP authors","models":1,"ranked":0,"top":null,"latest":null},{"slug":"bluusun","name":"Bluusun","models":1,"ranked":0,"top":null,"latest":null},{"slug":"caiovicentino1","name":"caiovicentino1","models":1,"ranked":0,"top":null,"latest":null},{"slug":"chinhnc","name":"chinhnc","models":1,"ranked":0,"top":null,"latest":null},{"slug":"classifier-dev","name":"classifier.dev","models":1,"ranked":0,"top":null,"latest":null},{"slug":"clause-logic","name":"Clause Logic","models":1,"ranked":0,"top":null,"latest":null},{"slug":"community","name":"Community","models":1,"ranked":0,"top":null,"latest":null},{"slug":"cortex-agent-llc","name":"cortex-agent-llc","models":1,"ranked":0,"top":null,"latest":null},{"slug":"crh225","name":"crh225","models":1,"ranked":0,"top":null,"latest":null},{"slug":"deepan-wadhwa","name":"Deepan Wadhwa","models":1,"ranked":0,"top":null,"latest":null},{"slug":"diva-authors","name":"DiVA authors","models":1,"ranked":0,"top":null,"latest":"2024-10-03"},{"slug":"einptein","name":"einptein","models":1,"ranked":0,"top":null,"latest":null},{"slug":"ekzhang","name":"ekzhang","models":1,"ranked":0,"top":null,"latest":null},{"slug":"eldan-ring","name":"Eldan Ring","models":1,"ranked":0,"top":null,"latest":null},{"slug":"eldanring","name":"EldanRing","models":1,"ranked":0,"top":null,"latest":null},{"slug":"elorian","name":"Elorian","models":1,"ranked":0,"top":null,"latest":"2026-10-17T03:10:29.000Z"},{"slug":"eval-engine-chromia","name":"Eval Engine / Chromia","models":1,"ranked":0,"top":null,"latest":null},{"slug":"everettjf","name":"everettjf","models":1,"ranked":0,"top":null,"latest":null},{"slug":"fireworks","name":"Fireworks","models":1,"ranked":0,"top":null,"latest":"2026-09-23"},{"slug":"fish-audio","name":"Fish Audio","models":1,"ranked":0,"top":null,"latest":null},{"slug":"franck-verrot-franckverrot","name":"Franck Verrot (franckverrot)","models":1,"ranked":0,"top":null,"latest":null},{"slug":"frontier-infra","name":"frontier-infra","models":1,"ranked":0,"top":null,"latest":null},{"slug":"gryphe","name":"Gryphe","models":1,"ranked":0,"top":null,"latest":"2023-07-02T00:00:00.000Z"},{"slug":"hanno-labs","name":"Hanno-Labs","models":1,"ranked":0,"top":null,"latest":null},{"slug":"hark","name":"Hark","models":1,"ranked":0,"top":null,"latest":"2026-08-05"},{"slug":"harshatheg","name":"harshatheg","models":1,"ranked":0,"top":null,"latest":null},{"slug":"hexgrad","name":"hexgrad","models":1,"ranked":0,"top":null,"latest":null},{"slug":"ictnlp","name":"ICTNLP","models":1,"ranked":0,"top":null,"latest":"2024-09-10"},{"slug":"ikermoel","name":"IkerMoel","models":1,"ranked":0,"top":null,"latest":null},{"slug":"infinigence-ai","name":"Infinigence AI","models":1,"ranked":0,"top":null,"latest":null},{"slug":"interfaze","name":"Interfaze","models":1,"ranked":0,"top":null,"latest":"2026-05-11"},{"slug":"interfaze-ai","name":"interfaze-ai","models":1,"ranked":0,"top":null,"latest":null},{"slug":"jan","name":"Jan","models":1,"ranked":0,"top":null,"latest":"2024-10-20"},{"slug":"joshuasp","name":"JoshuaSP","models":1,"ranked":0,"top":null,"latest":null},{"slug":"jsaurabh","name":"jsaurabh","models":1,"ranked":0,"top":null,"latest":null},{"slug":"kotoba-labs","name":"Kotoba Labs","models":1,"ranked":0,"top":null,"latest":null},{"slug":"kpiya","name":"kpiya","models":1,"ranked":0,"top":null,"latest":null},{"slug":"kt","name":"KT","models":1,"ranked":0,"top":null,"latest":"2026-11-05T16:55:01.000Z"},{"slug":"kwaipilot","name":"Kwaipilot","models":1,"ranked":0,"top":null,"latest":"2026-07-10T20:16:29.000Z"},{"slug":"libertai","name":"LibertAI","models":1,"ranked":0,"top":null,"latest":null},{"slug":"logan-markewich","name":"Logan Markewich","models":1,"ranked":0,"top":null,"latest":null},{"slug":"lukitaduarte","name":"Lukitaduarte","models":1,"ranked":0,"top":null,"latest":null},{"slug":"mair-hub","name":"Mair Hub","models":1,"ranked":0,"top":null,"latest":null},{"slug":"manavarya09-manav","name":"Manavarya09 (Manav)","models":1,"ranked":0,"top":null,"latest":null},{"slug":"mancer","name":"Mancer","models":1,"ranked":0,"top":null,"latest":"2023-08-02T00:00:00.000Z"},{"slug":"mantisshrimpdev","name":"MantisShrimpdev","models":1,"ranked":0,"top":null,"latest":null},{"slug":"michaljach","name":"michaljach","models":1,"ranked":0,"top":null,"latest":null},{"slug":"mikhail-gribov","name":"Mikhail Gribov","models":1,"ranked":0,"top":null,"latest":null},{"slug":"milliseconds-ai","name":"milliseconds.ai","models":1,"ranked":0,"top":null,"latest":null},{"slug":"mithalouni","name":"mithalouni","models":1,"ranked":0,"top":null,"latest":null},{"slug":"mixedbread","name":"Mixedbread","models":1,"ranked":0,"top":null,"latest":"2026-08-13"},{"slug":"moganai","name":"moganai","models":1,"ranked":0,"top":null,"latest":null},{"slug":"monotykamary","name":"monotykamary","models":1,"ranked":0,"top":null,"latest":null},{"slug":"motif-technologies","name":"Motif Technologies","models":1,"ranked":0,"top":null,"latest":"2026-11-02T21:35:34.000Z"},{"slug":"mourad-ghafiri","name":"Mourad Ghafiri","models":1,"ranked":0,"top":null,"latest":null},{"slug":"multiverse-computing","name":"Multiverse Computing","models":1,"ranked":0,"top":null,"latest":"2026-09-02"},{"slug":"murf-ai","name":"Murf AI","models":1,"ranked":0,"top":null,"latest":"2026-03-05"},{"slug":"nc-ai","name":"NC AI","models":1,"ranked":0,"top":null,"latest":null},{"slug":"next-gpt-authors","name":"NExT-GPT authors","models":1,"ranked":0,"top":null,"latest":null},{"slug":"notnotsamuel","name":"notnotsamuel","models":1,"ranked":0,"top":null,"latest":null},{"slug":"octalab","name":"Octalab","models":1,"ranked":0,"top":null,"latest":null},{"slug":"ola-omni","name":"Ola-Omni","models":1,"ranked":0,"top":null,"latest":"2025-02-06"},{"slug":"omnijev","name":"OmniJev","models":1,"ranked":0,"top":null,"latest":null},{"slug":"oraculumai","name":"oraculumai","models":1,"ranked":0,"top":null,"latest":null},{"slug":"orcarouter","name":"OrcaRouter","models":1,"ranked":0,"top":null,"latest":"2026-10-16"},{"slug":"pandagpt-authors","name":"PandaGPT authors","models":1,"ranked":0,"top":null,"latest":null},{"slug":"paper-instruments","name":"Paper Instruments","models":1,"ranked":0,"top":null,"latest":"2026-10-14"},{"slug":"particle-ai","name":"Particle AI","models":1,"ranked":0,"top":null,"latest":"2026-10-28"},{"slug":"phonic","name":"Phonic","models":1,"ranked":0,"top":null,"latest":"2026-10-29"},{"slug":"pixilabai","name":"PixilabAI","models":1,"ranked":0,"top":null,"latest":null},{"slug":"pokee-ai","name":"Pokee AI","models":1,"ranked":0,"top":null,"latest":"2026-08-03"},{"slug":"prismml","name":"PrismML","models":1,"ranked":0,"top":null,"latest":"2026-10-16T06:09:55.000Z"},{"slug":"pst2154","name":"pst2154","models":1,"ranked":0,"top":null,"latest":null},{"slug":"razorback16","name":"razorback16","models":1,"ranked":0,"top":null,"latest":null},{"slug":"razorback16-codiv","name":"razorback16 / Codiv","models":1,"ranked":0,"top":null,"latest":null},{"slug":"recursive","name":"Recursive","models":1,"ranked":0,"top":null,"latest":"2026-10-21"},{"slug":"reza-sayar","name":"Reza Sayar","models":1,"ranked":0,"top":null,"latest":null},{"slug":"sabeel111","name":"sabeel111","models":1,"ranked":0,"top":null,"latest":null},{"slug":"sean-goedecke","name":"Sean Goedecke","models":1,"ranked":0,"top":null,"latest":null},{"slug":"shanghai-artificial-intelligence-laboratory","name":"Shanghai Artificial Intelligence Laboratory","models":1,"ranked":0,"top":null,"latest":"2026-09-14"},{"slug":"shgao","name":"shgao","models":1,"ranked":0,"top":null,"latest":null},{"slug":"sky7350","name":"sky7350","models":1,"ranked":0,"top":null,"latest":null},{"slug":"smallest-ai","name":"Smallest AI","models":1,"ranked":0,"top":null,"latest":null},{"slug":"soofi-project","name":"Soofi Project","models":1,"ranked":0,"top":null,"latest":"2026-07-10"},{"slug":"standard-thinking","name":"Standard Thinking","models":1,"ranked":0,"top":null,"latest":null},{"slug":"stealth","name":"Stealth","models":1,"ranked":0,"top":null,"latest":"2026-11-03T01:38:50.000Z"},{"slug":"subquadratic","name":"Subquadratic","models":1,"ranked":0,"top":null,"latest":"2026-10-22"},{"slug":"supersoniclabs","name":"SupersonicLabs","models":1,"ranked":0,"top":null,"latest":null},{"slug":"theodore-lee-theoleecj","name":"Theodore Lee (TheoLeeCJ)","models":1,"ranked":0,"top":null,"latest":null},{"slug":"tokenrhythm","name":"TokenRhythm","models":1,"ranked":0,"top":null,"latest":null},{"slug":"typesafe","name":"TypeSafe","models":1,"ranked":0,"top":null,"latest":"2026-09-25T19:12:40.000Z"},{"slug":"undi95","name":"Undi95","models":1,"ranked":0,"top":null,"latest":"2023-07-22T00:00:00.000Z"},{"slug":"us-github","name":"us (GitHub)","models":1,"ranked":0,"top":null,"latest":null},{"slug":"useful-sensors","name":"Useful Sensors","models":1,"ranked":0,"top":null,"latest":null},{"slug":"venice","name":"Venice","models":1,"ranked":0,"top":null,"latest":"2025-07-09T21:02:46.000Z"},{"slug":"wayfind-metask-ai","name":"Wayfind (metask-ai)","models":1,"ranked":0,"top":null,"latest":null},{"slug":"wfzyx-victor-hugo","name":"wfzyx (Victor Hugo)","models":1,"ranked":0,"top":null,"latest":null},{"slug":"wilflin","name":"WilfLin","models":1,"ranked":0,"top":null,"latest":null},{"slug":"wispr","name":"Wispr","models":1,"ranked":0,"top":null,"latest":"2026-09-17"},{"slug":"writer","name":"Writer","models":1,"ranked":0,"top":null,"latest":"2026-01-21T13:57:03.000Z"},{"slug":"x-lance","name":"X-LANCE","models":1,"ranked":0,"top":null,"latest":"2024-12-20"},{"slug":"z","name":"Z","models":1,"ranked":0,"top":null,"latest":null},{"slug":"zeroentropy","name":"ZeroEntropy","models":1,"ranked":0,"top":null,"latest":null},{"slug":"zhengxu-yu","name":"Zhengxu Yu","models":1,"ranked":0,"top":null,"latest":null}],"directSources":[{"source":"vals-ai","publisher":"Vals AI","url":"https://www.vals.ai","license":"No licence stated. Read from the public leaderboard and credited to Vals AI","benchmarks":47,"scoredBenchmarks":20,"measurements":3323,"models":275,"modelFacts":0,"arenaRatings":0,"lastRunAt":"2026-11-12T10:05:26.427Z"},{"source":"euroeval","publisher":"EuroEval","url":"https://euroeval.com/leaderboards","license":"MIT — the leaderboard site and its CSV routes are in the licensed repository","benchmarks":8,"scoredBenchmarks":8,"measurements":1480,"models":256,"modelFacts":0,"arenaRatings":0,"lastRunAt":"2026-11-12T14:19:45.202Z"},{"source":"epoch-ai","publisher":"Epoch AI","url":"https://epoch.ai","license":"CC BY — free to use and redistribute with attribution","benchmarks":22,"scoredBenchmarks":17,"measurements":1359,"models":236,"modelFacts":0,"arenaRatings":0,"lastRunAt":"2026-11-12T10:18:41.021Z"},{"source":"livebench","publisher":"LiveBench","url":"https://livebench.ai","license":"No licence stated for the leaderboard. The site repo serving these CSVs has no LICENSE; the harness repo carries upstream Apache-2.0 and MIT copies that cover code, not results","benchmarks":12,"scoredBenchmarks":12,"measurements":772,"models":83,"modelFacts":0,"arenaRatings":0,"lastRunAt":"2026-11-12T14:19:42.376Z"},{"source":"arc-prize","publisher":"ARC Prize Foundation","url":"https://arcprize.org","license":"No licence stated. Their terms ask for written permission before commercial use","benchmarks":5,"scoredBenchmarks":3,"measurements":526,"models":135,"modelFacts":0,"arenaRatings":0,"lastRunAt":"2026-11-12T10:18:36.443Z"},{"source":"lmarena","publisher":"LMArena","url":"https://lmarena.ai","license":"CC BY 4.0 (lmarena-ai/leaderboard-dataset on Hugging Face)","benchmarks":3,"scoredBenchmarks":3,"measurements":164,"models":65,"modelFacts":0,"arenaRatings":824,"lastRunAt":"2026-11-12T14:13:36.654Z"},{"source":"swebench-experiments","publisher":"SWE-bench team","url":"https://github.com/SWE-bench/experiments","license":"No licence stated. The repository publishes submission records for reproducibility and transparency and asks that SWE-bench be cited","benchmarks":5,"scoredBenchmarks":5,"measurements":111,"models":73,"modelFacts":0,"arenaRatings":0,"lastRunAt":"2026-11-12T14:35:58.310Z"},{"source":"tau2-bench","publisher":"Sierra Research","url":"https://github.com/sierra-research/tau2-bench","license":"MIT — results are in the licensed repository","benchmarks":4,"scoredBenchmarks":4,"measurements":91,"models":44,"modelFacts":0,"arenaRatings":0,"lastRunAt":"2026-11-12T14:19:24.081Z"},{"source":"swe-bench","publisher":"SWE-bench team","url":"https://www.swebench.com","license":"No licence stated for the leaderboard. The harness repo is MIT","benchmarks":4,"scoredBenchmarks":4,"measurements":82,"models":60,"modelFacts":0,"arenaRatings":0,"lastRunAt":"2026-11-12T10:17:41.194Z"},{"source":"terminal-bench","publisher":"Terminal-Bench","url":"https://www.tbench.ai","license":"No licence stated for the leaderboard. The harness repo is Apache-2.0","benchmarks":2,"scoredBenchmarks":2,"measurements":46,"models":23,"modelFacts":0,"arenaRatings":0,"lastRunAt":"2026-11-12T10:17:34.915Z"},{"source":"openrouter","publisher":"OpenRouter","url":"https://openrouter.ai","license":"No licence stated. Their terms restrict automated copying","benchmarks":0,"scoredBenchmarks":0,"measurements":0,"models":545,"modelFacts":545,"arenaRatings":0,"lastRunAt":"2026-11-12T14:03:09.050Z"}],"dataUpdatedAt":"2026-11-12T14:09:31.783Z","arenaUpdatedAt":"2026-11-12T14:39:40.309Z","fetchedAt":"2026-11-12T14:39:40.309Z","methodologyVersion":"inference360-v5.0","methodology":{"version":"inference360-v5.0","weights":{"agentic":0.2,"coding":0.2,"reasoning":0.15,"knowledge":0.15,"math":0.1,"multimodal":0.1,"instruction":0.05,"multilingual":0.05},"minResults":8,"minCategories":4,"minCoverage":0.5,"minFamilyModels":10,"scoredBenchmarks":157,"benchmarkFamilies":19,"identityConflicts":1,"licensedOnly":false,"benchlm":{"license":"CC BY-NC 4.0","licenseUrl":"https://creativecommons.org/licenses/by-nc/4.0/","attribution":"Data from BenchLM.ai"},"prior":49.715071874107096,"excludedBenchmarks":119,"excludedMeasurements":2090,"exclusionReasons":[{"reason":"Benchmarks where a higher number is worse, such as a hallucination rate","benchmarks":3,"measurements":177,"kind":"dropped"},{"reason":"Ratings such as Elo. We cannot average these with scores","benchmarks":12,"measurements":429,"kind":"dropped"},{"reason":"Indexes made from other benchmarks","benchmarks":3,"measurements":345,"kind":"dropped"},{"reason":"Benchmarks outside the eight categories","benchmarks":101,"measurements":1139,"kind":"dropped"},{"reason":"Results we now read from the publisher","benchmarks":27,"measurements":966,"kind":"superseded"},{"reason":"Results from BenchLM (CC BY-NC 4.0)","benchmarks":0,"measurements":0,"kind":"dropped"}]},"stale":false,"sourceStatus":[{"id":"benchlm-models","name":"BenchLM catalog and benchmark rows","ok":true,"updatedAt":"2026-11-12T14:09:31.783Z","message":"851 models loaded"},{"id":"benchlm-benchmarks","name":"BenchLM benchmark definitions","ok":true,"updatedAt":"2026-11-12T14:09:31.783Z","message":"894 benchmark definitions loaded"},{"id":"benchlm-pricing","name":"BenchLM pricing","ok":true,"updatedAt":"2026-11-12T14:09:31.783Z","message":"932 pricing records loaded"},{"id":"benchlm-speed","name":"BenchLM runtime throughput","ok":true,"updatedAt":"2026-11-12T14:09:31.783Z","message":"165 records, used where OpenRouter has none"},{"id":"direct:vals-ai","name":"Vals AI (collected directly)","ok":true,"updatedAt":"2026-11-12T10:05:26.427Z","message":"3323 measurements across 275 models"},{"id":"direct:euroeval","name":"EuroEval (collected directly)","ok":true,"updatedAt":"2026-11-12T14:19:45.202Z","message":"1480 measurements across 256 models"},{"id":"direct:epoch-ai","name":"Epoch AI (collected directly)","ok":true,"updatedAt":"2026-11-12T10:18:41.021Z","message":"1359 measurements across 236 models"},{"id":"direct:livebench","name":"LiveBench (collected directly)","ok":true,"updatedAt":"2026-11-12T14:19:42.376Z","message":"772 measurements across 83 models"},{"id":"direct:arc-prize","name":"ARC Prize Foundation (collected directly)","ok":true,"updatedAt":"2026-11-12T10:18:36.443Z","message":"526 measurements across 135 models"},{"id":"direct:lmarena","name":"LMArena (collected directly)","ok":true,"updatedAt":"2026-11-12T14:13:36.654Z","message":"164 measurements across 65 models, 824 arena ratings"},{"id":"direct:swebench-experiments","name":"SWE-bench team (collected directly)","ok":true,"updatedAt":"2026-11-12T14:35:58.310Z","message":"111 measurements across 73 models"},{"id":"direct:tau2-bench","name":"Sierra Research (collected directly)","ok":true,"updatedAt":"2026-11-12T14:19:24.081Z","message":"91 measurements across 44 models"},{"id":"direct:swe-bench","name":"SWE-bench team (collected directly)","ok":true,"updatedAt":"2026-11-12T10:17:41.194Z","message":"82 measurements across 60 models"},{"id":"direct:terminal-bench","name":"Terminal-Bench (collected directly)","ok":true,"updatedAt":"2026-11-12T10:17:34.915Z","message":"46 measurements across 23 models"},{"id":"direct:openrouter","name":"OpenRouter (collected directly)","ok":true,"updatedAt":"2026-11-12T14:03:09.050Z","message":"545 model records"},{"id":"models-dev","name":"models.dev provider prices","ok":true,"updatedAt":"2026-11-12T14:39:40.309Z","message":"247 models with a provider price"}],"coverage":{"total":1424,"ranked":283,"priced":457,"runtime":411,"arena":215,"arenaTotal":824,"hosts":247}}