[{"id":"ours-pi1-codemode-luna-atlas","benchmark":"atlas","harness":"Pi 1.0 · Codemode","model":"GPT-6 Luna","effort":"xhigh","score":0.42818,"score_published":"42.8%","ci95":[0.37869,0.47916],"cost_per_attempt":0.04116,"cost_basis":"Harbor-recorded agent cost, mean over 369 of 369 attempts ($15.19 total)","tasks":124,"attempts":369,"k":3,"date":"2026-10-02","run":"Unmodified Pi 1.0.0 (@earendil-works/pi-coding-agent) with Codemode in its default \"on\" mode: the model sees read, bash, edit, write and codemode. Run by Reason: 124 tasks x 3 attempts on e2b sandboxes at 8 vCPU / 8 GiB, Harbor 0.22. 158 of 369 graded attempts passed. Graded by the task's Claude Opus 4.5 judge.","source_kind":"ours","featured":true,"sources":[{"label":"Harbor job atlas-hub-k3-pi1-codemode-luna-xhigh","url":null}],"run_ref":"atlas-hub-k3-pi1-codemode-luna-xhigh","cost_estimated":false},{"id":"ours-pi1-codemode-only-luna-atlas","benchmark":"atlas","harness":"Pi 1.0 · Codemode only","model":"GPT-6 Luna","effort":"xhigh","score":0.37634,"score_published":"37.6%","ci95":[0.32861,0.4266],"cost_per_attempt":0.04138,"cost_basis":"Harbor-recorded agent cost, mean over 372 of 372 attempts ($15.39 total)","tasks":124,"attempts":372,"k":3,"date":"2026-10-02","run":"Unmodified Pi 1.0.0 (@earendil-works/pi-coding-agent) with Codemode always on: every tool call goes through a codemode script. Run by Reason: 124 tasks x 3 attempts on e2b sandboxes at 8 vCPU / 8 GiB, Harbor 0.22. 140 of 372 graded attempts passed. Graded by the task's Claude Opus 4.5 judge.","source_kind":"ours","featured":false,"sources":[{"label":"Harbor job atlas-hub-k3-pi1-codemode-only-luna-xhigh","url":null}],"cost_estimated":false,"run_ref":"atlas-hub-k3-pi1-codemode-only-luna-xhigh"},{"id":"ours-pi1-codemode-luna-deepswe","benchmark":"deepswe","harness":"Pi 1.0 · Codemode","model":"GPT-6 Luna","effort":"xhigh","score":0.58702,"score_published":"58.7%","ci95":[0.53392,0.63817],"cost_per_attempt":0.11378,"cost_basis":"Harbor-recorded agent cost, mean over 339 of 339 attempts ($38.57 total)","tasks":113,"attempts":339,"k":3,"date":"2026-10-03","run":"Unmodified Pi 1.0.0 (@earendil-works/pi-coding-agent) with Codemode in its default \"on\" mode: the model sees read, bash, edit, write and codemode. Run by Reason: 113 tasks x 3 attempts on e2b sandboxes at 8 vCPU / 8 GiB, Harbor 0.22. 199 of 339 graded attempts passed.","source_kind":"ours","featured":true,"sources":[{"label":"Harbor job deepswe-hub-k3-pi1-codemode-luna-xhigh","url":null}],"run_ref":"deepswe-hub-k3-pi1-codemode-luna-xhigh","cost_estimated":false},{"id":"ours-pi1-codemode-luna-tb4","benchmark":"tb4","harness":"Pi 1.0 · Codemode","model":"GPT-6 Luna","effort":"xhigh","score":0.08929,"score_published":"8.9%","ci95":[0.05854,0.13388],"cost_per_attempt":0.10837,"cost_basis":"Harbor-recorded agent cost, mean over 224 of 224 attempts ($24.28 total)","tasks":45,"attempts":224,"k":5,"date":"2026-10-03","run":"Unmodified Pi 1.0.0 (@earendil-works/pi-coding-agent) with Codemode in its default \"on\" mode: the model sees read, bash, edit, write and codemode. Run by Reason: 45 tasks x 5 attempts on e2b sandboxes at 8 vCPU / 8 GiB, Harbor 0.22. 20 of 224 graded attempts passed. Terminal-Bench on the 45 of 66 tasks that run on e2b (the rest need multiple containers, a GPU or more than 8 CPU / 8 GiB), so it is not directly comparable with 66-task scores.","source_kind":"ours","featured":true,"sources":[{"label":"Harbor job tb4-hub-k5-pi1-codemode-luna-xhigh","url":null}],"run_ref":"tb4-hub-k5-pi1-codemode-luna-xhigh","cost_estimated":false},{"id":"ours-pi1-codemode-luna-index","benchmark":"index","harness":"Pi 1.0 · Codemode","model":"GPT-6 Luna","effort":"xhigh","score":0.36816,"score_published":"36.8","ci95":null,"cost_per_attempt":0.08373,"cost_basis":"Pooled agent cost over 932 attempts on the three benchmarks ($78.04 total), the same per-attempt basis as Artificial Analysis","tasks":null,"attempts":932,"k":null,"date":"2026-10-03","run":"Unmodified Pi 1.0.0 (@earendil-works/pi-coding-agent) with Codemode in its default \"on\" mode: the model sees read, bash, edit, write and codemode. Equal-weight mean of SWE-Atlas QnA 42.8%, DeepSWE 58.7% and Terminal-Bench 8.9%, the Artificial Analysis index formula. Terminal-Bench on the 45 of 66 tasks that run on e2b (the rest need multiple containers, a GPU or more than 8 CPU / 8 GiB), so it is not directly comparable with 66-task scores.","source_kind":"ours","featured":true,"sources":[{"label":"Harbor job deepswe-hub-k3-pi1-codemode-luna-xhigh","url":null},{"label":"Harbor job tb4-hub-k5-pi1-codemode-luna-xhigh","url":null},{"label":"Harbor job atlas-hub-k3-pi1-codemode-luna-xhigh","url":null}],"run_ref":"deepswe-hub-k3-pi1-codemode-luna-xhigh, tb4-hub-k5-pi1-codemode-luna-xhigh, atlas-hub-k3-pi1-codemode-luna-xhigh","cost_estimated":false},{"id":"ours-cc-opus55-atlas","benchmark":"atlas","harness":"Claude Code","model":"Claude Opus 5.5","effort":"high","score":0.5914,"score_published":"59.1%","ci95":[0.54076,0.64017],"cost_per_attempt":0.91866,"cost_basis":"Harbor-recorded agent cost, mean over 372 of 372 attempts ($341.74 total)","tasks":124,"attempts":372,"k":3,"date":"2026-09-29","run":"Claude Code run by Reason through the same Harbor adapter and sandboxes; 65 retried executions have no recorded cost. Run by Reason: 124 tasks x 3 attempts on e2b sandboxes at 8 vCPU / 8 GiB, Harbor 0.22. 220 of 372 graded attempts passed. Graded by the task's Claude Opus 4.5 judge.","source_kind":"ours","sources":[{"label":"Harbor job atlas-full-k3-claude-code-opus-5p5","url":null}],"run_ref":"atlas-full-k3-claude-code-opus-5p5","cost_estimated":false},{"id":"ours-pi-stock-luna-max-atlas","benchmark":"atlas","harness":"Pi","harness_version":"0.87.1","model":"GPT-6 Luna","effort":"max","score":0.48485,"score_published":"48.5%","ci95":[0.42857,0.54152],"cost_per_attempt":0.08433,"cost_basis":"Harbor-recorded agent cost, mean over 297 of 297 attempts ($25.05 total)","cost_estimated":false,"tasks":99,"attempts":297,"k":3,"date":"2026-09-28","run":"Unmodified Pi 0.87.1 (stock tools and prompt). Run by Reason as the control for sync-yield on 99 of the 124 tasks x 3 attempts on e2b sandboxes at 8 vCPU / 8 GiB, Harbor 0.22, at max effort. 144 of 297 graded attempts passed. Graded by the task's Claude Opus 4.5 judge.","source_kind":"ours","featured":false,"sources":[{"label":"Harbor job atlas-full-k3-stock","url":null},{"label":"Harbor job atlas-full-k3-stock-rerun-k1","url":null}],"also_reported":[],"retrieved_at":"2026-10-04"},{"id":"ours-pi-stock-luna-max-deepswe","benchmark":"deepswe","harness":"Pi","harness_version":"0.87.1","model":"GPT-6 Luna","effort":"max","score":0.66066,"score_published":"66.1%","ci95":[0.60823,0.70943],"cost_per_attempt":0.21528,"cost_basis":"Harbor-recorded agent cost, mean over 333 of 333 attempts ($71.69 total)","cost_estimated":false,"tasks":111,"attempts":333,"k":3,"date":"2026-09-28","run":"Unmodified Pi 0.87.1 (stock tools and prompt). Run by Reason as the control for sync-yield on 111 of the 113 tasks x 3 attempts on e2b sandboxes at 8 vCPU / 8 GiB, Harbor 0.22, at max effort. 220 of 333 graded attempts passed.","source_kind":"ours","featured":false,"sources":[{"label":"Harbor job deepswe-k3-stock","url":null},{"label":"Harbor job deepswe-k3-stock-rerun-k1","url":null}],"also_reported":[],"retrieved_at":"2026-10-04"},{"id":"ours-pi-stock-luna-max-tb4","benchmark":"tb4","harness":"Pi","harness_version":"0.87.1","model":"GPT-6 Luna","effort":"max","score":0.13636,"score_published":"13.6%","ci95":[0.08803,0.20527],"cost_per_attempt":0.16127,"cost_basis":"Harbor-recorded agent cost, mean over 132 of 132 attempts ($21.29 total)","cost_estimated":false,"tasks":44,"attempts":132,"k":3,"date":"2026-09-28","run":"Unmodified Pi 0.87.1 (stock tools and prompt). Run by Reason as the control for sync-yield on 44 of the 66 tasks x 3 attempts on e2b sandboxes at 8 vCPU / 8 GiB, Harbor 0.22, at max effort. 18 of 132 graded attempts passed.","source_kind":"ours","featured":false,"sources":[{"label":"Harbor job tb4-k3-stock","url":null}],"also_reported":[],"retrieved_at":"2026-10-04"},{"id":"ours-pi-stock-luna-max-index","benchmark":"index","harness":"Pi","harness_version":"0.87.1","model":"GPT-6 Luna","effort":"max","score":0.42729,"score_published":"42.7","ci95":null,"cost_per_attempt":0.15488,"cost_basis":"Pooled agent cost over 762 attempts on the three benchmarks ($118.02 total), the same per-attempt basis as Artificial Analysis","cost_estimated":false,"tasks":null,"attempts":762,"k":null,"date":"2026-09-28","run":"Unmodified Pi 0.87.1 (stock tools and prompt). Equal-weight mean of SWE-Atlas QnA 48.5% (99 tasks), DeepSWE 66.1% (111 tasks) and Terminal-Bench 13.6% (44 tasks); task subsets, so not directly comparable with full-suite scores.","source_kind":"ours","featured":false,"sources":[{"label":"Harbor jobs deepswe-k3-stock, tb4-k3-stock, atlas-full-k3-stock","url":null}],"also_reported":[],"retrieved_at":"2026-10-04"},{"id":"aa-a2c87c06","benchmark":"index","harness":"Claude Code","model":"Claude Sonnet 5.5","effort":"max","score":0.68358,"score_published":"68.4","ci95":null,"cost_per_attempt":14.1908,"cost_basis":"AA mean.costUsd $14.19 per attempt (its sums.costUsd $4,299.81 is that mean times one attempt per task, not the total over 909 attempts)","tasks":303,"attempts":909,"k":3,"date":null,"run":"Artificial Analysis Coding Agent Index row 'Claude Code - Sonnet 5.5 (max)': equal-weight mean of DeepSWE v1.1, Terminal-Bench 4.0, SWE-Atlas QnA, 3 attempts per task; AA lists $14.19 per attempt. No run date published.","source_kind":"aa","sources":[{"label":"Artificial Analysis Coding Agents (no per-run page)","url":"https://artificialanalysis.ai/agents/coding-agents"}],"cost_estimated":false},{"id":"aa-ab5074e4","benchmark":"index","harness":"Claude Code","model":"Claude Opus 5.5","effort":"max","score":0.65989,"score_published":"66.0","ci95":null,"cost_per_attempt":13.03638,"cost_basis":"AA mean.costUsd $13.04 per attempt (its sums.costUsd $3,950.02 is that mean times one attempt per task, not the total over 909 attempts)","tasks":303,"attempts":909,"k":3,"date":null,"run":"Artificial Analysis Coding Agent Index row 'Claude Code - Opus 5.5 (max)': equal-weight mean of DeepSWE v1.1, Terminal-Bench 4.0, SWE-Atlas QnA, 3 attempts per task; AA lists $13.04 per attempt. No run date published.","source_kind":"aa","sources":[{"label":"Artificial Analysis Coding Agents (no per-run page)","url":"https://artificialanalysis.ai/agents/coding-agents"}],"cost_estimated":false},{"id":"aa-ef63fafa","benchmark":"index","harness":"Antigravity CLI","model":"Gemini 4 Argon","effort":"high","score":0.63758,"score_published":"63.8","ci95":null,"cost_per_attempt":5.84391,"cost_basis":"AA mean.costUsd $5.84 per attempt (its sums.costUsd $1,764.86 is that mean times one attempt per task, not the total over 905 attempts)","tasks":303,"attempts":905,"k":3,"date":null,"run":"Artificial Analysis Coding Agent Index row 'Antigravity CLI - Gemini 4 Argon (high)': equal-weight mean of DeepSWE v1.1, Terminal-Bench 4.0, SWE-Atlas QnA, 3 attempts per task; AA lists $5.84 per attempt. No run date published. Row marked unavailable on AA.","source_kind":"aa","sources":[{"label":"Artificial Analysis Coding Agents (no per-run page)","url":"https://artificialanalysis.ai/agents/coding-agents"}],"cost_estimated":false},{"id":"aa-ed8c1d0b","benchmark":"index","harness":"Codex","model":"GPT-6.1 Sol","effort":"xhigh","score":0.62908,"score_published":"62.9","ci95":null,"cost_per_attempt":1.03971,"cost_basis":"AA mean.costUsd $1.04 per attempt (its sums.costUsd $313.42 is that mean times one attempt per task, not the total over 909 attempts)","tasks":303,"attempts":909,"k":3,"date":null,"run":"Artificial Analysis Coding Agent Index row 'Codex - GPT-6.1 Sol (xhigh)': equal-weight mean of DeepSWE v1.1, Terminal-Bench 4.0, SWE-Atlas QnA, 3 attempts per task; AA lists $1.04 per attempt. No run date published.","source_kind":"aa","sources":[{"label":"Artificial Analysis Coding Agents (no per-run page)","url":"https://artificialanalysis.ai/agents/coding-agents"}],"cost_estimated":false},{"id":"aa-666605c6","benchmark":"index","harness":"Claude Code","model":"Claude Sonnet 5.5","effort":"xhigh","score":0.62871,"score_published":"62.9","ci95":null,"cost_per_attempt":3.33339,"cost_basis":"AA mean.costUsd $3.33 per attempt (its sums.costUsd $1,010.02 is that mean times one attempt per task, not the total over 909 attempts)","tasks":303,"attempts":909,"k":3,"date":null,"run":"Artificial Analysis Coding Agent Index row 'Claude Code - Sonnet 5.5 (xhigh)': equal-weight mean of DeepSWE v1.1, Terminal-Bench 4.0, SWE-Atlas QnA, 3 attempts per task; AA lists $3.33 per attempt. No run date published.","source_kind":"aa","sources":[{"label":"Artificial Analysis Coding Agents (no per-run page)","url":"https://artificialanalysis.ai/agents/coding-agents"}],"cost_estimated":false},{"id":"aa-29a1cdad","benchmark":"index","harness":"Claude Code","model":"Claude Fable 5.1","effort":"max","score":0.62222,"score_published":"62.2","ci95":null,"cost_per_attempt":12.38809,"cost_basis":"AA mean.costUsd $12.39 per attempt (its sums.costUsd $3,753.59 is that mean times one attempt per task, not the total over 909 attempts)","tasks":303,"attempts":909,"k":3,"date":null,"run":"Artificial Analysis Coding Agent Index row 'Claude Code - Fable 5.1 (max) (with fallback)': equal-weight mean of DeepSWE v1.1, Terminal-Bench 4.0, SWE-Atlas QnA, 3 attempts per task; AA lists $12.39 per attempt. No run date published. Uses model fallback on refusals.","source_kind":"aa","sources":[{"label":"Artificial Analysis Coding Agents (no per-run page)","url":"https://artificialanalysis.ai/agents/coding-agents"}],"cost_estimated":false},{"id":"aa-34970c96","benchmark":"index","harness":"Devin Fusion CLI","model":"Claude Fable 5.1 + SWE-2 Medium","effort":"xhigh","score":0.61683,"score_published":"61.7","ci95":null,"cost_per_attempt":7.89713,"cost_basis":"AA mean.costUsd $7.90 per attempt (its sums.costUsd $2,392.83 is that mean times one attempt per task, not the total over 908 attempts)","tasks":303,"attempts":908,"k":3,"date":null,"run":"Artificial Analysis Coding Agent Index row 'Devin Fusion CLI - Claude Fable 5.1 XHigh + SWE-2 Medium': equal-weight mean of DeepSWE v1.1, Terminal-Bench 4.0, SWE-Atlas QnA, 3 attempts per task; AA lists $7.90 per attempt. No run date published.","source_kind":"aa","sources":[{"label":"Artificial Analysis Coding Agents (no per-run page)","url":"https://artificialanalysis.ai/agents/coding-agents"}],"cost_estimated":false},{"id":"aa-9e810ff3","benchmark":"index","harness":"Codex","model":"GPT-6 Astra","effort":"max","score":0.61645,"score_published":"61.6","ci95":null,"cost_per_attempt":7.4671,"cost_basis":"AA mean.costUsd $7.47 per attempt (its sums.costUsd $2,251.13 is that mean times one attempt per task, not the total over 909 attempts)","tasks":303,"attempts":909,"k":3,"date":null,"run":"Artificial Analysis Coding Agent Index row 'Codex - GPT-6 Astra (max)': equal-weight mean of DeepSWE v1.1, Terminal-Bench 4.0, SWE-Atlas QnA, 3 attempts per task; AA lists $7.47 per attempt. No run date published.","source_kind":"aa","sources":[{"label":"Artificial Analysis Coding Agents (no per-run page)","url":"https://artificialanalysis.ai/agents/coding-agents"}],"cost_estimated":false},{"id":"aa-93acd59f","benchmark":"index","harness":"Codex","model":"GPT-6.1 Sol","effort":"medium","score":0.61415,"score_published":"61.4","ci95":null,"cost_per_attempt":0.70467,"cost_basis":"AA mean.costUsd $0.70 per attempt (its sums.costUsd $212.41 is that mean times one attempt per task, not the total over 909 attempts)","tasks":303,"attempts":909,"k":3,"date":null,"run":"Artificial Analysis Coding Agent Index row 'Codex - GPT-6.1 Sol (medium)': equal-weight mean of DeepSWE v1.1, Terminal-Bench 4.0, SWE-Atlas QnA, 3 attempts per task; AA lists $0.70 per attempt. No run date published.","source_kind":"aa","sources":[{"label":"Artificial Analysis Coding Agents (no per-run page)","url":"https://artificialanalysis.ai/agents/coding-agents"}],"cost_estimated":false},{"id":"aa-4aee8aa2","benchmark":"index","harness":"Codex","model":"GPT-6.1 Sol","effort":"high","score":0.60149,"score_published":"60.1","ci95":null,"cost_per_attempt":0.88924,"cost_basis":"AA mean.costUsd $0.89 per attempt (its sums.costUsd $268.04 is that mean times one attempt per task, not the total over 909 attempts)","tasks":303,"attempts":909,"k":3,"date":null,"run":"Artificial Analysis Coding Agent Index row 'Codex - GPT-6.1 Sol (high)': equal-weight mean of DeepSWE v1.1, Terminal-Bench 4.0, SWE-Atlas QnA, 3 attempts per task; AA lists $0.89 per attempt. No run date published.","source_kind":"aa","sources":[{"label":"Artificial Analysis Coding Agents (no per-run page)","url":"https://artificialanalysis.ai/agents/coding-agents"}],"cost_estimated":false},{"id":"aa-ea296ce7","benchmark":"index","harness":"Codex","model":"GPT-6.1 Sol","effort":"max","score":0.60148,"score_published":"60.1","ci95":null,"cost_per_attempt":1.55412,"cost_basis":"AA mean.costUsd $1.55 per attempt (its sums.costUsd $468.72 is that mean times one attempt per task, not the total over 909 attempts)","tasks":303,"attempts":909,"k":3,"date":null,"run":"Artificial Analysis Coding Agent Index row 'Codex - GPT-6.1 Sol (max)': equal-weight mean of DeepSWE v1.1, Terminal-Bench 4.0, SWE-Atlas QnA, 3 attempts per task; AA lists $1.55 per attempt. No run date published.","source_kind":"aa","sources":[{"label":"Artificial Analysis Coding Agents (no per-run page)","url":"https://artificialanalysis.ai/agents/coding-agents"}],"cost_estimated":false},{"id":"aa-40951881","benchmark":"index","harness":"Claude Code","model":"Claude Opus 5","effort":"max","score":0.59726,"score_published":"59.7","ci95":null,"cost_per_attempt":10.78915,"cost_basis":"AA mean.costUsd $10.79 per attempt (its sums.costUsd $3,269.11 is that mean times one attempt per task, not the total over 908 attempts)","tasks":303,"attempts":908,"k":3,"date":null,"run":"Artificial Analysis Coding Agent Index row 'Claude Code - Opus 5 (max)': equal-weight mean of DeepSWE v1.1, Terminal-Bench 4.0, SWE-Atlas QnA, 3 attempts per task; AA lists $10.79 per attempt. No run date published.","source_kind":"aa","sources":[{"label":"Artificial Analysis Coding Agents (no per-run page)","url":"https://artificialanalysis.ai/agents/coding-agents"}],"cost_estimated":false},{"id":"aa-fa259509","benchmark":"index","harness":"Devin Fusion CLI","model":"GPT-6 Astra + SWE-2 Medium","effort":"xhigh","score":0.58888,"score_published":"58.9","ci95":null,"cost_per_attempt":4.53649,"cost_basis":"AA mean.costUsd $4.54 per attempt (its sums.costUsd $1,374.56 is that mean times one attempt per task, not the total over 909 attempts)","tasks":303,"attempts":909,"k":3,"date":null,"run":"Artificial Analysis Coding Agent Index row 'Devin Fusion CLI - GPT-6 Astra XHigh + SWE-2 Medium': equal-weight mean of DeepSWE v1.1, Terminal-Bench 4.0, SWE-Atlas QnA, 3 attempts per task; AA lists $4.54 per attempt. No run date published.","source_kind":"aa","sources":[{"label":"Artificial Analysis Coding Agents (no per-run page)","url":"https://artificialanalysis.ai/agents/coding-agents"}],"cost_estimated":false},{"id":"aa-fa589dad","benchmark":"index","harness":"Codex","model":"GPT-6.1 Sol","effort":"low","score":0.57216,"score_published":"57.2","ci95":null,"cost_per_attempt":0.4988,"cost_basis":"AA mean.costUsd $0.50 per attempt (its sums.costUsd $150.41 is that mean times one attempt per task, not the total over 909 attempts)","tasks":303,"attempts":909,"k":3,"date":null,"run":"Artificial Analysis Coding Agent Index row 'Codex - GPT-6.1 Sol (low)': equal-weight mean of DeepSWE v1.1, Terminal-Bench 4.0, SWE-Atlas QnA, 3 attempts per task; AA lists $0.50 per attempt. No run date published.","source_kind":"aa","sources":[{"label":"Artificial Analysis Coding Agents (no per-run page)","url":"https://artificialanalysis.ai/agents/coding-agents"}],"cost_estimated":false},{"id":"aa-7a0dfd2a","benchmark":"index","harness":"Codex","model":"GPT-6 Sol","effort":"max","score":0.56663,"score_published":"56.7","ci95":null,"cost_per_attempt":2.98973,"cost_basis":"AA mean.costUsd $2.99 per attempt (its sums.costUsd $905.89 is that mean times one attempt per task, not the total over 907 attempts)","tasks":303,"attempts":907,"k":3,"date":null,"run":"Artificial Analysis Coding Agent Index row 'Codex - GPT-6 Sol (max)': equal-weight mean of DeepSWE v1.1, Terminal-Bench 4.0, SWE-Atlas QnA, 3 attempts per task; AA lists $2.99 per attempt. No run date published.","source_kind":"aa","sources":[{"label":"Artificial Analysis Coding Agents (no per-run page)","url":"https://artificialanalysis.ai/agents/coding-agents"}],"cost_estimated":false},{"id":"aa-f927d9c4","benchmark":"index","harness":"Grok Build","model":"Grok 4.7","effort":"xhigh","score":0.56268,"score_published":"56.3","ci95":null,"cost_per_attempt":8.81747,"cost_basis":"AA mean.costUsd $8.82 per attempt (its sums.costUsd $2,671.69 is that mean times one attempt per task, not the total over 905 attempts)","tasks":303,"attempts":905,"k":3,"date":null,"run":"Artificial Analysis Coding Agent Index row 'Grok Build - Grok 4.7 (xhigh)': equal-weight mean of DeepSWE v1.1, Terminal-Bench 4.0, SWE-Atlas QnA, 3 attempts per task; AA lists $8.82 per attempt. No run date published.","source_kind":"aa","sources":[{"label":"Artificial Analysis Coding Agents (no per-run page)","url":"https://artificialanalysis.ai/agents/coding-agents"}],"cost_estimated":false},{"id":"aa-33e80f38","benchmark":"index","harness":"Claude Code","model":"Claude Sonnet 5.5","effort":"high","score":0.55012,"score_published":"55.0","ci95":null,"cost_per_attempt":1.2355,"cost_basis":"AA mean.costUsd $1.24 per attempt (its sums.costUsd $374.36 is that mean times one attempt per task, not the total over 909 attempts)","tasks":303,"attempts":909,"k":3,"date":null,"run":"Artificial Analysis Coding Agent Index row 'Claude Code - Sonnet 5.5 (high)': equal-weight mean of DeepSWE v1.1, Terminal-Bench 4.0, SWE-Atlas QnA, 3 attempts per task; AA lists $1.24 per attempt. No run date published.","source_kind":"aa","sources":[{"label":"Artificial Analysis Coding Agents (no per-run page)","url":"https://artificialanalysis.ai/agents/coding-agents"}],"cost_estimated":false},{"id":"aa-709f8012","benchmark":"index","harness":"Codex","model":"GPT-5.6 Sol","effort":"max","score":0.54559,"score_published":"54.6","ci95":null,"cost_per_attempt":6.35332,"cost_basis":"AA mean.costUsd $6.35 per attempt (its sums.costUsd $1,925.06 is that mean times one attempt per task, not the total over 909 attempts)","tasks":303,"attempts":909,"k":3,"date":null,"run":"Artificial Analysis Coding Agent Index row 'Codex - GPT-5.6 Sol (max)': equal-weight mean of DeepSWE v1.1, Terminal-Bench 4.0, SWE-Atlas QnA, 3 attempts per task; AA lists $6.35 per attempt. No run date published.","source_kind":"aa","sources":[{"label":"Artificial Analysis Coding Agents (no per-run page)","url":"https://artificialanalysis.ai/agents/coding-agents"}],"cost_estimated":false},{"id":"aa-f2de0737","benchmark":"index","harness":"Muse Code","model":"Muse Spark 1.3","effort":"max","score":0.54303,"score_published":"54.3","ci95":null,"cost_per_attempt":3.9775,"cost_basis":"AA mean.costUsd $3.98 per attempt (its sums.costUsd $1,205.18 is that mean times one attempt per task, not the total over 909 attempts)","tasks":303,"attempts":909,"k":3,"date":null,"run":"Artificial Analysis Coding Agent Index row 'Muse Code - Muse Spark 1.3 (max)': equal-weight mean of DeepSWE v1.1, Terminal-Bench 4.0, SWE-Atlas QnA, 3 attempts per task; AA lists $3.98 per attempt. No run date published.","source_kind":"aa","sources":[{"label":"Artificial Analysis Coding Agents (no per-run page)","url":"https://artificialanalysis.ai/agents/coding-agents"}],"cost_estimated":false},{"id":"aa-45493c27","benchmark":"index","harness":"Opencode","model":"GLM-5.3","effort":"max","score":0.53555,"score_published":"53.6","ci95":null,"cost_per_attempt":4.24139,"cost_basis":"AA mean.costUsd $4.24 per attempt (its sums.costUsd $1,285.14 is that mean times one attempt per task, not the total over 909 attempts)","tasks":303,"attempts":909,"k":3,"date":null,"run":"Artificial Analysis Coding Agent Index row 'Opencode - GLM-5.3': equal-weight mean of DeepSWE v1.1, Terminal-Bench 4.0, SWE-Atlas QnA, 3 attempts per task; AA lists $4.24 per attempt. No run date published.","source_kind":"aa","sources":[{"label":"Artificial Analysis Coding Agents (no per-run page)","url":"https://artificialanalysis.ai/agents/coding-agents"}],"cost_estimated":false},{"id":"aa-55b029d6","benchmark":"index","harness":"Kimi Code CLI","model":"Kimi K3","effort":null,"score":0.51926,"score_published":"51.9","ci95":null,"cost_per_attempt":5.0472,"cost_basis":"AA mean.costUsd $5.05 per attempt (its sums.costUsd $1,529.30 is that mean times one attempt per task, not the total over 909 attempts)","tasks":303,"attempts":909,"k":3,"date":null,"run":"Artificial Analysis Coding Agent Index row 'Kimi Code CLI - Kimi K3': equal-weight mean of DeepSWE v1.1, Terminal-Bench 4.0, SWE-Atlas QnA, 3 attempts per task; AA lists $5.05 per attempt. No run date published.","source_kind":"aa","sources":[{"label":"Artificial Analysis Coding Agents (no per-run page)","url":"https://artificialanalysis.ai/agents/coding-agents"}],"cost_estimated":false},{"id":"aa-cd60470c","benchmark":"index","harness":"Muse Code","model":"Muse Spark 1.3","effort":"xhigh","score":0.48299,"score_published":"48.3","ci95":null,"cost_per_attempt":3.46775,"cost_basis":"AA mean.costUsd $3.47 per attempt (its sums.costUsd $1,050.73 is that mean times one attempt per task, not the total over 909 attempts)","tasks":303,"attempts":909,"k":3,"date":null,"run":"Artificial Analysis Coding Agent Index row 'Muse Code - Muse Spark 1.3 (xhigh)': equal-weight mean of DeepSWE v1.1, Terminal-Bench 4.0, SWE-Atlas QnA, 3 attempts per task; AA lists $3.47 per attempt. No run date published.","source_kind":"aa","sources":[{"label":"Artificial Analysis Coding Agents (no per-run page)","url":"https://artificialanalysis.ai/agents/coding-agents"}],"cost_estimated":false},{"id":"aa-70a5d300","benchmark":"index","harness":"Grok Build","model":"Grok 4.6","effort":"xhigh","score":0.46969,"score_published":"47.0","ci95":null,"cost_per_attempt":3.57014,"cost_basis":"AA mean.costUsd $3.57 per attempt (its sums.costUsd $1,081.75 is that mean times one attempt per task, not the total over 909 attempts)","tasks":303,"attempts":909,"k":3,"date":null,"run":"Artificial Analysis Coding Agent Index row 'Grok Build - Grok 4.6 (xhigh)': equal-weight mean of DeepSWE v1.1, Terminal-Bench 4.0, SWE-Atlas QnA, 3 attempts per task; AA lists $3.57 per attempt. No run date published.","source_kind":"aa","sources":[{"label":"Artificial Analysis Coding Agents (no per-run page)","url":"https://artificialanalysis.ai/agents/coding-agents"}],"cost_estimated":false},{"id":"aa-4d6aa9e9","benchmark":"index","harness":"Claude Code","model":"Claude Sonnet 5.5","effort":"medium","score":0.45884,"score_published":"45.9","ci95":null,"cost_per_attempt":0.61875,"cost_basis":"AA mean.costUsd $0.62 per attempt (its sums.costUsd $187.48 is that mean times one attempt per task, not the total over 909 attempts)","tasks":303,"attempts":909,"k":3,"date":null,"run":"Artificial Analysis Coding Agent Index row 'Claude Code - Sonnet 5.5 (medium)': equal-weight mean of DeepSWE v1.1, Terminal-Bench 4.0, SWE-Atlas QnA, 3 attempts per task; AA lists $0.62 per attempt. No run date published.","source_kind":"aa","sources":[{"label":"Artificial Analysis Coding Agents (no per-run page)","url":"https://artificialanalysis.ai/agents/coding-agents"}],"cost_estimated":false},{"id":"aa-cd4c865d","benchmark":"index","harness":"Claude Code","model":"Qwen3.8 Max","effort":null,"score":0.43265,"score_published":"43.3","ci95":null,"cost_per_attempt":3.47897,"cost_basis":"AA mean.costUsd $3.48 per attempt (its sums.costUsd $1,048.21 is that mean times one attempt per task, not the total over 909 attempts)","tasks":303,"attempts":909,"k":3,"date":null,"run":"Artificial Analysis Coding Agent Index row 'Claude Code - Qwen3.8 Max': equal-weight mean of DeepSWE v1.1, Terminal-Bench 4.0, SWE-Atlas QnA, 3 attempts per task; AA lists $3.48 per attempt. No run date published.","source_kind":"aa","sources":[{"label":"Artificial Analysis Coding Agents (no per-run page)","url":"https://artificialanalysis.ai/agents/coding-agents"}],"cost_estimated":false},{"id":"aa-b3c56ec6","benchmark":"index","harness":"Codex","model":"GPT-5.6 Luna","effort":"max","score":0.43225,"score_published":"43.2","ci95":null,"cost_per_attempt":0.43802,"cost_basis":"AA mean.costUsd $0.44 per attempt (its sums.costUsd $132.72 is that mean times one attempt per task, not the total over 909 attempts)","tasks":303,"attempts":909,"k":3,"date":null,"run":"Artificial Analysis Coding Agent Index row 'Codex - GPT-5.6 Luna (max)': equal-weight mean of DeepSWE v1.1, Terminal-Bench 4.0, SWE-Atlas QnA, 3 attempts per task; AA lists $0.44 per attempt. No run date published.","source_kind":"aa","sources":[{"label":"Artificial Analysis Coding Agents (no per-run page)","url":"https://artificialanalysis.ai/agents/coding-agents"}],"cost_estimated":false},{"id":"aa-542590ec","benchmark":"index","harness":"Codex","model":"DeepSeek V4 Pro 0813","effort":"max","score":0.43052,"score_published":"43.1","ci95":null,"cost_per_attempt":0.23805,"cost_basis":"AA mean.costUsd $0.24 per attempt (its sums.costUsd $72.13 is that mean times one attempt per task, not the total over 909 attempts)","tasks":303,"attempts":909,"k":3,"date":null,"run":"Artificial Analysis Coding Agent Index row 'Codex - DeepSeek V4 Pro 0813 (max)': equal-weight mean of DeepSWE v1.1, Terminal-Bench 4.0, SWE-Atlas QnA, 3 attempts per task; AA lists $0.24 per attempt. No run date published.","source_kind":"aa","sources":[{"label":"Artificial Analysis Coding Agents (no per-run page)","url":"https://artificialanalysis.ai/agents/coding-agents"}],"cost_estimated":false},{"id":"aa-e9f9d3ae","benchmark":"index","harness":"Claude Code","model":"Claude Sonnet 5.5","effort":"low","score":0.42059,"score_published":"42.1","ci95":null,"cost_per_attempt":0.48311,"cost_basis":"AA mean.costUsd $0.48 per attempt (its sums.costUsd $146.38 is that mean times one attempt per task, not the total over 909 attempts)","tasks":303,"attempts":909,"k":3,"date":null,"run":"Artificial Analysis Coding Agent Index row 'Claude Code - Sonnet 5.5 (low)': equal-weight mean of DeepSWE v1.1, Terminal-Bench 4.0, SWE-Atlas QnA, 3 attempts per task; AA lists $0.48 per attempt. No run date published.","source_kind":"aa","sources":[{"label":"Artificial Analysis Coding Agents (no per-run page)","url":"https://artificialanalysis.ai/agents/coding-agents"}],"cost_estimated":false},{"id":"aa-d1003684","benchmark":"index","harness":"Antigravity SDK","model":"Gemini 3.8 Flash","effort":"high","score":0.41863,"score_published":"41.9","ci95":null,"cost_per_attempt":2.46739,"cost_basis":"AA mean.costUsd $2.47 per attempt (its sums.costUsd $747.62 is that mean times one attempt per task, not the total over 909 attempts)","tasks":303,"attempts":909,"k":3,"date":null,"run":"Artificial Analysis Coding Agent Index row 'Antigravity SDK - Gemini 3.8 Flash (high)': equal-weight mean of DeepSWE v1.1, Terminal-Bench 4.0, SWE-Atlas QnA, 3 attempts per task; AA lists $2.47 per attempt. No run date published.","source_kind":"aa","sources":[{"label":"Artificial Analysis Coding Agents (no per-run page)","url":"https://artificialanalysis.ai/agents/coding-agents"}],"cost_estimated":false},{"id":"aa-ac8276cd","benchmark":"index","harness":"Codex","model":"GPT-6 Luna","effort":"max","score":0.41074,"score_published":"41.1","ci95":null,"cost_per_attempt":0.17591,"cost_basis":"AA mean.costUsd $0.18 per attempt (its sums.costUsd $53.30 is that mean times one attempt per task, not the total over 908 attempts)","tasks":303,"attempts":908,"k":3,"date":null,"run":"Artificial Analysis Coding Agent Index row 'Codex - GPT-6 Luna (max)': equal-weight mean of DeepSWE v1.1, Terminal-Bench 4.0, SWE-Atlas QnA, 3 attempts per task; AA lists $0.18 per attempt. No run date published.","source_kind":"aa","sources":[{"label":"Artificial Analysis Coding Agents (no per-run page)","url":"https://artificialanalysis.ai/agents/coding-agents"}],"cost_estimated":false},{"id":"aa-28ef6513","benchmark":"index","harness":"Codex","model":"DeepSeek V4 Flash 0731","effort":"max","score":0.38742,"score_published":"38.7","ci95":null,"cost_per_attempt":0.08506,"cost_basis":"AA mean.costUsd $0.09 per attempt (its sums.costUsd $25.77 is that mean times one attempt per task, not the total over 909 attempts)","tasks":303,"attempts":909,"k":3,"date":null,"run":"Artificial Analysis Coding Agent Index row 'Codex - DeepSeek V4 Flash 0731 (max)': equal-weight mean of DeepSWE v1.1, Terminal-Bench 4.0, SWE-Atlas QnA, 3 attempts per task; AA lists $0.09 per attempt. No run date published.","source_kind":"aa","sources":[{"label":"Artificial Analysis Coding Agents (no per-run page)","url":"https://artificialanalysis.ai/agents/coding-agents"}],"cost_estimated":false},{"id":"aa-ef63fafa-deepswe","benchmark":"deepswe","harness":"Antigravity CLI","model":"Gemini 4 Argon","effort":"high","score":0.78761,"score_published":"78.8%","ci95":null,"cost_per_attempt":8.43889,"cost_basis":"ESTIMATE: AA publishes only the row's mean cost, $5.84 per attempt; allocated 53.6% to this benchmark by attempts x (input x (1 - 0.9 x cacheHitRate 0.87) + 5 x output tokens), as a share of that mean","cost_estimated":true,"tasks":113,"attempts":336,"k":3,"date":null,"run":"Artificial Analysis per-benchmark mean reward for 'Antigravity CLI - Gemini 4 Argon (high)', 336 attempts. No run date published. Row marked unavailable on AA.","source_kind":"aa","sources":[{"label":"Artificial Analysis Coding Agents (no per-run page)","url":"https://artificialanalysis.ai/agents/coding-agents"}]},{"id":"deepswe-mini_swe_agent_gpt_6_astra_xhigh","benchmark":"deepswe","harness":"mini-swe-agent","model":"GPT-6 Astra","effort":"xhigh","score":0.74115,"score_published":"74%","ci95":[0.69886,0.77938],"cost_per_attempt":4.42912,"cost_basis":"leaderboard mean_cost_usd per attempt over 452 attempts; leaderboard note: Current pricing at all context lengths: $10/M uncached input, $12.50/M cache writes, $1/M cache reads, $50/M output; no separate compute-unit fee.","tasks":113,"attempts":452,"k":4,"date":"2026-09-22","run":"Datacurve DeepSWE v1.1 leaderboard (mini-swe-agent), 113 tasks x 4 runs; 335/452 passed. Leaderboard CI (run-to-run) [0.712, 0.770]; ci95 here is Wilson over attempts.","source_kind":"leaderboard","sources":[{"label":"Trial index (JSON)","url":"https://deepswe.datacurve.ai/artifacts/v1.1/trials.json"},{"label":"DeepSWE v1.1 leaderboard-live.json","url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json"},{"label":"DeepSWE v1.1 blog","url":"https://deepswe.datacurve.ai/blog/deepswe-v1-1"}],"run_ref":"Config mini_swe_agent_gpt_6_astra_xhigh","cost_estimated":false},{"id":"deepswe-mini_swe_agent_gemini_3_8_flash_high","benchmark":"deepswe","harness":"mini-swe-agent","model":"Gemini 3.8 Flash","effort":"high","score":0.73826,"score_published":"74%","ci95":[0.6956,0.77685],"cost_per_attempt":2.36235,"cost_basis":"leaderboard mean_cost_usd per attempt over 447 attempts","tasks":113,"attempts":447,"k":4,"date":"2026-09-22","run":"Datacurve DeepSWE v1.1 leaderboard (mini-swe-agent), 113 tasks x 4 runs; 330/447 passed. Leaderboard CI (run-to-run) [0.724, 0.752]; ci95 here is Wilson over attempts.","source_kind":"leaderboard","sources":[{"label":"Trial index (JSON)","url":"https://deepswe.datacurve.ai/artifacts/v1.1/trials.json"},{"label":"DeepSWE v1.1 leaderboard-live.json","url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json"},{"label":"DeepSWE v1.1 blog","url":"https://deepswe.datacurve.ai/blog/deepswe-v1-1"}],"run_ref":"Config mini_swe_agent_gemini_3_8_flash_high","cost_estimated":false},{"id":"deepswe-mini_swe_agent_claude_opus_5_max","benchmark":"deepswe","harness":"mini-swe-agent","model":"Claude Opus 5","effort":"max","score":0.73649,"score_published":"74%","ci95":[0.69361,0.77531],"cost_per_attempt":11.83758,"cost_basis":"leaderboard mean_cost_usd per attempt over 444 attempts","tasks":113,"attempts":444,"k":4,"date":"2026-09-22","run":"Datacurve DeepSWE v1.1 leaderboard (mini-swe-agent), 113 tasks x 4 runs; 327/444 passed. Leaderboard CI (run-to-run) [0.698, 0.775]; ci95 here is Wilson over attempts.","source_kind":"leaderboard","sources":[{"label":"Trial index (JSON)","url":"https://deepswe.datacurve.ai/artifacts/v1.1/trials.json"},{"label":"DeepSWE v1.1 leaderboard-live.json","url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json"},{"label":"DeepSWE v1.1 blog","url":"https://deepswe.datacurve.ai/blog/deepswe-v1-1"}],"run_ref":"Config mini_swe_agent_claude_opus_5_max","cost_estimated":false},{"id":"deepswe-mini_swe_agent_gpt_6_astra_high","benchmark":"deepswe","harness":"mini-swe-agent","model":"GPT-6 Astra","effort":"high","score":0.7323,"score_published":"73%","ci95":[0.68965,0.77104],"cost_per_attempt":3.92372,"cost_basis":"leaderboard mean_cost_usd per attempt over 452 attempts; leaderboard note: Current pricing at all context lengths: $10/M uncached input, $12.50/M cache writes, $1/M cache reads, $50/M output; no separate compute-unit fee.","tasks":113,"attempts":452,"k":4,"date":"2026-09-22","run":"Datacurve DeepSWE v1.1 leaderboard (mini-swe-agent), 113 tasks x 4 runs; 331/452 passed. Leaderboard CI (run-to-run) [0.698, 0.767]; ci95 here is Wilson over attempts.","source_kind":"leaderboard","sources":[{"label":"Trial index (JSON)","url":"https://deepswe.datacurve.ai/artifacts/v1.1/trials.json"},{"label":"DeepSWE v1.1 leaderboard-live.json","url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json"},{"label":"DeepSWE v1.1 blog","url":"https://deepswe.datacurve.ai/blog/deepswe-v1-1"}],"run_ref":"Config mini_swe_agent_gpt_6_astra_high","cost_estimated":false},{"id":"deepswe-mini_swe_agent_gpt_6_astra_max","benchmark":"deepswe","harness":"mini-swe-agent","model":"GPT-6 Astra","effort":"max","score":0.7323,"score_published":"73%","ci95":[0.68965,0.77104],"cost_per_attempt":7.49783,"cost_basis":"leaderboard mean_cost_usd per attempt over 452 attempts; leaderboard note: Current pricing at all context lengths: $10/M uncached input, $12.50/M cache writes, $1/M cache reads, $50/M output; no separate compute-unit fee.","tasks":113,"attempts":452,"k":4,"date":"2026-09-22","run":"Datacurve DeepSWE v1.1 leaderboard (mini-swe-agent), 113 tasks x 4 runs; 331/452 passed. Leaderboard CI (run-to-run) [0.724, 0.741]; ci95 here is Wilson over attempts.","source_kind":"leaderboard","sources":[{"label":"Trial index (JSON)","url":"https://deepswe.datacurve.ai/artifacts/v1.1/trials.json"},{"label":"DeepSWE v1.1 leaderboard-live.json","url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json"},{"label":"DeepSWE v1.1 blog","url":"https://deepswe.datacurve.ai/blog/deepswe-v1-1"}],"run_ref":"Config mini_swe_agent_gpt_6_astra_max","cost_estimated":false},{"id":"aa-ed8c1d0b-deepswe","benchmark":"deepswe","harness":"Codex","model":"GPT-6.1 Sol","effort":"xhigh","score":0.73156,"score_published":"73.2%","ci95":null,"cost_per_attempt":0.93927,"cost_basis":"ESTIMATE: AA publishes only the row's mean cost, $1.04 per attempt; allocated 33.7% to this benchmark by attempts x (input x (1 - 0.9 x cacheHitRate 0.93) + 5 x output tokens), as a share of that mean","cost_estimated":true,"tasks":113,"attempts":339,"k":3,"date":null,"run":"Artificial Analysis per-benchmark mean reward for 'Codex - GPT-6.1 Sol (xhigh)', 339 attempts. No run date published.","source_kind":"aa","sources":[{"label":"Artificial Analysis Coding Agents (no per-run page)","url":"https://artificialanalysis.ai/agents/coding-agents"}]},{"id":"aa-cd60470c-deepswe","benchmark":"deepswe","harness":"Muse Code","model":"Muse Spark 1.3","effort":"xhigh","score":0.73156,"score_published":"73.2%","ci95":null,"cost_per_attempt":2.91764,"cost_basis":"ESTIMATE: AA publishes only the row's mean cost, $3.47 per attempt; allocated 31.4% to this benchmark by attempts x (input x (1 - 0.9 x cacheHitRate 0.97) + 5 x output tokens), as a share of that mean","cost_estimated":true,"tasks":113,"attempts":339,"k":3,"date":null,"run":"Artificial Analysis per-benchmark mean reward for 'Muse Code - Muse Spark 1.3 (xhigh)', 339 attempts. No run date published.","source_kind":"aa","sources":[{"label":"Artificial Analysis Coding Agents (no per-run page)","url":"https://artificialanalysis.ai/agents/coding-agents"}]},{"id":"deepswe-mini_swe_agent_claude_opus_5_xhigh","benchmark":"deepswe","harness":"mini-swe-agent","model":"Claude Opus 5","effort":"xhigh","score":0.73154,"score_published":"73%","ci95":[0.68862,0.77052],"cost_per_attempt":9.0722,"cost_basis":"leaderboard mean_cost_usd per attempt over 447 attempts","tasks":113,"attempts":447,"k":4,"date":"2026-09-22","run":"Datacurve DeepSWE v1.1 leaderboard (mini-swe-agent), 113 tasks x 4 runs; 327/447 passed. Leaderboard CI (run-to-run) [0.701, 0.762]; ci95 here is Wilson over attempts.","source_kind":"leaderboard","sources":[{"label":"Trial index (JSON)","url":"https://deepswe.datacurve.ai/artifacts/v1.1/trials.json"},{"label":"DeepSWE v1.1 leaderboard-live.json","url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json"},{"label":"DeepSWE v1.1 blog","url":"https://deepswe.datacurve.ai/blog/deepswe-v1-1"}],"run_ref":"Config mini_swe_agent_claude_opus_5_xhigh","cost_estimated":false},{"id":"deepswe-mini_swe_agent_claude_opus_5_high","benchmark":"deepswe","harness":"mini-swe-agent","model":"Claude Opus 5","effort":"high","score":0.72829,"score_published":"73%","ci95":[0.68533,0.76737],"cost_per_attempt":6.07608,"cost_basis":"leaderboard mean_cost_usd per attempt over 449 attempts","tasks":113,"attempts":449,"k":4,"date":"2026-09-22","run":"Datacurve DeepSWE v1.1 leaderboard (mini-swe-agent), 113 tasks x 4 runs; 327/449 passed. Leaderboard CI (run-to-run) [0.709, 0.748]; ci95 here is Wilson over attempts.","source_kind":"leaderboard","sources":[{"label":"Trial index (JSON)","url":"https://deepswe.datacurve.ai/artifacts/v1.1/trials.json"},{"label":"DeepSWE v1.1 leaderboard-live.json","url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json"},{"label":"DeepSWE v1.1 blog","url":"https://deepswe.datacurve.ai/blog/deepswe-v1-1"}],"run_ref":"Config mini_swe_agent_claude_opus_5_high","cost_estimated":false},{"id":"deepswe-mini_swe_agent_gpt_6_astra_medium","benchmark":"deepswe","harness":"mini-swe-agent","model":"GPT-6 Astra","effort":"medium","score":0.72788,"score_published":"73%","ci95":[0.68505,0.76686],"cost_per_attempt":3.07547,"cost_basis":"leaderboard mean_cost_usd per attempt over 452 attempts; leaderboard note: Current pricing at all context lengths: $10/M uncached input, $12.50/M cache writes, $1/M cache reads, $50/M output; no separate compute-unit fee.","tasks":113,"attempts":452,"k":4,"date":"2026-09-22","run":"Datacurve DeepSWE v1.1 leaderboard (mini-swe-agent), 113 tasks x 4 runs; 329/452 passed. Leaderboard CI (run-to-run) [0.702, 0.754]; ci95 here is Wilson over attempts.","source_kind":"leaderboard","sources":[{"label":"Trial index (JSON)","url":"https://deepswe.datacurve.ai/artifacts/v1.1/trials.json"},{"label":"DeepSWE v1.1 leaderboard-live.json","url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json"},{"label":"DeepSWE v1.1 blog","url":"https://deepswe.datacurve.ai/blog/deepswe-v1-1"}],"run_ref":"Config mini_swe_agent_gpt_6_astra_medium","cost_estimated":false},{"id":"deepswe-mini_swe_agent_gpt_5_6_sol_max","benchmark":"deepswe","harness":"mini-swe-agent","model":"GPT-5.6 Sol","effort":"max","score":0.72667,"score_published":"73%","ci95":[0.6837,0.7658],"cost_per_attempt":8.38644,"cost_basis":"leaderboard mean_cost_usd per attempt over 450 attempts","tasks":113,"attempts":450,"k":4,"date":"2026-09-22","run":"Datacurve DeepSWE v1.1 leaderboard (mini-swe-agent), 113 tasks x 4 runs; 327/450 passed. Leaderboard CI (run-to-run) [0.698, 0.755]; ci95 here is Wilson over attempts.","source_kind":"leaderboard","sources":[{"label":"Trial index (JSON)","url":"https://deepswe.datacurve.ai/artifacts/v1.1/trials.json"},{"label":"DeepSWE v1.1 leaderboard-live.json","url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json"},{"label":"DeepSWE v1.1 blog","url":"https://deepswe.datacurve.ai/blog/deepswe-v1-1"}],"run_ref":"Config mini_swe_agent_gpt_5_6_sol_max","cost_estimated":false},{"id":"aa-f927d9c4-deepswe","benchmark":"deepswe","harness":"Grok Build","model":"Grok 4.7","effort":"xhigh","score":0.72566,"score_published":"72.6%","ci95":null,"cost_per_attempt":9.50584,"cost_basis":"ESTIMATE: AA publishes only the row's mean cost, $8.82 per attempt; allocated 39.9% to this benchmark by attempts x (input x (1 - 0.9 x cacheHitRate 0.94) + 5 x output tokens), as a share of that mean","cost_estimated":true,"tasks":113,"attempts":335,"k":3,"date":null,"run":"Artificial Analysis per-benchmark mean reward for 'Grok Build - Grok 4.7 (xhigh)', 335 attempts. No run date published.","source_kind":"aa","sources":[{"label":"Artificial Analysis Coding Agents (no per-run page)","url":"https://artificialanalysis.ai/agents/coding-agents"}]},{"id":"aa-709f8012-deepswe","benchmark":"deepswe","harness":"Codex","model":"GPT-5.6 Sol","effort":"max","score":0.72271,"score_published":"72.3%","ci95":null,"cost_per_attempt":6.91918,"cost_basis":"ESTIMATE: AA publishes only the row's mean cost, $6.35 per attempt; allocated 40.6% to this benchmark by attempts x (input x (1 - 0.9 x cacheHitRate 0.95) + 5 x output tokens), as a share of that mean","cost_estimated":true,"tasks":113,"attempts":339,"k":3,"date":null,"run":"Artificial Analysis per-benchmark mean reward for 'Codex - GPT-5.6 Sol (max)', 339 attempts. No run date published.","source_kind":"aa","sources":[{"label":"Artificial Analysis Coding Agents (no per-run page)","url":"https://artificialanalysis.ai/agents/coding-agents"}]},{"id":"aa-a2c87c06-deepswe","benchmark":"deepswe","harness":"Claude Code","model":"Claude Sonnet 5.5","effort":"max","score":0.71976,"score_published":"72.0%","ci95":null,"cost_per_attempt":12.96281,"cost_basis":"ESTIMATE: AA publishes only the row's mean cost, $14.19 per attempt; allocated 34.1% to this benchmark by attempts x (input x (1 - 0.9 x cacheHitRate 0.95) + 5 x output tokens), as a share of that mean","cost_estimated":true,"tasks":113,"attempts":339,"k":3,"date":null,"run":"Artificial Analysis per-benchmark mean reward for 'Claude Code - Sonnet 5.5 (max)', 339 attempts. No run date published.","source_kind":"aa","sources":[{"label":"Artificial Analysis Coding Agents (no per-run page)","url":"https://artificialanalysis.ai/agents/coding-agents"}]},{"id":"aa-93acd59f-deepswe","benchmark":"deepswe","harness":"Codex","model":"GPT-6.1 Sol","effort":"medium","score":0.71976,"score_published":"72.0%","ci95":null,"cost_per_attempt":0.61264,"cost_basis":"ESTIMATE: AA publishes only the row's mean cost, $0.70 per attempt; allocated 32.4% to this benchmark by attempts x (input x (1 - 0.9 x cacheHitRate 0.92) + 5 x output tokens), as a share of that mean","cost_estimated":true,"tasks":113,"attempts":339,"k":3,"date":null,"run":"Artificial Analysis per-benchmark mean reward for 'Codex - GPT-6.1 Sol (medium)', 339 attempts. No run date published.","source_kind":"aa","sources":[{"label":"Artificial Analysis Coding Agents (no per-run page)","url":"https://artificialanalysis.ai/agents/coding-agents"}]},{"id":"aa-f2de0737-deepswe","benchmark":"deepswe","harness":"Muse Code","model":"Muse Spark 1.3","effort":"max","score":0.71681,"score_published":"71.7%","ci95":null,"cost_per_attempt":2.71828,"cost_basis":"ESTIMATE: AA publishes only the row's mean cost, $3.98 per attempt; allocated 25.5% to this benchmark by attempts x (input x (1 - 0.9 x cacheHitRate 0.97) + 5 x output tokens), as a share of that mean","cost_estimated":true,"tasks":113,"attempts":339,"k":3,"date":null,"run":"Artificial Analysis per-benchmark mean reward for 'Muse Code - Muse Spark 1.3 (max)', 339 attempts. No run date published.","source_kind":"aa","sources":[{"label":"Artificial Analysis Coding Agents (no per-run page)","url":"https://artificialanalysis.ai/agents/coding-agents"}]},{"id":"deepswe-mini_swe_agent_gemini_3_8_flash_medium","benchmark":"deepswe","harness":"mini-swe-agent","model":"Gemini 3.8 Flash","effort":"medium","score":0.71018,"score_published":"71%","ci95":[0.66672,0.75009],"cost_per_attempt":1.9671,"cost_basis":"leaderboard mean_cost_usd per attempt over 452 attempts","tasks":113,"attempts":452,"k":4,"date":"2026-09-22","run":"Datacurve DeepSWE v1.1 leaderboard (mini-swe-agent), 113 tasks x 4 runs; 321/452 passed. Leaderboard CI (run-to-run) [0.687, 0.733]; ci95 here is Wilson over attempts.","source_kind":"leaderboard","sources":[{"label":"Trial index (JSON)","url":"https://deepswe.datacurve.ai/artifacts/v1.1/trials.json"},{"label":"DeepSWE v1.1 leaderboard-live.json","url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json"},{"label":"DeepSWE v1.1 blog","url":"https://deepswe.datacurve.ai/blog/deepswe-v1-1"}],"run_ref":"Config mini_swe_agent_gemini_3_8_flash_medium","cost_estimated":false},{"id":"deepswe-mini_swe_agent_gpt_5_6_sol_xhigh","benchmark":"deepswe","harness":"mini-swe-agent","model":"GPT-5.6 Sol","effort":"xhigh","score":0.70732,"score_published":"71%","ci95":[0.66372,0.74742],"cost_per_attempt":4.70366,"cost_basis":"leaderboard mean_cost_usd per attempt over 451 attempts","tasks":113,"attempts":451,"k":4,"date":"2026-09-22","run":"Datacurve DeepSWE v1.1 leaderboard (mini-swe-agent), 113 tasks x 4 runs; 319/451 passed. Leaderboard CI (run-to-run) [0.699, 0.716]; ci95 here is Wilson over attempts.","source_kind":"leaderboard","sources":[{"label":"Trial index (JSON)","url":"https://deepswe.datacurve.ai/artifacts/v1.1/trials.json"},{"label":"DeepSWE v1.1 leaderboard-live.json","url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json"},{"label":"DeepSWE v1.1 blog","url":"https://deepswe.datacurve.ai/blog/deepswe-v1-1"}],"run_ref":"Config mini_swe_agent_gpt_5_6_sol_xhigh","cost_estimated":false},{"id":"aa-4aee8aa2-deepswe","benchmark":"deepswe","harness":"Codex","model":"GPT-6.1 Sol","effort":"high","score":0.70501,"score_published":"70.5%","ci95":null,"cost_per_attempt":0.80812,"cost_basis":"ESTIMATE: AA publishes only the row's mean cost, $0.89 per attempt; allocated 33.9% to this benchmark by attempts x (input x (1 - 0.9 x cacheHitRate 0.93) + 5 x output tokens), as a share of that mean","cost_estimated":true,"tasks":113,"attempts":339,"k":3,"date":null,"run":"Artificial Analysis per-benchmark mean reward for 'Codex - GPT-6.1 Sol (high)', 339 attempts. No run date published.","source_kind":"aa","sources":[{"label":"Artificial Analysis Coding Agents (no per-run page)","url":"https://artificialanalysis.ai/agents/coding-agents"}]},{"id":"deepswe-mini_swe_agent_claude_fable_5_xhigh","benchmark":"deepswe","harness":"mini-swe-agent","model":"Claude Fable 5","effort":"xhigh","score":0.69912,"score_published":"70%","ci95":[0.6553,0.73957],"cost_per_attempt":13.41452,"cost_basis":"leaderboard mean_cost_usd per attempt over 452 attempts","tasks":113,"attempts":452,"k":4,"date":"2026-09-22","run":"Datacurve DeepSWE v1.1 leaderboard (mini-swe-agent), 113 tasks x 4 runs; 316/452 passed. Leaderboard CI (run-to-run) [0.667, 0.732]; ci95 here is Wilson over attempts.","source_kind":"leaderboard","sources":[{"label":"Trial index (JSON)","url":"https://deepswe.datacurve.ai/artifacts/v1.1/trials.json"},{"label":"DeepSWE v1.1 leaderboard-live.json","url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json"},{"label":"DeepSWE v1.1 blog","url":"https://deepswe.datacurve.ai/blog/deepswe-v1-1"}],"run_ref":"Config mini_swe_agent_claude_fable_5_xhigh","cost_estimated":false},{"id":"deepswe-mini_swe_agent_claude_fable_5_max","benchmark":"deepswe","harness":"mini-swe-agent","model":"Claude Fable 5","effort":"max","score":0.69725,"score_published":"70%","ci95":[0.65255,0.7385],"cost_per_attempt":21.6347,"cost_basis":"leaderboard mean_cost_usd per attempt over 436 attempts","tasks":113,"attempts":436,"k":4,"date":"2026-09-22","run":"Datacurve DeepSWE v1.1 leaderboard (mini-swe-agent), 113 tasks x 4 runs; 304/436 passed. Leaderboard CI (run-to-run) [0.657, 0.738]; ci95 here is Wilson over attempts.","source_kind":"leaderboard","sources":[{"label":"Trial index (JSON)","url":"https://deepswe.datacurve.ai/artifacts/v1.1/trials.json"},{"label":"DeepSWE v1.1 leaderboard-live.json","url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json"},{"label":"DeepSWE v1.1 blog","url":"https://deepswe.datacurve.ai/blog/deepswe-v1-1"}],"run_ref":"Config mini_swe_agent_claude_fable_5_max","cost_estimated":false},{"id":"deepswe-mini_swe_agent_gpt_5_6_terra_max","benchmark":"deepswe","harness":"mini-swe-agent","model":"GPT-5.6 Terra","effort":"max","score":0.69623,"score_published":"70%","ci95":[0.65228,0.73687],"cost_per_attempt":4.94585,"cost_basis":"leaderboard mean_cost_usd per attempt over 451 attempts","tasks":113,"attempts":451,"k":4,"date":"2026-09-22","run":"Datacurve DeepSWE v1.1 leaderboard (mini-swe-agent), 113 tasks x 4 runs; 314/451 passed. Leaderboard CI (run-to-run) [0.671, 0.722]; ci95 here is Wilson over attempts.","source_kind":"leaderboard","sources":[{"label":"Trial index (JSON)","url":"https://deepswe.datacurve.ai/artifacts/v1.1/trials.json"},{"label":"DeepSWE v1.1 leaderboard-live.json","url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json"},{"label":"DeepSWE v1.1 blog","url":"https://deepswe.datacurve.ai/blog/deepswe-v1-1"}],"run_ref":"Config mini_swe_agent_gpt_5_6_terra_max","cost_estimated":false},{"id":"hub-cd7d8de6","benchmark":"deepswe","harness":"Pi","harness_version":"0.85.1","model":"GPT-6 Astra","effort":"xhigh","score":0.69617,"score_published":"236/339","ci95":[0.64523,0.7427],"cost_per_attempt":4.67183,"cost_basis":"Harbor trial cost_usd, mean over 339 of 339 graded attempts","tasks":113,"attempts":339,"k":3,"date":"2026-09-17","run":"Harbor job '20260916-deepswe-pi-astra-xhigh-modal' by Philip Sinitsin on Modal; 113 tasks x 3, 339 graded attempts (reward>=1 is a pass, timeouts score 0); Pi 0.85.1; dataset ref sha256:5affcd534fd9.","source_kind":"harbor","sources":[{"label":"Harbor Hub job cd7d8de6","url":"https://hub.harborframework.com/jobs/cd7d8de6-c152-48de-9840-b22bbea96d99"}],"run_ref":"Job cd7d8de6-c152-48de-9840-b22bbea96d99","cost_estimated":false},{"id":"aa-ea296ce7-deepswe","benchmark":"deepswe","harness":"Codex","model":"GPT-6.1 Sol","effort":"max","score":0.69617,"score_published":"69.6%","ci95":null,"cost_per_attempt":1.55804,"cost_basis":"ESTIMATE: AA publishes only the row's mean cost, $1.55 per attempt; allocated 37.4% to this benchmark by attempts x (input x (1 - 0.9 x cacheHitRate 0.93) + 5 x output tokens), as a share of that mean","cost_estimated":true,"tasks":113,"attempts":339,"k":3,"date":null,"run":"Artificial Analysis per-benchmark mean reward for 'Codex - GPT-6.1 Sol (max)', 339 attempts. No run date published.","source_kind":"aa","sources":[{"label":"Artificial Analysis Coding Agents (no per-run page)","url":"https://artificialanalysis.ai/agents/coding-agents"}]},{"id":"deepswe-mini_swe_agent_gpt_5_6_sol_high","benchmark":"deepswe","harness":"mini-swe-agent","model":"GPT-5.6 Sol","effort":"high","score":0.69401,"score_published":"69%","ci95":[0.64999,0.73476],"cost_per_attempt":3.46983,"cost_basis":"leaderboard mean_cost_usd per attempt over 451 attempts","tasks":113,"attempts":451,"k":4,"date":"2026-09-22","run":"Datacurve DeepSWE v1.1 leaderboard (mini-swe-agent), 113 tasks x 4 runs; 313/451 passed. Leaderboard CI (run-to-run) [0.680, 0.708]; ci95 here is Wilson over attempts.","source_kind":"leaderboard","sources":[{"label":"Trial index (JSON)","url":"https://deepswe.datacurve.ai/artifacts/v1.1/trials.json"},{"label":"DeepSWE v1.1 leaderboard-live.json","url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json"},{"label":"DeepSWE v1.1 blog","url":"https://deepswe.datacurve.ai/blog/deepswe-v1-1"}],"run_ref":"Config mini_swe_agent_gpt_5_6_sol_high","cost_estimated":false},{"id":"aa-7a0dfd2a-deepswe","benchmark":"deepswe","harness":"Codex","model":"GPT-6 Sol","effort":"max","score":0.69027,"score_published":"69.0%","ci95":null,"cost_per_attempt":3.05954,"cost_basis":"ESTIMATE: AA publishes only the row's mean cost, $2.99 per attempt; allocated 38.0% to this benchmark by attempts x (input x (1 - 0.9 x cacheHitRate 0.96) + 5 x output tokens), as a share of that mean","cost_estimated":true,"tasks":113,"attempts":337,"k":3,"date":null,"run":"Artificial Analysis per-benchmark mean reward for 'Codex - GPT-6 Sol (max)', 337 attempts. No run date published.","source_kind":"aa","sources":[{"label":"Artificial Analysis Coding Agents (no per-run page)","url":"https://artificialanalysis.ai/agents/coding-agents"}]},{"id":"hub-e20ecafd","benchmark":"deepswe","harness":"Codex","harness_version":"0.155.1","model":"GPT-6 Astra","effort":"xhigh","score":0.69027,"score_published":"234/339","ci95":[0.63914,0.73712],"cost_per_attempt":4.87213,"cost_basis":"Harbor trial cost_usd, mean over 339 of 339 graded attempts","tasks":113,"attempts":339,"k":3,"date":"2026-09-21","run":"Harbor job '20260916-deepswe-codex-astra-xhigh-modal' by Philip Sinitsin on Modal; 113 tasks x 3, 339 graded attempts (reward>=1 is a pass, timeouts score 0); Codex 0.155.1; dataset ref sha256:5affcd534fd9.","source_kind":"harbor","sources":[{"label":"Harbor Hub job e20ecafd","url":"https://hub.harborframework.com/jobs/e20ecafd-695a-417c-90f5-35fd36d2786f"}],"run_ref":"Job e20ecafd-695a-417c-90f5-35fd36d2786f","cost_estimated":false},{"id":"deepswe-mini_swe_agent_glm_5_3_max","benchmark":"deepswe","harness":"mini-swe-agent","model":"GLM-5.3","effort":"max","score":0.68958,"score_published":"69%","ci95":[0.64543,0.73053],"cost_per_attempt":3.99336,"cost_basis":"leaderboard mean_cost_usd per attempt over 451 attempts","tasks":113,"attempts":451,"k":4,"date":"2026-09-22","run":"Datacurve DeepSWE v1.1 leaderboard (mini-swe-agent), 113 tasks x 4 runs; 311/451 passed. Leaderboard CI (run-to-run) [0.659, 0.720]; ci95 here is Wilson over attempts.","source_kind":"leaderboard","sources":[{"label":"Trial index (JSON)","url":"https://deepswe.datacurve.ai/artifacts/v1.1/trials.json"},{"label":"DeepSWE v1.1 leaderboard-live.json","url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json"},{"label":"DeepSWE v1.1 blog","url":"https://deepswe.datacurve.ai/blog/deepswe-v1-1"}],"run_ref":"Config mini_swe_agent_glm_5_3_max","cost_estimated":false},{"id":"deepswe-mini_swe_agent_claude_opus_5_medium","benchmark":"deepswe","harness":"mini-swe-agent","model":"Claude Opus 5","effort":"medium","score":0.68904,"score_published":"69%","ci95":[0.64467,0.73019],"cost_per_attempt":3.28979,"cost_basis":"leaderboard mean_cost_usd per attempt over 447 attempts","tasks":113,"attempts":447,"k":4,"date":"2026-09-22","run":"Datacurve DeepSWE v1.1 leaderboard (mini-swe-agent), 113 tasks x 4 runs; 308/447 passed. Leaderboard CI (run-to-run) [0.677, 0.701]; ci95 here is Wilson over attempts.","source_kind":"leaderboard","sources":[{"label":"Trial index (JSON)","url":"https://deepswe.datacurve.ai/artifacts/v1.1/trials.json"},{"label":"DeepSWE v1.1 leaderboard-live.json","url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json"},{"label":"DeepSWE v1.1 blog","url":"https://deepswe.datacurve.ai/blog/deepswe-v1-1"}],"run_ref":"Config mini_swe_agent_claude_opus_5_medium","cost_estimated":false},{"id":"deepswe-mini_swe_agent_claude_fable_5_high","benchmark":"deepswe","harness":"mini-swe-agent","model":"Claude Fable 5","effort":"high","score":0.68605,"score_published":"69%","ci95":[0.6407,0.7281],"cost_per_attempt":9.17764,"cost_basis":"leaderboard mean_cost_usd per attempt over 430 attempts","tasks":113,"attempts":430,"k":4,"date":"2026-09-22","run":"Datacurve DeepSWE v1.1 leaderboard (mini-swe-agent), 113 tasks x 4 runs; 295/430 passed. Leaderboard CI (run-to-run) [0.675, 0.697]; ci95 here is Wilson over attempts.","source_kind":"leaderboard","sources":[{"label":"Trial index (JSON)","url":"https://deepswe.datacurve.ai/artifacts/v1.1/trials.json"},{"label":"DeepSWE v1.1 leaderboard-live.json","url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json"},{"label":"DeepSWE v1.1 blog","url":"https://deepswe.datacurve.ai/blog/deepswe-v1-1"}],"run_ref":"Config mini_swe_agent_claude_fable_5_high","cost_estimated":false},{"id":"deepswe-mini_swe_agent_kimi_k3_max","benchmark":"deepswe","harness":"mini-swe-agent","model":"Kimi K3","effort":"max","score":0.68514,"score_published":"69%","ci95":[0.64087,0.72629],"cost_per_attempt":4.65468,"cost_basis":"leaderboard mean_cost_usd per attempt over 451 attempts","tasks":113,"attempts":451,"k":4,"date":"2026-09-22","run":"Datacurve DeepSWE v1.1 leaderboard (mini-swe-agent), 113 tasks x 4 runs; 309/451 passed. Leaderboard CI (run-to-run) [0.640, 0.731]; ci95 here is Wilson over attempts.","source_kind":"leaderboard","sources":[{"label":"Trial index (JSON)","url":"https://deepswe.datacurve.ai/artifacts/v1.1/trials.json"},{"label":"DeepSWE v1.1 leaderboard-live.json","url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json"},{"label":"DeepSWE v1.1 blog","url":"https://deepswe.datacurve.ai/blog/deepswe-v1-1"}],"run_ref":"Config mini_swe_agent_kimi_k3_max","cost_estimated":false},{"id":"aa-ab5074e4-deepswe","benchmark":"deepswe","harness":"Claude Code","model":"Claude Opus 5.5","effort":"max","score":0.68437,"score_published":"68.4%","ci95":null,"cost_per_attempt":14.60076,"cost_basis":"ESTIMATE: AA publishes only the row's mean cost, $13.04 per attempt; allocated 41.8% to this benchmark by attempts x (input x (1 - 0.9 x cacheHitRate 0.95) + 5 x output tokens), as a share of that mean","cost_estimated":true,"tasks":113,"attempts":339,"k":3,"date":null,"run":"Artificial Analysis per-benchmark mean reward for 'Claude Code - Opus 5.5 (max)', 339 attempts. No run date published.","source_kind":"aa","sources":[{"label":"Artificial Analysis Coding Agents (no per-run page)","url":"https://artificialanalysis.ai/agents/coding-agents"}]},{"id":"aa-55b029d6-deepswe","benchmark":"deepswe","harness":"Kimi Code CLI","model":"Kimi K3","effort":null,"score":0.68437,"score_published":"68.4%","ci95":null,"cost_per_attempt":4.85502,"cost_basis":"ESTIMATE: AA publishes only the row's mean cost, $5.05 per attempt; allocated 35.9% to this benchmark by attempts x (input x (1 - 0.9 x cacheHitRate 0.95) + 5 x output tokens), as a share of that mean","cost_estimated":true,"tasks":113,"attempts":339,"k":3,"date":null,"run":"Artificial Analysis per-benchmark mean reward for 'Kimi Code CLI - Kimi K3', 339 attempts. No run date published.","source_kind":"aa","sources":[{"label":"Artificial Analysis Coding Agents (no per-run page)","url":"https://artificialanalysis.ai/agents/coding-agents"}]},{"id":"aa-666605c6-deepswe","benchmark":"deepswe","harness":"Claude Code","model":"Claude Sonnet 5.5","effort":"xhigh","score":0.68437,"score_published":"68.4%","ci95":null,"cost_per_attempt":3.7239,"cost_basis":"ESTIMATE: AA publishes only the row's mean cost, $3.33 per attempt; allocated 41.7% to this benchmark by attempts x (input x (1 - 0.9 x cacheHitRate 0.96) + 5 x output tokens), as a share of that mean","cost_estimated":true,"tasks":113,"attempts":339,"k":3,"date":null,"run":"Artificial Analysis per-benchmark mean reward for 'Claude Code - Sonnet 5.5 (xhigh)', 339 attempts. No run date published.","source_kind":"aa","sources":[{"label":"Artificial Analysis Coding Agents (no per-run page)","url":"https://artificialanalysis.ai/agents/coding-agents"}]},{"id":"aa-9e810ff3-deepswe","benchmark":"deepswe","harness":"Codex","model":"GPT-6 Astra","effort":"max","score":0.67552,"score_published":"67.6%","ci95":null,"cost_per_attempt":7.98565,"cost_basis":"ESTIMATE: AA publishes only the row's mean cost, $7.47 per attempt; allocated 39.9% to this benchmark by attempts x (input x (1 - 0.9 x cacheHitRate 0.93) + 5 x output tokens), as a share of that mean","cost_estimated":true,"tasks":113,"attempts":339,"k":3,"date":null,"run":"Artificial Analysis per-benchmark mean reward for 'Codex - GPT-6 Astra (max)', 339 attempts. No run date published.","source_kind":"aa","sources":[{"label":"Artificial Analysis Coding Agents (no per-run page)","url":"https://artificialanalysis.ai/agents/coding-agents"}]},{"id":"aa-fa589dad-deepswe","benchmark":"deepswe","harness":"Codex","model":"GPT-6.1 Sol","effort":"low","score":0.67552,"score_published":"67.6%","ci95":null,"cost_per_attempt":0.35353,"cost_basis":"ESTIMATE: AA publishes only the row's mean cost, $0.50 per attempt; allocated 26.4% to this benchmark by attempts x (input x (1 - 0.9 x cacheHitRate 0.91) + 5 x output tokens), as a share of that mean","cost_estimated":true,"tasks":113,"attempts":339,"k":3,"date":null,"run":"Artificial Analysis per-benchmark mean reward for 'Codex - GPT-6.1 Sol (low)', 339 attempts. No run date published.","source_kind":"aa","sources":[{"label":"Artificial Analysis Coding Agents (no per-run page)","url":"https://artificialanalysis.ai/agents/coding-agents"}]},{"id":"deepswe-mini_swe_agent_grok_4_6_medium","benchmark":"deepswe","harness":"mini-swe-agent","model":"Grok 4.6","effort":"medium","score":0.67478,"score_published":"67%","ci95":[0.63028,0.71634],"cost_per_attempt":3.44898,"cost_basis":"leaderboard mean_cost_usd per attempt over 452 attempts","tasks":113,"attempts":452,"k":4,"date":"2026-09-22","run":"Datacurve DeepSWE v1.1 leaderboard (mini-swe-agent), 113 tasks x 4 runs; 305/452 passed. Leaderboard CI (run-to-run) [0.652, 0.698]; ci95 here is Wilson over attempts.","source_kind":"leaderboard","sources":[{"label":"Trial index (JSON)","url":"https://deepswe.datacurve.ai/artifacts/v1.1/trials.json"},{"label":"DeepSWE v1.1 leaderboard-live.json","url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json"},{"label":"DeepSWE v1.1 blog","url":"https://deepswe.datacurve.ai/blog/deepswe-v1-1"}],"run_ref":"Config mini_swe_agent_grok_4_6_medium","cost_estimated":false},{"id":"aa-fa259509-deepswe","benchmark":"deepswe","harness":"Devin Fusion CLI","model":"GPT-6 Astra + SWE-2 Medium","effort":"xhigh","score":0.67257,"score_published":"67.3%","ci95":null,"cost_per_attempt":7.54838,"cost_basis":"ESTIMATE: AA publishes only the row's mean cost, $4.54 per attempt; allocated 62.1% to this benchmark by attempts x (input x (1 - 0.9 x cacheHitRate 0.92) + 5 x output tokens), as a share of that mean","cost_estimated":true,"tasks":113,"attempts":339,"k":3,"date":null,"run":"Artificial Analysis per-benchmark mean reward for 'Devin Fusion CLI - GPT-6 Astra XHigh + SWE-2 Medium', 339 attempts. No run date published.","source_kind":"aa","sources":[{"label":"Artificial Analysis Coding Agents (no per-run page)","url":"https://artificialanalysis.ai/agents/coding-agents"}]},{"id":"deepswe-mini_swe_agent_gpt_5_6_luna_max","benchmark":"deepswe","harness":"mini-swe-agent","model":"GPT-5.6 Luna","effort":"max","score":0.67188,"score_published":"67%","ci95":[0.6271,0.71373],"cost_per_attempt":3.02812,"cost_basis":"leaderboard mean_cost_usd per attempt over 448 attempts","tasks":113,"attempts":448,"k":4,"date":"2026-09-22","run":"Datacurve DeepSWE v1.1 leaderboard (mini-swe-agent), 113 tasks x 4 runs; 301/448 passed. Leaderboard CI (run-to-run) [0.632, 0.712]; ci95 here is Wilson over attempts.","source_kind":"leaderboard","sources":[{"label":"Trial index (JSON)","url":"https://deepswe.datacurve.ai/artifacts/v1.1/trials.json"},{"label":"DeepSWE v1.1 leaderboard-live.json","url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json"},{"label":"DeepSWE v1.1 blog","url":"https://deepswe.datacurve.ai/blog/deepswe-v1-1"}],"run_ref":"Config mini_swe_agent_gpt_5_6_luna_max","cost_estimated":false},{"id":"deepswe-mini_swe_agent_gpt_5_5_xhigh","benchmark":"deepswe","harness":"mini-swe-agent","model":"GPT-5.5","effort":"xhigh","score":0.67035,"score_published":"67%","ci95":[0.62574,0.7121],"cost_per_attempt":7.22624,"cost_basis":"leaderboard mean_cost_usd per attempt over 452 attempts","tasks":113,"attempts":452,"k":4,"date":"2026-09-22","run":"Datacurve DeepSWE v1.1 leaderboard (mini-swe-agent), 113 tasks x 4 runs; 303/452 passed. Leaderboard CI (run-to-run) [0.606, 0.735]; ci95 here is Wilson over attempts.","source_kind":"leaderboard","sources":[{"label":"Trial index (JSON)","url":"https://deepswe.datacurve.ai/artifacts/v1.1/trials.json"},{"label":"DeepSWE v1.1 leaderboard-live.json","url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json"},{"label":"DeepSWE v1.1 blog","url":"https://deepswe.datacurve.ai/blog/deepswe-v1-1"}],"run_ref":"Config mini_swe_agent_gpt_5_5_xhigh","cost_estimated":false},{"id":"deepswe-mini_swe_agent_gpt_6_astra_low","benchmark":"deepswe","harness":"mini-swe-agent","model":"GPT-6 Astra","effort":"low","score":0.67035,"score_published":"67%","ci95":[0.62574,0.7121],"cost_per_attempt":1.59522,"cost_basis":"leaderboard mean_cost_usd per attempt over 452 attempts; leaderboard note: Current pricing at all context lengths: $10/M uncached input, $12.50/M cache writes, $1/M cache reads, $50/M output; no separate compute-unit fee.","tasks":113,"attempts":452,"k":4,"date":"2026-09-22","run":"Datacurve DeepSWE v1.1 leaderboard (mini-swe-agent), 113 tasks x 4 runs; 303/452 passed. Leaderboard CI (run-to-run) [0.657, 0.683]; ci95 here is Wilson over attempts.","source_kind":"leaderboard","sources":[{"label":"Trial index (JSON)","url":"https://deepswe.datacurve.ai/artifacts/v1.1/trials.json"},{"label":"DeepSWE v1.1 leaderboard-live.json","url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json"},{"label":"DeepSWE v1.1 blog","url":"https://deepswe.datacurve.ai/blog/deepswe-v1-1"}],"run_ref":"Config mini_swe_agent_gpt_6_astra_low","cost_estimated":false},{"id":"deepswe-mini_swe_agent_grok_4_6_xhigh","benchmark":"deepswe","harness":"mini-swe-agent","model":"Grok 4.6","effort":"xhigh","score":0.66741,"score_published":"67%","ci95":[0.62267,0.70931],"cost_per_attempt":5.49767,"cost_basis":"leaderboard mean_cost_usd per attempt over 451 attempts","tasks":113,"attempts":451,"k":4,"date":"2026-09-22","run":"Datacurve DeepSWE v1.1 leaderboard (mini-swe-agent), 113 tasks x 4 runs; 301/451 passed. Leaderboard CI (run-to-run) [0.646, 0.689]; ci95 here is Wilson over attempts.","source_kind":"leaderboard","sources":[{"label":"Trial index (JSON)","url":"https://deepswe.datacurve.ai/artifacts/v1.1/trials.json"},{"label":"DeepSWE v1.1 leaderboard-live.json","url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json"},{"label":"DeepSWE v1.1 blog","url":"https://deepswe.datacurve.ai/blog/deepswe-v1-1"}],"run_ref":"Config mini_swe_agent_grok_4_6_xhigh","cost_estimated":false},{"id":"aa-33e80f38-deepswe","benchmark":"deepswe","harness":"Claude Code","model":"Claude Sonnet 5.5","effort":"high","score":0.66667,"score_published":"66.7%","ci95":null,"cost_per_attempt":1.28464,"cost_basis":"ESTIMATE: AA publishes only the row's mean cost, $1.24 per attempt; allocated 38.8% to this benchmark by attempts x (input x (1 - 0.9 x cacheHitRate 0.95) + 5 x output tokens), as a share of that mean","cost_estimated":true,"tasks":113,"attempts":339,"k":3,"date":null,"run":"Artificial Analysis per-benchmark mean reward for 'Claude Code - Sonnet 5.5 (high)', 339 attempts. No run date published.","source_kind":"aa","sources":[{"label":"Artificial Analysis Coding Agents (no per-run page)","url":"https://artificialanalysis.ai/agents/coding-agents"}]},{"id":"aa-b3c56ec6-deepswe","benchmark":"deepswe","harness":"Codex","model":"GPT-5.6 Luna","effort":"max","score":0.66372,"score_published":"66.4%","ci95":null,"cost_per_attempt":0.47756,"cost_basis":"ESTIMATE: AA publishes only the row's mean cost, $0.44 per attempt; allocated 40.7% to this benchmark by attempts x (input x (1 - 0.9 x cacheHitRate 0.97) + 5 x output tokens), as a share of that mean","cost_estimated":true,"tasks":113,"attempts":339,"k":3,"date":null,"run":"Artificial Analysis per-benchmark mean reward for 'Codex - GPT-5.6 Luna (max)', 339 attempts. No run date published.","source_kind":"aa","sources":[{"label":"Artificial Analysis Coding Agents (no per-run page)","url":"https://artificialanalysis.ai/agents/coding-agents"}]},{"id":"aa-d1003684-deepswe","benchmark":"deepswe","harness":"Antigravity SDK","model":"Gemini 3.8 Flash","effort":"high","score":0.65782,"score_published":"65.8%","ci95":null,"cost_per_attempt":3.04379,"cost_basis":"ESTIMATE: AA publishes only the row's mean cost, $2.47 per attempt; allocated 46.0% to this benchmark by attempts x (input x (1 - 0.9 x cacheHitRate 0.87) + 5 x output tokens), as a share of that mean","cost_estimated":true,"tasks":113,"attempts":339,"k":3,"date":null,"run":"Artificial Analysis per-benchmark mean reward for 'Antigravity SDK - Gemini 3.8 Flash (high)', 339 attempts. No run date published.","source_kind":"aa","sources":[{"label":"Artificial Analysis Coding Agents (no per-run page)","url":"https://artificialanalysis.ai/agents/coding-agents"}]},{"id":"deepswe-mini_swe_agent_gemini_3_7_flash_medium","benchmark":"deepswe","harness":"mini-swe-agent","model":"Gemini 3.7 Flash","effort":"medium","score":0.65487,"score_published":"65%","ci95":[0.6099,0.69722],"cost_per_attempt":2.02514,"cost_basis":"leaderboard mean_cost_usd per attempt over 452 attempts","tasks":113,"attempts":452,"k":4,"date":"2026-09-22","run":"Datacurve DeepSWE v1.1 leaderboard (mini-swe-agent), 113 tasks x 4 runs; 296/452 passed. Leaderboard CI (run-to-run) [0.624, 0.686]; ci95 here is Wilson over attempts.","source_kind":"leaderboard","sources":[{"label":"Trial index (JSON)","url":"https://deepswe.datacurve.ai/artifacts/v1.1/trials.json"},{"label":"DeepSWE v1.1 leaderboard-live.json","url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json"},{"label":"DeepSWE v1.1 blog","url":"https://deepswe.datacurve.ai/blog/deepswe-v1-1"}],"run_ref":"Config mini_swe_agent_gemini_3_7_flash_medium","cost_estimated":false},{"id":"aa-4d6aa9e9-deepswe","benchmark":"deepswe","harness":"Claude Code","model":"Claude Sonnet 5.5","effort":"medium","score":0.65487,"score_published":"65.5%","ci95":null,"cost_per_attempt":0.57765,"cost_basis":"ESTIMATE: AA publishes only the row's mean cost, $0.62 per attempt; allocated 34.8% to this benchmark by attempts x (input x (1 - 0.9 x cacheHitRate 0.93) + 5 x output tokens), as a share of that mean","cost_estimated":true,"tasks":113,"attempts":339,"k":3,"date":null,"run":"Artificial Analysis per-benchmark mean reward for 'Claude Code - Sonnet 5.5 (medium)', 339 attempts. No run date published.","source_kind":"aa","sources":[{"label":"Artificial Analysis Coding Agents (no per-run page)","url":"https://artificialanalysis.ai/agents/coding-agents"}]},{"id":"deepswe-mini_swe_agent_claude_fable_5_medium","benchmark":"deepswe","harness":"mini-swe-agent","model":"Claude Fable 5","effort":"medium","score":0.65367,"score_published":"65%","ci95":[0.60784,0.69681],"cost_per_attempt":6.08819,"cost_basis":"leaderboard mean_cost_usd per attempt over 436 attempts","tasks":113,"attempts":436,"k":4,"date":"2026-09-22","run":"Datacurve DeepSWE v1.1 leaderboard (mini-swe-agent), 113 tasks x 4 runs; 285/436 passed. Leaderboard CI (run-to-run) [0.609, 0.698]; ci95 here is Wilson over attempts.","source_kind":"leaderboard","sources":[{"label":"Trial index (JSON)","url":"https://deepswe.datacurve.ai/artifacts/v1.1/trials.json"},{"label":"DeepSWE v1.1 leaderboard-live.json","url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json"},{"label":"DeepSWE v1.1 blog","url":"https://deepswe.datacurve.ai/blog/deepswe-v1-1"}],"run_ref":"Config mini_swe_agent_claude_fable_5_medium","cost_estimated":false},{"id":"deepswe-mini_swe_agent_gemini_3_7_flash_high","benchmark":"deepswe","harness":"mini-swe-agent","model":"Gemini 3.7 Flash","effort":"high","score":0.65265,"score_published":"65%","ci95":[0.60764,0.6951],"cost_per_attempt":2.17629,"cost_basis":"leaderboard mean_cost_usd per attempt over 452 attempts","tasks":113,"attempts":452,"k":4,"date":"2026-09-22","run":"Datacurve DeepSWE v1.1 leaderboard (mini-swe-agent), 113 tasks x 4 runs; 295/452 passed. Leaderboard CI (run-to-run) [0.635, 0.671]; ci95 here is Wilson over attempts.","source_kind":"leaderboard","sources":[{"label":"Trial index (JSON)","url":"https://deepswe.datacurve.ai/artifacts/v1.1/trials.json"},{"label":"DeepSWE v1.1 leaderboard-live.json","url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json"},{"label":"DeepSWE v1.1 blog","url":"https://deepswe.datacurve.ai/blog/deepswe-v1-1"}],"run_ref":"Config mini_swe_agent_gemini_3_7_flash_high","cost_estimated":false},{"id":"deepswe-mini_swe_agent_grok_4_6_high","benchmark":"deepswe","harness":"mini-swe-agent","model":"Grok 4.6","effort":"high","score":0.65188,"score_published":"65%","ci95":[0.6068,0.6944],"cost_per_attempt":4.38487,"cost_basis":"leaderboard mean_cost_usd per attempt over 451 attempts","tasks":113,"attempts":451,"k":4,"date":"2026-09-22","run":"Datacurve DeepSWE v1.1 leaderboard (mini-swe-agent), 113 tasks x 4 runs; 294/451 passed. Leaderboard CI (run-to-run) [0.637, 0.667]; ci95 here is Wilson over attempts.","source_kind":"leaderboard","sources":[{"label":"Trial index (JSON)","url":"https://deepswe.datacurve.ai/artifacts/v1.1/trials.json"},{"label":"DeepSWE v1.1 leaderboard-live.json","url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json"},{"label":"DeepSWE v1.1 blog","url":"https://deepswe.datacurve.ai/blog/deepswe-v1-1"}],"run_ref":"Config mini_swe_agent_grok_4_6_high","cost_estimated":false},{"id":"aa-70a5d300-deepswe","benchmark":"deepswe","harness":"Grok Build","model":"Grok 4.6","effort":"xhigh","score":0.64897,"score_published":"64.9%","ci95":null,"cost_per_attempt":4.18415,"cost_basis":"ESTIMATE: AA publishes only the row's mean cost, $3.57 per attempt; allocated 43.7% to this benchmark by attempts x (input x (1 - 0.9 x cacheHitRate 0.91) + 5 x output tokens), as a share of that mean","cost_estimated":true,"tasks":113,"attempts":339,"k":3,"date":null,"run":"Artificial Analysis per-benchmark mean reward for 'Grok Build - Grok 4.6 (xhigh)', 339 attempts. No run date published.","source_kind":"aa","sources":[{"label":"Artificial Analysis Coding Agents (no per-run page)","url":"https://artificialanalysis.ai/agents/coding-agents"}]},{"id":"deepswe-mini_swe_agent_gpt_5_5_high","benchmark":"deepswe","harness":"mini-swe-agent","model":"GPT-5.5","effort":"high","score":0.64381,"score_published":"64%","ci95":[0.59862,0.68657],"cost_per_attempt":5.10044,"cost_basis":"leaderboard mean_cost_usd per attempt over 452 attempts","tasks":113,"attempts":452,"k":4,"date":"2026-09-22","run":"Datacurve DeepSWE v1.1 leaderboard (mini-swe-agent), 113 tasks x 4 runs; 291/452 passed. Leaderboard CI (run-to-run) [0.613, 0.675]; ci95 here is Wilson over attempts.","source_kind":"leaderboard","sources":[{"label":"Trial index (JSON)","url":"https://deepswe.datacurve.ai/artifacts/v1.1/trials.json"},{"label":"DeepSWE v1.1 leaderboard-live.json","url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json"},{"label":"DeepSWE v1.1 blog","url":"https://deepswe.datacurve.ai/blog/deepswe-v1-1"}],"run_ref":"Config mini_swe_agent_gpt_5_5_high","cost_estimated":false},{"id":"aa-29a1cdad-deepswe","benchmark":"deepswe","harness":"Claude Code","model":"Claude Fable 5.1","effort":"max","score":0.64307,"score_published":"64.3%","ci95":null,"cost_per_attempt":14.32948,"cost_basis":"ESTIMATE: AA publishes only the row's mean cost, $12.39 per attempt; allocated 43.1% to this benchmark by attempts x (input x (1 - 0.9 x cacheHitRate 0.92) + 5 x output tokens), as a share of that mean","cost_estimated":true,"tasks":113,"attempts":339,"k":3,"date":null,"run":"Artificial Analysis per-benchmark mean reward for 'Claude Code - Fable 5.1 (max) (with fallback)', 339 attempts. No run date published. Uses model fallback on refusals.","source_kind":"aa","sources":[{"label":"Artificial Analysis Coding Agents (no per-run page)","url":"https://artificialanalysis.ai/agents/coding-agents"}]},{"id":"aa-ac8276cd-deepswe","benchmark":"deepswe","harness":"Codex","model":"GPT-6 Luna","effort":"max","score":0.63717,"score_published":"63.7%","ci95":null,"cost_per_attempt":0.20966,"cost_basis":"ESTIMATE: AA publishes only the row's mean cost, $0.18 per attempt; allocated 44.4% to this benchmark by attempts x (input x (1 - 0.9 x cacheHitRate 0.96) + 5 x output tokens), as a share of that mean","cost_estimated":true,"tasks":113,"attempts":338,"k":3,"date":null,"run":"Artificial Analysis per-benchmark mean reward for 'Codex - GPT-6 Luna (max)', 338 attempts. No run date published.","source_kind":"aa","sources":[{"label":"Artificial Analysis Coding Agents (no per-run page)","url":"https://artificialanalysis.ai/agents/coding-agents"}]},{"id":"deepswe-mini_swe_agent_glm_5_3_flash_max","benchmark":"deepswe","harness":"mini-swe-agent","model":"GLM-5.3 Flash","effort":"max","score":0.63393,"score_published":"63%","ci95":[0.58836,0.67722],"cost_per_attempt":0.48196,"cost_basis":"leaderboard mean_cost_usd per attempt over 448 attempts","tasks":113,"attempts":448,"k":4,"date":"2026-09-22","run":"Datacurve DeepSWE v1.1 leaderboard (mini-swe-agent), 113 tasks x 4 runs; 284/448 passed. Leaderboard CI (run-to-run) [0.590, 0.678]; ci95 here is Wilson over attempts.","source_kind":"leaderboard","sources":[{"label":"Trial index (JSON)","url":"https://deepswe.datacurve.ai/artifacts/v1.1/trials.json"},{"label":"DeepSWE v1.1 leaderboard-live.json","url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json"},{"label":"DeepSWE v1.1 blog","url":"https://deepswe.datacurve.ai/blog/deepswe-v1-1"}],"run_ref":"Config mini_swe_agent_glm_5_3_flash_max","cost_estimated":false},{"id":"aa-34970c96-deepswe","benchmark":"deepswe","harness":"Devin Fusion CLI","model":"Claude Fable 5.1 + SWE-2 Medium","effort":"xhigh","score":0.63127,"score_published":"63.1%","ci95":null,"cost_per_attempt":8.42386,"cost_basis":"ESTIMATE: AA publishes only the row's mean cost, $7.90 per attempt; allocated 39.8% to this benchmark by attempts x (input x (1 - 0.9 x cacheHitRate 0.96) + 5 x output tokens), as a share of that mean","cost_estimated":true,"tasks":113,"attempts":339,"k":3,"date":null,"run":"Artificial Analysis per-benchmark mean reward for 'Devin Fusion CLI - Claude Fable 5.1 XHigh + SWE-2 Medium', 339 attempts. No run date published.","source_kind":"aa","sources":[{"label":"Artificial Analysis Coding Agents (no per-run page)","url":"https://artificialanalysis.ai/agents/coding-agents"}]},{"id":"deepswe-mini_swe_agent_deepseek_v4_pro_max","benchmark":"deepswe","harness":"mini-swe-agent","model":"DeepSeek V4 Pro","effort":"max","score":0.62832,"score_published":"63%","ci95":[0.58286,0.67161],"cost_per_attempt":0.24139,"cost_basis":"leaderboard mean_cost_usd per attempt over 452 attempts","tasks":113,"attempts":452,"k":4,"date":"2026-09-22","run":"Datacurve DeepSWE v1.1 leaderboard (mini-swe-agent), 113 tasks x 4 runs; 284/452 passed. Leaderboard CI (run-to-run) [0.565, 0.692]; ci95 here is Wilson over attempts.","source_kind":"leaderboard","sources":[{"label":"Trial index (JSON)","url":"https://deepswe.datacurve.ai/artifacts/v1.1/trials.json"},{"label":"DeepSWE v1.1 leaderboard-live.json","url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json"},{"label":"DeepSWE v1.1 blog","url":"https://deepswe.datacurve.ai/blog/deepswe-v1-1"}],"run_ref":"Config mini_swe_agent_deepseek_v4_pro_max","cost_estimated":false},{"id":"aa-40951881-deepswe","benchmark":"deepswe","harness":"Claude Code","model":"Claude Opus 5","effort":"max","score":0.62537,"score_published":"62.5%","ci95":null,"cost_per_attempt":11.99808,"cost_basis":"ESTIMATE: AA publishes only the row's mean cost, $10.79 per attempt; allocated 41.5% to this benchmark by attempts x (input x (1 - 0.9 x cacheHitRate 0.97) + 5 x output tokens), as a share of that mean","cost_estimated":true,"tasks":113,"attempts":339,"k":3,"date":null,"run":"Artificial Analysis per-benchmark mean reward for 'Claude Code - Opus 5 (max)', 339 attempts. No run date published.","source_kind":"aa","sources":[{"label":"Artificial Analysis Coding Agents (no per-run page)","url":"https://artificialanalysis.ai/agents/coding-agents"}]},{"id":"aa-e9f9d3ae-deepswe","benchmark":"deepswe","harness":"Claude Code","model":"Claude Sonnet 5.5","effort":"low","score":0.61947,"score_published":"61.9%","ci95":null,"cost_per_attempt":0.44279,"cost_basis":"ESTIMATE: AA publishes only the row's mean cost, $0.48 per attempt; allocated 34.2% to this benchmark by attempts x (input x (1 - 0.9 x cacheHitRate 0.93) + 5 x output tokens), as a share of that mean","cost_estimated":true,"tasks":113,"attempts":339,"k":3,"date":null,"run":"Artificial Analysis per-benchmark mean reward for 'Claude Code - Sonnet 5.5 (low)', 339 attempts. No run date published.","source_kind":"aa","sources":[{"label":"Artificial Analysis Coding Agents (no per-run page)","url":"https://artificialanalysis.ai/agents/coding-agents"}]},{"id":"aa-45493c27-deepswe","benchmark":"deepswe","harness":"Opencode","model":"GLM-5.3","effort":"max","score":0.61357,"score_published":"61.4%","ci95":null,"cost_per_attempt":4.85077,"cost_basis":"ESTIMATE: AA publishes only the row's mean cost, $4.24 per attempt; allocated 42.7% to this benchmark by attempts x (input x (1 - 0.9 x cacheHitRate 0.97) + 5 x output tokens), as a share of that mean","cost_estimated":true,"tasks":113,"attempts":339,"k":3,"date":null,"run":"Artificial Analysis per-benchmark mean reward for 'Opencode - GLM-5.3', 339 attempts. No run date published.","source_kind":"aa","sources":[{"label":"Artificial Analysis Coding Agents (no per-run page)","url":"https://artificialanalysis.ai/agents/coding-agents"}]},{"id":"deepswe-mini_swe_agent_gpt_5_6_sol_medium","benchmark":"deepswe","harness":"mini-swe-agent","model":"GPT-5.6 Sol","effort":"medium","score":0.61062,"score_published":"61%","ci95":[0.56492,0.65446],"cost_per_attempt":1.86203,"cost_basis":"leaderboard mean_cost_usd per attempt over 452 attempts","tasks":113,"attempts":452,"k":4,"date":"2026-09-22","run":"Datacurve DeepSWE v1.1 leaderboard (mini-swe-agent), 113 tasks x 4 runs; 276/452 passed. Leaderboard CI (run-to-run) [0.595, 0.626]; ci95 here is Wilson over attempts.","source_kind":"leaderboard","sources":[{"label":"Trial index (JSON)","url":"https://deepswe.datacurve.ai/artifacts/v1.1/trials.json"},{"label":"DeepSWE v1.1 leaderboard-live.json","url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json"},{"label":"DeepSWE v1.1 blog","url":"https://deepswe.datacurve.ai/blog/deepswe-v1-1"}],"run_ref":"Config mini_swe_agent_gpt_5_6_sol_medium","cost_estimated":false},{"id":"deepswe-mini_swe_agent_gpt_5_6_terra_xhigh","benchmark":"deepswe","harness":"mini-swe-agent","model":"GPT-5.6 Terra","effort":"xhigh","score":0.60177,"score_published":"60%","ci95":[0.55597,0.64586],"cost_per_attempt":2.12719,"cost_basis":"leaderboard mean_cost_usd per attempt over 452 attempts","tasks":113,"attempts":452,"k":4,"date":"2026-09-22","run":"Datacurve DeepSWE v1.1 leaderboard (mini-swe-agent), 113 tasks x 4 runs; 272/452 passed. Leaderboard CI (run-to-run) [0.581, 0.623]; ci95 here is Wilson over attempts.","source_kind":"leaderboard","sources":[{"label":"Trial index (JSON)","url":"https://deepswe.datacurve.ai/artifacts/v1.1/trials.json"},{"label":"DeepSWE v1.1 leaderboard-live.json","url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json"},{"label":"DeepSWE v1.1 blog","url":"https://deepswe.datacurve.ai/blog/deepswe-v1-1"}],"run_ref":"Config mini_swe_agent_gpt_5_6_terra_xhigh","cost_estimated":false},{"id":"deepswe-mini_swe_agent_claude_fable_5_low","benchmark":"deepswe","harness":"mini-swe-agent","model":"Claude Fable 5","effort":"low","score":0.59584,"score_published":"60%","ci95":[0.54897,0.64103],"cost_per_attempt":3.75787,"cost_basis":"leaderboard mean_cost_usd per attempt over 433 attempts","tasks":113,"attempts":433,"k":4,"date":"2026-09-22","run":"Datacurve DeepSWE v1.1 leaderboard (mini-swe-agent), 113 tasks x 4 runs; 258/433 passed. Leaderboard CI (run-to-run) [0.568, 0.624]; ci95 here is Wilson over attempts.","source_kind":"leaderboard","sources":[{"label":"Trial index (JSON)","url":"https://deepswe.datacurve.ai/artifacts/v1.1/trials.json"},{"label":"DeepSWE v1.1 leaderboard-live.json","url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json"},{"label":"DeepSWE v1.1 blog","url":"https://deepswe.datacurve.ai/blog/deepswe-v1-1"}],"run_ref":"Config mini_swe_agent_claude_fable_5_low","cost_estimated":false},{"id":"deepswe-mini_swe_agent_claude_opus_4_8_max","benchmark":"deepswe","harness":"mini-swe-agent","model":"Claude Opus 4.8","effort":"max","score":0.58974,"score_published":"59%","ci95":[0.5426,0.63529],"cost_per_attempt":13.22258,"cost_basis":"leaderboard mean_cost_usd per attempt over 429 attempts","tasks":111,"attempts":429,"k":4,"date":"2026-09-22","run":"Datacurve DeepSWE v1.1 leaderboard (mini-swe-agent), 111 tasks x 4 runs; 253/429 passed. Leaderboard CI (run-to-run) [0.572, 0.607]; ci95 here is Wilson over attempts.","source_kind":"leaderboard","sources":[{"label":"Trial index (JSON)","url":"https://deepswe.datacurve.ai/artifacts/v1.1/trials.json"},{"label":"DeepSWE v1.1 leaderboard-live.json","url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json"},{"label":"DeepSWE v1.1 blog","url":"https://deepswe.datacurve.ai/blog/deepswe-v1-1"}],"run_ref":"Config mini_swe_agent_claude_opus_4_8_max","cost_estimated":false},{"id":"deepswe-mini_swe_agent_claude_opus_5_low","benchmark":"deepswe","harness":"mini-swe-agent","model":"Claude Opus 5","effort":"low","score":0.58129,"score_published":"58%","ci95":[0.53516,0.62605],"cost_per_attempt":1.66262,"cost_basis":"leaderboard mean_cost_usd per attempt over 449 attempts","tasks":113,"attempts":449,"k":4,"date":"2026-09-22","run":"Datacurve DeepSWE v1.1 leaderboard (mini-swe-agent), 113 tasks x 4 runs; 261/449 passed. Leaderboard CI (run-to-run) [0.558, 0.605]; ci95 here is Wilson over attempts.","source_kind":"leaderboard","sources":[{"label":"Trial index (JSON)","url":"https://deepswe.datacurve.ai/artifacts/v1.1/trials.json"},{"label":"DeepSWE v1.1 leaderboard-live.json","url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json"},{"label":"DeepSWE v1.1 blog","url":"https://deepswe.datacurve.ai/blog/deepswe-v1-1"}],"run_ref":"Config mini_swe_agent_claude_opus_5_low","cost_estimated":false},{"id":"deepswe-mini_swe_agent_qwen3_8_max_xhigh","benchmark":"deepswe","harness":"mini-swe-agent","model":"Qwen3.8 Max","effort":"xhigh","score":0.57461,"score_published":"57%","ci95":[0.52844,0.61952],"cost_per_attempt":3.72906,"cost_basis":"leaderboard mean_cost_usd per attempt over 449 attempts","tasks":113,"attempts":449,"k":4,"date":"2026-09-22","run":"Datacurve DeepSWE v1.1 leaderboard (mini-swe-agent), 113 tasks x 4 runs; 258/449 passed. Leaderboard CI (run-to-run) [0.548, 0.601]; ci95 here is Wilson over attempts.","source_kind":"leaderboard","sources":[{"label":"Trial index (JSON)","url":"https://deepswe.datacurve.ai/artifacts/v1.1/trials.json"},{"label":"DeepSWE v1.1 leaderboard-live.json","url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json"},{"label":"DeepSWE v1.1 blog","url":"https://deepswe.datacurve.ai/blog/deepswe-v1-1"}],"run_ref":"Config mini_swe_agent_qwen3_8_max_xhigh","cost_estimated":false},{"id":"aa-542590ec-deepswe","benchmark":"deepswe","harness":"Codex","model":"DeepSeek V4 Pro 0813","effort":"max","score":0.57227,"score_published":"57.2%","ci95":null,"cost_per_attempt":0.22026,"cost_basis":"ESTIMATE: AA publishes only the row's mean cost, $0.24 per attempt; allocated 34.5% to this benchmark by attempts x (input x (1 - 0.9 x cacheHitRate 0.99) + 5 x output tokens), as a share of that mean","cost_estimated":true,"tasks":113,"attempts":339,"k":3,"date":null,"run":"Artificial Analysis per-benchmark mean reward for 'Codex - DeepSeek V4 Pro 0813 (max)', 339 attempts. No run date published.","source_kind":"aa","sources":[{"label":"Artificial Analysis Coding Agents (no per-run page)","url":"https://artificialanalysis.ai/agents/coding-agents"}]},{"id":"deepswe-mini_swe_agent_gpt_5_6_luna_xhigh","benchmark":"deepswe","harness":"mini-swe-agent","model":"GPT-5.6 Luna","effort":"xhigh","score":0.56858,"score_published":"57%","ci95":[0.52254,0.61348],"cost_per_attempt":1.53562,"cost_basis":"leaderboard mean_cost_usd per attempt over 452 attempts","tasks":113,"attempts":452,"k":4,"date":"2026-09-22","run":"Datacurve DeepSWE v1.1 leaderboard (mini-swe-agent), 113 tasks x 4 runs; 257/452 passed. Leaderboard CI (run-to-run) [0.547, 0.590]; ci95 here is Wilson over attempts.","source_kind":"leaderboard","sources":[{"label":"Trial index (JSON)","url":"https://deepswe.datacurve.ai/artifacts/v1.1/trials.json"},{"label":"DeepSWE v1.1 leaderboard-live.json","url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json"},{"label":"DeepSWE v1.1 blog","url":"https://deepswe.datacurve.ai/blog/deepswe-v1-1"}],"run_ref":"Config mini_swe_agent_gpt_5_6_luna_xhigh","cost_estimated":false},{"id":"deepswe-mini_swe_agent_muse_spark_1_2_xhigh","benchmark":"deepswe","harness":"mini-swe-agent","model":"Muse Spark 1.2","effort":"xhigh","score":0.54867,"score_published":"55%","ci95":[0.50258,0.59395],"cost_per_attempt":3.6955,"cost_basis":"leaderboard mean_cost_usd per attempt over 452 attempts","tasks":113,"attempts":452,"k":4,"date":"2026-09-22","run":"Datacurve DeepSWE v1.1 leaderboard (mini-swe-agent), 113 tasks x 4 runs; 248/452 passed. Leaderboard CI (run-to-run) [0.527, 0.570]; ci95 here is Wilson over attempts.","source_kind":"leaderboard","sources":[{"label":"Trial index (JSON)","url":"https://deepswe.datacurve.ai/artifacts/v1.1/trials.json"},{"label":"DeepSWE v1.1 leaderboard-live.json","url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json"},{"label":"DeepSWE v1.1 blog","url":"https://deepswe.datacurve.ai/blog/deepswe-v1-1"}],"run_ref":"Config mini_swe_agent_muse_spark_1_2_xhigh","cost_estimated":false},{"id":"deepswe-mini_swe_agent_claude_opus_4_8_xhigh","benchmark":"deepswe","harness":"mini-swe-agent","model":"Claude Opus 4.8","effort":"xhigh","score":0.54362,"score_published":"54%","ci95":[0.49727,0.58923],"cost_per_attempt":8.00636,"cost_basis":"leaderboard mean_cost_usd per attempt over 447 attempts","tasks":113,"attempts":447,"k":4,"date":"2026-09-22","run":"Datacurve DeepSWE v1.1 leaderboard (mini-swe-agent), 113 tasks x 4 runs; 243/447 passed. Leaderboard CI (run-to-run) [0.506, 0.581]; ci95 here is Wilson over attempts.","source_kind":"leaderboard","sources":[{"label":"Trial index (JSON)","url":"https://deepswe.datacurve.ai/artifacts/v1.1/trials.json"},{"label":"DeepSWE v1.1 leaderboard-live.json","url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json"},{"label":"DeepSWE v1.1 blog","url":"https://deepswe.datacurve.ai/blog/deepswe-v1-1"}],"run_ref":"Config mini_swe_agent_claude_opus_4_8_xhigh","cost_estimated":false},{"id":"aa-28ef6513-deepswe","benchmark":"deepswe","harness":"Codex","model":"DeepSeek V4 Flash 0731","effort":"max","score":0.54277,"score_published":"54.3%","ci95":null,"cost_per_attempt":0.10391,"cost_basis":"ESTIMATE: AA publishes only the row's mean cost, $0.09 per attempt; allocated 45.5% to this benchmark by attempts x (input x (1 - 0.9 x cacheHitRate 0.99) + 5 x output tokens), as a share of that mean","cost_estimated":true,"tasks":113,"attempts":339,"k":3,"date":null,"run":"Artificial Analysis per-benchmark mean reward for 'Codex - DeepSeek V4 Flash 0731 (max)', 339 attempts. No run date published.","source_kind":"aa","sources":[{"label":"Artificial Analysis Coding Agents (no per-run page)","url":"https://artificialanalysis.ai/agents/coding-agents"}]},{"id":"deepswe-mini_swe_agent_gpt_5_5_medium","benchmark":"deepswe","harness":"mini-swe-agent","model":"GPT-5.5","effort":"medium","score":0.53982,"score_published":"54%","ci95":[0.49373,0.58524],"cost_per_attempt":2.74934,"cost_basis":"leaderboard mean_cost_usd per attempt over 452 attempts","tasks":113,"attempts":452,"k":4,"date":"2026-09-22","run":"Datacurve DeepSWE v1.1 leaderboard (mini-swe-agent), 113 tasks x 4 runs; 244/452 passed. Leaderboard CI (run-to-run) [0.514, 0.565]; ci95 here is Wilson over attempts.","source_kind":"leaderboard","sources":[{"label":"Trial index (JSON)","url":"https://deepswe.datacurve.ai/artifacts/v1.1/trials.json"},{"label":"DeepSWE v1.1 leaderboard-live.json","url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json"},{"label":"DeepSWE v1.1 blog","url":"https://deepswe.datacurve.ai/blog/deepswe-v1-1"}],"run_ref":"Config mini_swe_agent_gpt_5_5_medium","cost_estimated":false},{"id":"deepswe-mini_swe_agent_claude_sonnet_5_max","benchmark":"deepswe","harness":"mini-swe-agent","model":"Claude Sonnet 5","effort":"max","score":0.53846,"score_published":"54%","ci95":[0.49185,0.58441],"cost_per_attempt":26.39986,"cost_basis":"leaderboard mean_cost_usd per attempt over 442 attempts","tasks":113,"attempts":442,"k":4,"date":"2026-09-22","run":"Datacurve DeepSWE v1.1 leaderboard (mini-swe-agent), 113 tasks x 4 runs; 238/442 passed. Leaderboard CI (run-to-run) [0.496, 0.581]; ci95 here is Wilson over attempts.","source_kind":"leaderboard","sources":[{"label":"Trial index (JSON)","url":"https://deepswe.datacurve.ai/artifacts/v1.1/trials.json"},{"label":"DeepSWE v1.1 leaderboard-live.json","url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json"},{"label":"DeepSWE v1.1 blog","url":"https://deepswe.datacurve.ai/blog/deepswe-v1-1"}],"run_ref":"Config mini_swe_agent_claude_sonnet_5_max","cost_estimated":false},{"id":"deepswe-mini_swe_agent_gemini_3_7_flash_low","benchmark":"deepswe","harness":"mini-swe-agent","model":"Gemini 3.7 Flash","effort":"low","score":0.53761,"score_published":"54%","ci95":[0.49152,0.58306],"cost_per_attempt":1.83226,"cost_basis":"leaderboard mean_cost_usd per attempt over 452 attempts","tasks":113,"attempts":452,"k":4,"date":"2026-09-22","run":"Datacurve DeepSWE v1.1 leaderboard (mini-swe-agent), 113 tasks x 4 runs; 243/452 passed. Leaderboard CI (run-to-run) [0.512, 0.564]; ci95 here is Wilson over attempts.","source_kind":"leaderboard","sources":[{"label":"Trial index (JSON)","url":"https://deepswe.datacurve.ai/artifacts/v1.1/trials.json"},{"label":"DeepSWE v1.1 leaderboard-live.json","url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json"},{"label":"DeepSWE v1.1 blog","url":"https://deepswe.datacurve.ai/blog/deepswe-v1-1"}],"run_ref":"Config mini_swe_agent_gemini_3_7_flash_low","cost_estimated":false},{"id":"deepswe-mini_swe_agent_gpt_5_6_terra_high","benchmark":"deepswe","harness":"mini-swe-agent","model":"GPT-5.6 Terra","effort":"high","score":0.53761,"score_published":"54%","ci95":[0.49152,0.58306],"cost_per_attempt":1.13438,"cost_basis":"leaderboard mean_cost_usd per attempt over 452 attempts","tasks":113,"attempts":452,"k":4,"date":"2026-09-22","run":"Datacurve DeepSWE v1.1 leaderboard (mini-swe-agent), 113 tasks x 4 runs; 243/452 passed. Leaderboard CI (run-to-run) [0.494, 0.581]; ci95 here is Wilson over attempts.","source_kind":"leaderboard","sources":[{"label":"Trial index (JSON)","url":"https://deepswe.datacurve.ai/artifacts/v1.1/trials.json"},{"label":"DeepSWE v1.1 leaderboard-live.json","url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json"},{"label":"DeepSWE v1.1 blog","url":"https://deepswe.datacurve.ai/blog/deepswe-v1-1"}],"run_ref":"Config mini_swe_agent_gpt_5_6_terra_high","cost_estimated":false},{"id":"deepswe-mini_swe_agent_grok_4_5_high","benchmark":"deepswe","harness":"mini-swe-agent","model":"Grok 4.5","effort":"high","score":0.53761,"score_published":"54%","ci95":[0.49152,0.58306],"cost_per_attempt":2.41575,"cost_basis":"leaderboard mean_cost_usd per attempt over 452 attempts","tasks":113,"attempts":452,"k":4,"date":"2026-09-22","run":"Datacurve DeepSWE v1.1 leaderboard (mini-swe-agent), 113 tasks x 4 runs; 243/452 passed. Leaderboard CI (run-to-run) [0.515, 0.560]; ci95 here is Wilson over attempts.","source_kind":"leaderboard","sources":[{"label":"Trial index (JSON)","url":"https://deepswe.datacurve.ai/artifacts/v1.1/trials.json"},{"label":"DeepSWE v1.1 leaderboard-live.json","url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json"},{"label":"DeepSWE v1.1 blog","url":"https://deepswe.datacurve.ai/blog/deepswe-v1-1"}],"run_ref":"Config mini_swe_agent_grok_4_5_high","cost_estimated":false},{"id":"deepswe-mini_swe_agent_deepseek_v4_flash_max","benchmark":"deepswe","harness":"mini-swe-agent","model":"DeepSeek V4 Flash","effort":"max","score":0.53319,"score_published":"53%","ci95":[0.48711,0.57871],"cost_per_attempt":0.10024,"cost_basis":"leaderboard mean_cost_usd per attempt over 452 attempts","tasks":113,"attempts":452,"k":4,"date":"2026-09-22","run":"Datacurve DeepSWE v1.1 leaderboard (mini-swe-agent), 113 tasks x 4 runs; 241/452 passed. Leaderboard CI (run-to-run) [0.498, 0.569]; ci95 here is Wilson over attempts.","source_kind":"leaderboard","sources":[{"label":"Trial index (JSON)","url":"https://deepswe.datacurve.ai/artifacts/v1.1/trials.json"},{"label":"DeepSWE v1.1 leaderboard-live.json","url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json"},{"label":"DeepSWE v1.1 blog","url":"https://deepswe.datacurve.ai/blog/deepswe-v1-1"}],"run_ref":"Config mini_swe_agent_deepseek_v4_flash_max","cost_estimated":false},{"id":"deepswe-mini_swe_agent_muse_spark_1_1_xhigh","benchmark":"deepswe","harness":"mini-swe-agent","model":"Muse Spark 1.1","effort":"xhigh","score":0.53319,"score_published":"53%","ci95":[0.48711,0.57871],"cost_per_attempt":2.36114,"cost_basis":"leaderboard mean_cost_usd per attempt over 452 attempts","tasks":113,"attempts":452,"k":4,"date":"2026-09-22","run":"Datacurve DeepSWE v1.1 leaderboard (mini-swe-agent), 113 tasks x 4 runs; 241/452 passed. Leaderboard CI (run-to-run) [0.503, 0.564]; ci95 here is Wilson over attempts.","source_kind":"leaderboard","sources":[{"label":"Trial index (JSON)","url":"https://deepswe.datacurve.ai/artifacts/v1.1/trials.json"},{"label":"DeepSWE v1.1 leaderboard-live.json","url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json"},{"label":"DeepSWE v1.1 blog","url":"https://deepswe.datacurve.ai/blog/deepswe-v1-1"}],"run_ref":"Config mini_swe_agent_muse_spark_1_1_xhigh","cost_estimated":false},{"id":"deepswe-mini_swe_agent_claude_opus_4_8_high","benchmark":"deepswe","harness":"mini-swe-agent","model":"Claude Opus 4.8","effort":"high","score":0.5177,"score_published":"52%","ci95":[0.47168,0.56342],"cost_per_attempt":4.28194,"cost_basis":"leaderboard mean_cost_usd per attempt over 452 attempts","tasks":113,"attempts":452,"k":4,"date":"2026-09-22","run":"Datacurve DeepSWE v1.1 leaderboard (mini-swe-agent), 113 tasks x 4 runs; 234/452 passed. Leaderboard CI (run-to-run) [0.472, 0.563]; ci95 here is Wilson over attempts.","source_kind":"leaderboard","sources":[{"label":"Trial index (JSON)","url":"https://deepswe.datacurve.ai/artifacts/v1.1/trials.json"},{"label":"DeepSWE v1.1 leaderboard-live.json","url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json"},{"label":"DeepSWE v1.1 blog","url":"https://deepswe.datacurve.ai/blog/deepswe-v1-1"}],"run_ref":"Config mini_swe_agent_claude_opus_4_8_high","cost_estimated":false},{"id":"deepswe-mini_swe_agent_gpt_5_4_xhigh","benchmark":"deepswe","harness":"mini-swe-agent","model":"GPT-5.4","effort":"xhigh","score":0.5177,"score_published":"52%","ci95":[0.47168,0.56342],"cost_per_attempt":5.65246,"cost_basis":"leaderboard mean_cost_usd per attempt over 452 attempts","tasks":113,"attempts":452,"k":4,"date":"2026-09-22","run":"Datacurve DeepSWE v1.1 leaderboard (mini-swe-agent), 113 tasks x 4 runs; 234/452 passed. Leaderboard CI (run-to-run) [0.503, 0.533]; ci95 here is Wilson over attempts.","source_kind":"leaderboard","sources":[{"label":"Trial index (JSON)","url":"https://deepswe.datacurve.ai/artifacts/v1.1/trials.json"},{"label":"DeepSWE v1.1 leaderboard-live.json","url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json"},{"label":"DeepSWE v1.1 blog","url":"https://deepswe.datacurve.ai/blog/deepswe-v1-1"}],"run_ref":"Config mini_swe_agent_gpt_5_4_xhigh","cost_estimated":false},{"id":"aa-cd4c865d-deepswe","benchmark":"deepswe","harness":"Claude Code","model":"Qwen3.8 Max","effort":null,"score":0.51032,"score_published":"51.0%","ci95":null,"cost_per_attempt":3.94372,"cost_basis":"ESTIMATE: AA publishes only the row's mean cost, $3.48 per attempt; allocated 42.3% to this benchmark by attempts x (input x (1 - 0.9 x cacheHitRate 0.95) + 5 x output tokens), as a share of that mean","cost_estimated":true,"tasks":113,"attempts":339,"k":3,"date":null,"run":"Artificial Analysis per-benchmark mean reward for 'Claude Code - Qwen3.8 Max', 339 attempts. No run date published.","source_kind":"aa","sources":[{"label":"Artificial Analysis Coding Agents (no per-run page)","url":"https://artificialanalysis.ai/agents/coding-agents"}]},{"id":"deepswe-mini_swe_agent_claude_sonnet_5_xhigh","benchmark":"deepswe","harness":"mini-swe-agent","model":"Claude Sonnet 5","effort":"xhigh","score":0.49667,"score_published":"50%","ci95":[0.45075,0.54265],"cost_per_attempt":11.89063,"cost_basis":"leaderboard mean_cost_usd per attempt over 451 attempts","tasks":113,"attempts":451,"k":4,"date":"2026-09-22","run":"Datacurve DeepSWE v1.1 leaderboard (mini-swe-agent), 113 tasks x 4 runs; 224/451 passed. Leaderboard CI (run-to-run) [0.462, 0.531]; ci95 here is Wilson over attempts.","source_kind":"leaderboard","sources":[{"label":"Trial index (JSON)","url":"https://deepswe.datacurve.ai/artifacts/v1.1/trials.json"},{"label":"DeepSWE v1.1 leaderboard-live.json","url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json"},{"label":"DeepSWE v1.1 blog","url":"https://deepswe.datacurve.ai/blog/deepswe-v1-1"}],"run_ref":"Config mini_swe_agent_claude_sonnet_5_xhigh","cost_estimated":false},{"id":"deepswe-mini_swe_agent_claude_opus_4_8_medium","benchmark":"deepswe","harness":"mini-swe-agent","model":"Claude Opus 4.8","effort":"medium","score":0.48673,"score_published":"49%","ci95":[0.44095,0.53272],"cost_per_attempt":3.44386,"cost_basis":"leaderboard mean_cost_usd per attempt over 452 attempts","tasks":113,"attempts":452,"k":4,"date":"2026-09-22","run":"Datacurve DeepSWE v1.1 leaderboard (mini-swe-agent), 113 tasks x 4 runs; 220/452 passed. Leaderboard CI (run-to-run) [0.464, 0.509]; ci95 here is Wilson over attempts.","source_kind":"leaderboard","sources":[{"label":"Trial index (JSON)","url":"https://deepswe.datacurve.ai/artifacts/v1.1/trials.json"},{"label":"DeepSWE v1.1 leaderboard-live.json","url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json"},{"label":"DeepSWE v1.1 blog","url":"https://deepswe.datacurve.ai/blog/deepswe-v1-1"}],"run_ref":"Config mini_swe_agent_claude_opus_4_8_medium","cost_estimated":false},{"id":"deepswe-mini_swe_agent_claude_sonnet_5_high","benchmark":"deepswe","harness":"mini-swe-agent","model":"Claude Sonnet 5","effort":"high","score":0.4823,"score_published":"48%","ci95":[0.43658,0.52832],"cost_per_attempt":7.42556,"cost_basis":"leaderboard mean_cost_usd per attempt over 452 attempts","tasks":113,"attempts":452,"k":4,"date":"2026-09-22","run":"Datacurve DeepSWE v1.1 leaderboard (mini-swe-agent), 113 tasks x 4 runs; 218/452 passed. Leaderboard CI (run-to-run) [0.437, 0.527]; ci95 here is Wilson over attempts.","source_kind":"leaderboard","sources":[{"label":"Trial index (JSON)","url":"https://deepswe.datacurve.ai/artifacts/v1.1/trials.json"},{"label":"DeepSWE v1.1 leaderboard-live.json","url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json"},{"label":"DeepSWE v1.1 blog","url":"https://deepswe.datacurve.ai/blog/deepswe-v1-1"}],"run_ref":"Config mini_swe_agent_claude_sonnet_5_high","cost_estimated":false},{"id":"deepswe-mini_swe_agent_gemini_3_6_flash_high","benchmark":"deepswe","harness":"mini-swe-agent","model":"Gemini 3.6 Flash","effort":"high","score":0.46681,"score_published":"47%","ci95":[0.42129,0.51289],"cost_per_attempt":4.41901,"cost_basis":"leaderboard mean_cost_usd per attempt over 452 attempts","tasks":113,"attempts":452,"k":4,"date":"2026-09-22","run":"Datacurve DeepSWE v1.1 leaderboard (mini-swe-agent), 113 tasks x 4 runs; 211/452 passed. Leaderboard CI (run-to-run) [0.430, 0.504]; ci95 here is Wilson over attempts.","source_kind":"leaderboard","sources":[{"label":"Trial index (JSON)","url":"https://deepswe.datacurve.ai/artifacts/v1.1/trials.json"},{"label":"DeepSWE v1.1 leaderboard-live.json","url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json"},{"label":"DeepSWE v1.1 blog","url":"https://deepswe.datacurve.ai/blog/deepswe-v1-1"}],"run_ref":"Config mini_swe_agent_gemini_3_6_flash_high","cost_estimated":false},{"id":"deepswe-mini_swe_agent_gpt_5_6_sol_low","benchmark":"deepswe","harness":"mini-swe-agent","model":"GPT-5.6 Sol","effort":"low","score":0.45354,"score_published":"45%","ci95":[0.40823,0.49963],"cost_per_attempt":1.07427,"cost_basis":"leaderboard mean_cost_usd per attempt over 452 attempts","tasks":113,"attempts":452,"k":4,"date":"2026-09-22","run":"Datacurve DeepSWE v1.1 leaderboard (mini-swe-agent), 113 tasks x 4 runs; 205/452 passed. Leaderboard CI (run-to-run) [0.430, 0.477]; ci95 here is Wilson over attempts.","source_kind":"leaderboard","sources":[{"label":"Trial index (JSON)","url":"https://deepswe.datacurve.ai/artifacts/v1.1/trials.json"},{"label":"DeepSWE v1.1 leaderboard-live.json","url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json"},{"label":"DeepSWE v1.1 blog","url":"https://deepswe.datacurve.ai/blog/deepswe-v1-1"}],"run_ref":"Config mini_swe_agent_gpt_5_6_sol_low","cost_estimated":false},{"id":"deepswe-mini_swe_agent_gpt_5_6_luna_high","benchmark":"deepswe","harness":"mini-swe-agent","model":"GPT-5.6 Luna","effort":"high","score":0.44248,"score_published":"44%","ci95":[0.39736,0.48856],"cost_per_attempt":0.7779,"cost_basis":"leaderboard mean_cost_usd per attempt over 452 attempts","tasks":113,"attempts":452,"k":4,"date":"2026-09-22","run":"Datacurve DeepSWE v1.1 leaderboard (mini-swe-agent), 113 tasks x 4 runs; 200/452 passed. Leaderboard CI (run-to-run) [0.413, 0.472]; ci95 here is Wilson over attempts.","source_kind":"leaderboard","sources":[{"label":"Trial index (JSON)","url":"https://deepswe.datacurve.ai/artifacts/v1.1/trials.json"},{"label":"DeepSWE v1.1 leaderboard-live.json","url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json"},{"label":"DeepSWE v1.1 blog","url":"https://deepswe.datacurve.ai/blog/deepswe-v1-1"}],"run_ref":"Config mini_swe_agent_gpt_5_6_luna_high","cost_estimated":false},{"id":"deepswe-mini_swe_agent_glm_5_2_max","benchmark":"deepswe","harness":"mini-swe-agent","model":"GLM-5.2","effort":"max","score":0.43778,"score_published":"44%","ci95":[0.39266,0.48395],"cost_per_attempt":3.91991,"cost_basis":"leaderboard mean_cost_usd per attempt over 450 attempts","tasks":113,"attempts":450,"k":4,"date":"2026-09-22","run":"Datacurve DeepSWE v1.1 leaderboard (mini-swe-agent), 113 tasks x 4 runs; 197/450 passed. Leaderboard CI (run-to-run) [0.421, 0.455]; ci95 here is Wilson over attempts.","source_kind":"leaderboard","sources":[{"label":"Trial index (JSON)","url":"https://deepswe.datacurve.ai/artifacts/v1.1/trials.json"},{"label":"DeepSWE v1.1 leaderboard-live.json","url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json"},{"label":"DeepSWE v1.1 blog","url":"https://deepswe.datacurve.ai/blog/deepswe-v1-1"}],"run_ref":"Config mini_swe_agent_glm_5_2_max","cost_estimated":false},{"id":"deepswe-mini_swe_agent_grok_4_6_low","benchmark":"deepswe","harness":"mini-swe-agent","model":"Grok 4.6","effort":"low","score":0.41648,"score_published":"42%","ci95":[0.37178,0.4626],"cost_per_attempt":1.04236,"cost_basis":"leaderboard mean_cost_usd per attempt over 449 attempts","tasks":113,"attempts":449,"k":4,"date":"2026-09-22","run":"Datacurve DeepSWE v1.1 leaderboard (mini-swe-agent), 113 tasks x 4 runs; 187/449 passed. Leaderboard CI (run-to-run) [0.393, 0.440]; ci95 here is Wilson over attempts.","source_kind":"leaderboard","sources":[{"label":"Trial index (JSON)","url":"https://deepswe.datacurve.ai/artifacts/v1.1/trials.json"},{"label":"DeepSWE v1.1 leaderboard-live.json","url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json"},{"label":"DeepSWE v1.1 blog","url":"https://deepswe.datacurve.ai/blog/deepswe-v1-1"}],"run_ref":"Config mini_swe_agent_grok_4_6_low","cost_estimated":false},{"id":"deepswe-mini_swe_agent_claude_opus_4_8_low","benchmark":"deepswe","harness":"mini-swe-agent","model":"Claude Opus 4.8","effort":"low","score":0.40798,"score_published":"41%","ci95":[0.36359,0.45393],"cost_per_attempt":2.29337,"cost_basis":"leaderboard mean_cost_usd per attempt over 451 attempts","tasks":113,"attempts":451,"k":4,"date":"2026-09-22","run":"Datacurve DeepSWE v1.1 leaderboard (mini-swe-agent), 113 tasks x 4 runs; 184/451 passed. Leaderboard CI (run-to-run) [0.393, 0.423]; ci95 here is Wilson over attempts.","source_kind":"leaderboard","sources":[{"label":"Trial index (JSON)","url":"https://deepswe.datacurve.ai/artifacts/v1.1/trials.json"},{"label":"DeepSWE v1.1 leaderboard-live.json","url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json"},{"label":"DeepSWE v1.1 blog","url":"https://deepswe.datacurve.ai/blog/deepswe-v1-1"}],"run_ref":"Config mini_swe_agent_claude_opus_4_8_low","cost_estimated":false},{"id":"deepswe-mini_swe_agent_claude_sonnet_5_medium","benchmark":"deepswe","harness":"mini-swe-agent","model":"Claude Sonnet 5","effort":"medium","score":0.39778,"score_published":"40%","ci95":[0.35361,0.44368],"cost_per_attempt":4.07904,"cost_basis":"leaderboard mean_cost_usd per attempt over 450 attempts","tasks":113,"attempts":450,"k":4,"date":"2026-09-22","run":"Datacurve DeepSWE v1.1 leaderboard (mini-swe-agent), 113 tasks x 4 runs; 179/450 passed. Leaderboard CI (run-to-run) [0.367, 0.429]; ci95 here is Wilson over attempts.","source_kind":"leaderboard","sources":[{"label":"Trial index (JSON)","url":"https://deepswe.datacurve.ai/artifacts/v1.1/trials.json"},{"label":"DeepSWE v1.1 leaderboard-live.json","url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json"},{"label":"DeepSWE v1.1 blog","url":"https://deepswe.datacurve.ai/blog/deepswe-v1-1"}],"run_ref":"Config mini_swe_agent_claude_sonnet_5_medium","cost_estimated":false},{"id":"deepswe-mini_swe_agent_glm_5_2_high","benchmark":"deepswe","harness":"mini-swe-agent","model":"GLM-5.2","effort":"high","score":0.36283,"score_published":"36%","ci95":[0.31983,0.40814],"cost_per_attempt":2.83552,"cost_basis":"leaderboard mean_cost_usd per attempt over 452 attempts","tasks":113,"attempts":452,"k":4,"date":"2026-09-22","run":"Datacurve DeepSWE v1.1 leaderboard (mini-swe-agent), 113 tasks x 4 runs; 164/452 passed. Leaderboard CI (run-to-run) [0.315, 0.410]; ci95 here is Wilson over attempts.","source_kind":"leaderboard","sources":[{"label":"Trial index (JSON)","url":"https://deepswe.datacurve.ai/artifacts/v1.1/trials.json"},{"label":"DeepSWE v1.1 leaderboard-live.json","url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json"},{"label":"DeepSWE v1.1 blog","url":"https://deepswe.datacurve.ai/blog/deepswe-v1-1"}],"run_ref":"Config mini_swe_agent_glm_5_2_high","cost_estimated":false},{"id":"deepswe-mini_swe_agent_gemini_3_5_flash_high","benchmark":"deepswe","harness":"mini-swe-agent","model":"Gemini 3.5 Flash","effort":"high","score":0.36062,"score_published":"36%","ci95":[0.3177,0.40589],"cost_per_attempt":3.44671,"cost_basis":"leaderboard mean_cost_usd per attempt over 452 attempts","tasks":113,"attempts":452,"k":4,"date":"2026-09-22","run":"Datacurve DeepSWE v1.1 leaderboard (mini-swe-agent), 113 tasks x 4 runs; 163/452 passed. Leaderboard CI (run-to-run) [0.321, 0.400]; ci95 here is Wilson over attempts.","source_kind":"leaderboard","sources":[{"label":"Trial index (JSON)","url":"https://deepswe.datacurve.ai/artifacts/v1.1/trials.json"},{"label":"DeepSWE v1.1 leaderboard-live.json","url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json"},{"label":"DeepSWE v1.1 blog","url":"https://deepswe.datacurve.ai/blog/deepswe-v1-1"}],"run_ref":"Config mini_swe_agent_gemini_3_5_flash_high","cost_estimated":false},{"id":"deepswe-mini_swe_agent_gpt_5_6_terra_medium","benchmark":"deepswe","harness":"mini-swe-agent","model":"GPT-5.6 Terra","effort":"medium","score":0.35111,"score_published":"35%","ci95":[0.30844,0.3963],"cost_per_attempt":0.58325,"cost_basis":"leaderboard mean_cost_usd per attempt over 450 attempts","tasks":113,"attempts":450,"k":4,"date":"2026-09-22","run":"Datacurve DeepSWE v1.1 leaderboard (mini-swe-agent), 113 tasks x 4 runs; 158/450 passed. Leaderboard CI (run-to-run) [0.317, 0.385]; ci95 here is Wilson over attempts.","source_kind":"leaderboard","sources":[{"label":"Trial index (JSON)","url":"https://deepswe.datacurve.ai/artifacts/v1.1/trials.json"},{"label":"DeepSWE v1.1 leaderboard-live.json","url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json"},{"label":"DeepSWE v1.1 blog","url":"https://deepswe.datacurve.ai/blog/deepswe-v1-1"}],"run_ref":"Config mini_swe_agent_gpt_5_6_terra_medium","cost_estimated":false},{"id":"deepswe-mini_swe_agent_kimi_k2_7_code_default","benchmark":"deepswe","harness":"mini-swe-agent","model":"Kimi K2.7 Code","effort":null,"score":0.30531,"score_published":"31%","ci95":[0.26464,0.34926],"cost_per_attempt":2.81554,"cost_basis":"leaderboard mean_cost_usd per attempt over 452 attempts","tasks":113,"attempts":452,"k":4,"date":"2026-09-22","run":"Datacurve DeepSWE v1.1 leaderboard (mini-swe-agent), 113 tasks x 4 runs; 138/452 passed. Leaderboard CI (run-to-run) [0.300, 0.310]; ci95 here is Wilson over attempts.","source_kind":"leaderboard","sources":[{"label":"Trial index (JSON)","url":"https://deepswe.datacurve.ai/artifacts/v1.1/trials.json"},{"label":"DeepSWE v1.1 leaderboard-live.json","url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json"},{"label":"DeepSWE v1.1 blog","url":"https://deepswe.datacurve.ai/blog/deepswe-v1-1"}],"run_ref":"Config mini_swe_agent_kimi_k2_7_code_default","cost_estimated":false},{"id":"deepswe-mini_swe_agent_claude_sonnet_5_low","benchmark":"deepswe","harness":"mini-swe-agent","model":"Claude Sonnet 5","effort":"low","score":0.30512,"score_published":"31%","ci95":[0.26433,0.34922],"cost_per_attempt":2.18656,"cost_basis":"leaderboard mean_cost_usd per attempt over 449 attempts","tasks":113,"attempts":449,"k":4,"date":"2026-09-22","run":"Datacurve DeepSWE v1.1 leaderboard (mini-swe-agent), 113 tasks x 4 runs; 137/449 passed. Leaderboard CI (run-to-run) [0.294, 0.316]; ci95 here is Wilson over attempts.","source_kind":"leaderboard","sources":[{"label":"Trial index (JSON)","url":"https://deepswe.datacurve.ai/artifacts/v1.1/trials.json"},{"label":"DeepSWE v1.1 leaderboard-live.json","url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json"},{"label":"DeepSWE v1.1 blog","url":"https://deepswe.datacurve.ai/blog/deepswe-v1-1"}],"run_ref":"Config mini_swe_agent_claude_sonnet_5_low","cost_estimated":false},{"id":"deepswe-mini_swe_agent_claude_sonnet_4_6_high","benchmark":"deepswe","harness":"mini-swe-agent","model":"Claude Sonnet 4.6","effort":"high","score":0.29933,"score_published":"30%","ci95":[0.25891,0.34315],"cost_per_attempt":5.52235,"cost_basis":"leaderboard mean_cost_usd per attempt over 451 attempts","tasks":113,"attempts":451,"k":4,"date":"2026-09-22","run":"Datacurve DeepSWE v1.1 leaderboard (mini-swe-agent), 113 tasks x 4 runs; 135/451 passed. Leaderboard CI (run-to-run) [0.258, 0.340]; ci95 here is Wilson over attempts.","source_kind":"leaderboard","sources":[{"label":"Trial index (JSON)","url":"https://deepswe.datacurve.ai/artifacts/v1.1/trials.json"},{"label":"DeepSWE v1.1 leaderboard-live.json","url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json"},{"label":"DeepSWE v1.1 blog","url":"https://deepswe.datacurve.ai/blog/deepswe-v1-1"}],"run_ref":"Config mini_swe_agent_claude_sonnet_4_6_high","cost_estimated":false},{"id":"deepswe-mini_swe_agent_gpt_5_5_low","benchmark":"deepswe","harness":"mini-swe-agent","model":"GPT-5.5","effort":"low","score":0.26991,"score_published":"27%","ci95":[0.23105,0.31265],"cost_per_attempt":1.2002,"cost_basis":"leaderboard mean_cost_usd per attempt over 452 attempts","tasks":113,"attempts":452,"k":4,"date":"2026-09-22","run":"Datacurve DeepSWE v1.1 leaderboard (mini-swe-agent), 113 tasks x 4 runs; 122/452 passed. Leaderboard CI (run-to-run) [0.247, 0.293]; ci95 here is Wilson over attempts.","source_kind":"leaderboard","sources":[{"label":"Trial index (JSON)","url":"https://deepswe.datacurve.ai/artifacts/v1.1/trials.json"},{"label":"DeepSWE v1.1 leaderboard-live.json","url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json"},{"label":"DeepSWE v1.1 blog","url":"https://deepswe.datacurve.ai/blog/deepswe-v1-1"}],"run_ref":"Config mini_swe_agent_gpt_5_5_low","cost_estimated":false},{"id":"deepswe-mini_swe_agent_gpt_5_6_terra_low","benchmark":"deepswe","harness":"mini-swe-agent","model":"GPT-5.6 Terra","effort":"low","score":0.24053,"score_published":"24%","ci95":[0.20331,0.28216],"cost_per_attempt":0.42775,"cost_basis":"leaderboard mean_cost_usd per attempt over 449 attempts","tasks":113,"attempts":449,"k":4,"date":"2026-09-22","run":"Datacurve DeepSWE v1.1 leaderboard (mini-swe-agent), 113 tasks x 4 runs; 108/449 passed. Leaderboard CI (run-to-run) [0.233, 0.248]; ci95 here is Wilson over attempts.","source_kind":"leaderboard","sources":[{"label":"Trial index (JSON)","url":"https://deepswe.datacurve.ai/artifacts/v1.1/trials.json"},{"label":"DeepSWE v1.1 leaderboard-live.json","url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json"},{"label":"DeepSWE v1.1 blog","url":"https://deepswe.datacurve.ai/blog/deepswe-v1-1"}],"run_ref":"Config mini_swe_agent_gpt_5_6_terra_low","cost_estimated":false},{"id":"deepswe-mini_swe_agent_gemini_3_1_pro_preview_high","benchmark":"deepswe","harness":"mini-swe-agent","model":"Gemini 3.1 Pro Preview","effort":"high","score":0.11726,"score_published":"12%","ci95":[0.09077,0.15019],"cost_per_attempt":2.1434,"cost_basis":"leaderboard mean_cost_usd per attempt over 452 attempts","tasks":113,"attempts":452,"k":4,"date":"2026-09-22","run":"Datacurve DeepSWE v1.1 leaderboard (mini-swe-agent), 113 tasks x 4 runs; 53/452 passed. Leaderboard CI (run-to-run) [0.102, 0.132]; ci95 here is Wilson over attempts.","source_kind":"leaderboard","sources":[{"label":"Trial index (JSON)","url":"https://deepswe.datacurve.ai/artifacts/v1.1/trials.json"},{"label":"DeepSWE v1.1 leaderboard-live.json","url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json"},{"label":"DeepSWE v1.1 blog","url":"https://deepswe.datacurve.ai/blog/deepswe-v1-1"}],"run_ref":"Config mini_swe_agent_gemini_3_1_pro_preview_high","cost_estimated":false},{"id":"deepswe-mini_swe_agent_gpt_5_6_luna_medium","benchmark":"deepswe","harness":"mini-swe-agent","model":"GPT-5.6 Luna","effort":"medium","score":0.11283,"score_published":"11%","ci95":[0.08687,0.14532],"cost_per_attempt":0.21631,"cost_basis":"leaderboard mean_cost_usd per attempt over 452 attempts","tasks":113,"attempts":452,"k":4,"date":"2026-09-22","run":"Datacurve DeepSWE v1.1 leaderboard (mini-swe-agent), 113 tasks x 4 runs; 51/452 passed. Leaderboard CI (run-to-run) [0.105, 0.121]; ci95 here is Wilson over attempts.","source_kind":"leaderboard","sources":[{"label":"Trial index (JSON)","url":"https://deepswe.datacurve.ai/artifacts/v1.1/trials.json"},{"label":"DeepSWE v1.1 leaderboard-live.json","url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json"},{"label":"DeepSWE v1.1 blog","url":"https://deepswe.datacurve.ai/blog/deepswe-v1-1"}],"run_ref":"Config mini_swe_agent_gpt_5_6_luna_medium","cost_estimated":false},{"id":"deepswe-mini_swe_agent_gpt_5_6_luna_low","benchmark":"deepswe","harness":"mini-swe-agent","model":"GPT-5.6 Luna","effort":"low","score":0.01549,"score_published":"2%","ci95":[0.00752,0.03162],"cost_per_attempt":0.07241,"cost_basis":"leaderboard mean_cost_usd per attempt over 452 attempts","tasks":113,"attempts":452,"k":4,"date":"2026-09-22","run":"Datacurve DeepSWE v1.1 leaderboard (mini-swe-agent), 113 tasks x 4 runs; 7/452 passed. Leaderboard CI (run-to-run) [0.007, 0.024]; ci95 here is Wilson over attempts.","source_kind":"leaderboard","sources":[{"label":"Trial index (JSON)","url":"https://deepswe.datacurve.ai/artifacts/v1.1/trials.json"},{"label":"DeepSWE v1.1 leaderboard-live.json","url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json"},{"label":"DeepSWE v1.1 blog","url":"https://deepswe.datacurve.ai/blog/deepswe-v1-1"}],"run_ref":"Config mini_swe_agent_gpt_5_6_luna_low","cost_estimated":false},{"id":"aa-a2c87c06-tb4","benchmark":"tb4","harness":"Claude Code","model":"Claude Sonnet 5.5","effort":"max","score":0.66162,"score_published":"66.2%","ci95":null,"cost_per_attempt":13.12985,"cost_basis":"ESTIMATE: AA publishes only the row's mean cost, $14.19 per attempt; allocated 20.2% to this benchmark by attempts x (input x (1 - 0.9 x cacheHitRate 0.95) + 5 x output tokens), as a share of that mean","cost_estimated":true,"tasks":66,"attempts":198,"k":3,"date":null,"run":"Artificial Analysis per-benchmark mean reward for 'Claude Code - Sonnet 5.5 (max)', 198 attempts. No run date published.","source_kind":"aa","sources":[{"label":"Artificial Analysis Coding Agents (no per-run page)","url":"https://artificialanalysis.ai/agents/coding-agents"}]},{"id":"vals-claude-opus-5-5","benchmark":"tb4","harness":"mini-swe-agent","model":"Claude Opus 5.5","effort":null,"score":0.65152,"score_published":"65.15%","ci95":null,"cost_per_attempt":13.19863,"cost_basis":"Vals.ai cost_per_test (mean per task attempt)","tasks":66,"attempts":198,"k":3,"date":"2026-10-01","run":"Vals.ai run: mini-swe-agent on Harbor/Daytona, avg@3 (three full pass@1 runs); reasoning effort not published; some rows include provider fallbacks/refusals noted on the page.","source_kind":"leaderboard","sources":[{"label":"Vals.ai results for Claude Opus 5.5","url":"https://www.vals.ai/models/anthropic_claude-opus-5-5"},{"label":"Vals.ai Terminal-Bench 4.0","url":"https://www.vals.ai/benchmarks/terminal-bench-4"}],"cost_estimated":false},{"id":"vals-claude-sonnet-5-5","benchmark":"tb4","harness":"mini-swe-agent","model":"Claude Sonnet 5.5","effort":null,"score":0.64141,"score_published":"64.14%","ci95":null,"cost_per_attempt":16.51301,"cost_basis":"Vals.ai cost_per_test (mean per task attempt)","tasks":66,"attempts":198,"k":3,"date":"2026-10-01","run":"Vals.ai run: mini-swe-agent on Harbor/Daytona, avg@3 (three full pass@1 runs); reasoning effort not published; some rows include provider fallbacks/refusals noted on the page.","source_kind":"leaderboard","sources":[{"label":"Vals.ai results for Claude Sonnet 5.5","url":"https://www.vals.ai/models/anthropic_claude-sonnet-5-5"},{"label":"Vals.ai Terminal-Bench 4.0","url":"https://www.vals.ai/benchmarks/terminal-bench-4"}],"cost_estimated":false},{"id":"aa-ab5074e4-tb4","benchmark":"tb4","harness":"Claude Code","model":"Claude Opus 5.5","effort":"max","score":0.63131,"score_published":"63.1%","ci95":null,"cost_per_attempt":14.44662,"cost_basis":"ESTIMATE: AA publishes only the row's mean cost, $13.04 per attempt; allocated 24.1% to this benchmark by attempts x (input x (1 - 0.9 x cacheHitRate 0.95) + 5 x output tokens), as a share of that mean","cost_estimated":true,"tasks":66,"attempts":198,"k":3,"date":null,"run":"Artificial Analysis per-benchmark mean reward for 'Claude Code - Opus 5.5 (max)', 198 attempts. No run date published.","source_kind":"aa","sources":[{"label":"Artificial Analysis Coding Agents (no per-run page)","url":"https://artificialanalysis.ai/agents/coding-agents"}]},{"id":"vals-gpt-6-astra","benchmark":"tb4","harness":"mini-swe-agent","model":"GPT-6 Astra","effort":null,"score":0.59596,"score_published":"59.60%","ci95":null,"cost_per_attempt":9.58426,"cost_basis":"Vals.ai cost_per_test (mean per task attempt)","tasks":66,"attempts":198,"k":3,"date":"2026-10-01","run":"Vals.ai run: mini-swe-agent on Harbor/Daytona, avg@3 (three full pass@1 runs); reasoning effort not published; some rows include provider fallbacks/refusals noted on the page.","source_kind":"leaderboard","sources":[{"label":"Vals.ai results for GPT-6 Astra","url":"https://www.vals.ai/models/openai_gpt-6-astra"},{"label":"Vals.ai Terminal-Bench 4.0","url":"https://www.vals.ai/benchmarks/terminal-bench-4"}],"cost_estimated":false},{"id":"hub-0f01715e","benchmark":"tb4","harness":"Codex","harness_version":"0.151.0","model":"GPT-6 Astra","effort":"max","score":0.58182,"score_published":"192/330","ci95":[0.52796,0.6338],"cost_per_attempt":9.90054,"cost_basis":"Harbor trial cost_usd, mean over 330 of 330 graded attempts","tasks":66,"attempts":330,"k":5,"date":"2026-09-01","run":"Harbor job 'Terminal-Bench 4.0.0 · GPT-6 Astra / Codex · reasoning effort max' by Terminal-Bench; 66 tasks x 5, 330 graded attempts (reward>=1 is a pass, timeouts score 0); Codex 0.151.0; dataset ref sha256:39d9f44b4042.","source_kind":"harbor","sources":[{"label":"Harbor Hub job 0f01715e","url":"https://hub.harborframework.com/jobs/0f01715e-2f98-40e3-836f-1ac4fdce39a4"},{"label":"Terminal-Bench 4.0 leaderboard row","url":"https://hub.harborframework.com/datasets/terminal-bench/terminal-bench/4/leaderboards/4-0-0/rows/5c537be4-7fc3-449b-8bfc-ceb9061c2535"}],"run_ref":"Job 0f01715e-2f98-40e3-836f-1ac4fdce39a4","cost_estimated":false},{"id":"vals-claude-fable-5-1","benchmark":"tb4","harness":"mini-swe-agent","model":"Claude Fable 5.1","effort":null,"score":0.58081,"score_published":"58.08%","ci95":null,"cost_per_attempt":17.17534,"cost_basis":"Vals.ai cost_per_test (mean per task attempt)","tasks":66,"attempts":198,"k":3,"date":"2026-10-01","run":"Vals.ai run: mini-swe-agent on Harbor/Daytona, avg@3 (three full pass@1 runs); reasoning effort not published; some rows include provider fallbacks/refusals noted on the page.","source_kind":"leaderboard","sources":[{"label":"Vals.ai results for Claude Fable 5.1","url":"https://www.vals.ai/models/anthropic_claude-fable-5-1"},{"label":"Vals.ai Terminal-Bench 4.0","url":"https://www.vals.ai/benchmarks/terminal-bench-4"}],"cost_estimated":false},{"id":"aa-666605c6-tb4","benchmark":"tb4","harness":"Claude Code","model":"Claude Sonnet 5.5","effort":"xhigh","score":0.58081,"score_published":"58.1%","ci95":null,"cost_per_attempt":4.69521,"cost_basis":"ESTIMATE: AA publishes only the row's mean cost, $3.33 per attempt; allocated 30.7% to this benchmark by attempts x (input x (1 - 0.9 x cacheHitRate 0.96) + 5 x output tokens), as a share of that mean","cost_estimated":true,"tasks":66,"attempts":198,"k":3,"date":null,"run":"Artificial Analysis per-benchmark mean reward for 'Claude Code - Sonnet 5.5 (xhigh)', 198 attempts. No run date published.","source_kind":"aa","sources":[{"label":"Artificial Analysis Coding Agents (no per-run page)","url":"https://artificialanalysis.ai/agents/coding-agents"}]},{"id":"hub-46b9c704","benchmark":"tb4","harness":"Claude Code","harness_version":"2.1.273","model":"Claude Fable 5.1","effort":"xhigh","score":0.58055,"score_published":"191/329","ci95":[0.5266,0.63264],"cost_per_attempt":14.80865,"cost_basis":"Harbor trial cost_usd, mean over 329 of 329 graded attempts","tasks":66,"attempts":329,"k":5,"date":"2026-09-17","run":"Harbor job 'Terminal-Bench 4.0.0 · Fable 5.1 / Claude Code · reasoning effort xhigh' by Terminal-Bench; 66 tasks x 5, 329 graded attempts (reward>=1 is a pass, timeouts score 0); Claude Code 2.1.273; dataset ref sha256:39d9f44b4042.","source_kind":"harbor","sources":[{"label":"Harbor Hub job 46b9c704","url":"https://hub.harborframework.com/jobs/46b9c704-4b5b-4a44-ba22-03e74fefc257"},{"label":"Terminal-Bench 4.0 leaderboard row","url":"https://hub.harborframework.com/datasets/terminal-bench/terminal-bench/4/leaderboards/4-0-0/rows/9e1c5e11-748f-4f2e-911a-070a396d5bf8"}],"run_ref":"Job 46b9c704-4b5b-4a44-ba22-03e74fefc257","cost_estimated":false},{"id":"hub-86e3ac63","benchmark":"tb4","harness":"Codex","harness_version":"0.151.0","model":"GPT-6 Astra","effort":"high","score":0.57879,"score_published":"191/330","ci95":[0.52491,0.63085],"cost_per_attempt":6.87703,"cost_basis":"Harbor trial cost_usd, mean over 330 of 330 graded attempts","tasks":66,"attempts":330,"k":5,"date":"2026-09-01","run":"Harbor job 'Terminal-Bench 4.0.0 · GPT-6 Astra / Codex · reasoning effort high' by Terminal-Bench; 66 tasks x 5, 330 graded attempts (reward>=1 is a pass, timeouts score 0); Codex 0.151.0; dataset ref sha256:39d9f44b4042.","source_kind":"harbor","sources":[{"label":"Harbor Hub job 86e3ac63","url":"https://hub.harborframework.com/jobs/86e3ac63-37c4-4003-8248-483fdb9b43ce"},{"label":"Terminal-Bench 4.0 leaderboard row","url":"https://hub.harborframework.com/datasets/terminal-bench/terminal-bench/4/leaderboards/4-0-0/rows/3475050c-bf5e-4261-a3f6-0af5350af13f"}],"run_ref":"Job 86e3ac63-37c4-4003-8248-483fdb9b43ce","cost_estimated":false},{"id":"hub-a6319b9c","benchmark":"tb4","harness":"Claude Code","harness_version":"2.1.257","model":"Claude Fable 5.1","effort":"max","score":0.57879,"score_published":"191/330","ci95":[0.52491,0.63085],"cost_per_attempt":18.91971,"cost_basis":"Harbor trial cost_usd, mean over 330 of 330 graded attempts","tasks":66,"attempts":330,"k":5,"date":"2026-09-02","run":"Harbor job 'Terminal-Bench 4.0.0 · Fable 5.1 / Claude Code' by Terminal-Bench; 66 tasks x 5, 330 graded attempts (reward>=1 is a pass, timeouts score 0); Claude Code 2.1.257; dataset ref sha256:39d9f44b4042.","source_kind":"harbor","sources":[{"label":"Harbor Hub job a6319b9c","url":"https://hub.harborframework.com/jobs/a6319b9c-7dbc-42b1-8483-6c022e292ae2"},{"label":"Terminal-Bench 4.0 leaderboard row","url":"https://hub.harborframework.com/datasets/terminal-bench/terminal-bench/4/leaderboards/4-0-0/rows/c741608e-c94e-417d-b7a8-e67111a0c887"}],"run_ref":"Job a6319b9c-7dbc-42b1-8483-6c022e292ae2","cost_estimated":false},{"id":"hub-caf6e433","benchmark":"tb4","harness":"Codex","harness_version":"0.151.0","model":"GPT-6 Astra","effort":"xhigh","score":0.57879,"score_published":"191/330","ci95":[0.52491,0.63085],"cost_per_attempt":7.12277,"cost_basis":"Harbor trial cost_usd, mean over 330 of 330 graded attempts","tasks":66,"attempts":330,"k":5,"date":"2026-09-01","run":"Harbor job 'Terminal-Bench 4.0.0 · GPT-6 Astra / Codex · reasoning effort xhigh' by Terminal-Bench; 66 tasks x 5, 330 graded attempts (reward>=1 is a pass, timeouts score 0); Codex 0.151.0; dataset ref sha256:39d9f44b4042.","source_kind":"harbor","sources":[{"label":"Harbor Hub job caf6e433","url":"https://hub.harborframework.com/jobs/caf6e433-9fff-4475-b0bc-977f14ac7052"},{"label":"Terminal-Bench 4.0 leaderboard row","url":"https://hub.harborframework.com/datasets/terminal-bench/terminal-bench/4/leaderboards/4-0-0/rows/16db8ad5-84aa-4588-b660-1ce68c0d45e2"}],"run_ref":"Job caf6e433-9fff-4475-b0bc-977f14ac7052","cost_estimated":false},{"id":"vals-gemini-4-argon","benchmark":"tb4","harness":"mini-swe-agent","model":"Gemini 4 Argon","effort":null,"score":0.57576,"score_published":"57.58%","ci95":null,"cost_per_attempt":17.63994,"cost_basis":"Vals.ai cost_per_test (mean per task attempt)","tasks":66,"attempts":198,"k":3,"date":"2026-10-01","run":"Vals.ai run: mini-swe-agent on Harbor/Daytona, avg@3 (three full pass@1 runs); reasoning effort not published; some rows include provider fallbacks/refusals noted on the page.","source_kind":"leaderboard","sources":[{"label":"Vals.ai results for Gemini 4 Argon","url":"https://www.vals.ai/models/google_gemini-4-argon"},{"label":"Vals.ai Terminal-Bench 4.0","url":"https://www.vals.ai/benchmarks/terminal-bench-4"}],"cost_estimated":false},{"id":"aa-ef63fafa-tb4","benchmark":"tb4","harness":"Antigravity CLI","model":"Gemini 4 Argon","effort":"high","score":0.56061,"score_published":"56.1%","ci95":null,"cost_per_attempt":6.85442,"cost_basis":"ESTIMATE: AA publishes only the row's mean cost, $5.84 per attempt; allocated 25.7% to this benchmark by attempts x (input x (1 - 0.9 x cacheHitRate 0.87) + 5 x output tokens), as a share of that mean","cost_estimated":true,"tasks":66,"attempts":198,"k":3,"date":null,"run":"Artificial Analysis per-benchmark mean reward for 'Antigravity CLI - Gemini 4 Argon (high)', 198 attempts. No run date published. Row marked unavailable on AA.","source_kind":"aa","sources":[{"label":"Artificial Analysis Coding Agents (no per-run page)","url":"https://artificialanalysis.ai/agents/coding-agents"}]},{"id":"aa-34970c96-tb4","benchmark":"tb4","harness":"Devin Fusion CLI","model":"Claude Fable 5.1 + SWE-2 Medium","effort":"xhigh","score":0.56061,"score_published":"56.1%","ci95":null,"cost_per_attempt":13.44019,"cost_basis":"ESTIMATE: AA publishes only the row's mean cost, $7.90 per attempt; allocated 36.9% to this benchmark by attempts x (input x (1 - 0.9 x cacheHitRate 0.96) + 5 x output tokens), as a share of that mean","cost_estimated":true,"tasks":66,"attempts":197,"k":3,"date":null,"run":"Artificial Analysis per-benchmark mean reward for 'Devin Fusion CLI - Claude Fable 5.1 XHigh + SWE-2 Medium', 197 attempts. No run date published.","source_kind":"aa","sources":[{"label":"Artificial Analysis Coding Agents (no per-run page)","url":"https://artificialanalysis.ai/agents/coding-agents"}]},{"id":"vals-gpt-6.1-sol","benchmark":"tb4","harness":"mini-swe-agent","model":"GPT-6.1 Sol","effort":null,"score":0.55051,"score_published":"55.05%","ci95":null,"cost_per_attempt":1.72496,"cost_basis":"Vals.ai cost_per_test (mean per task attempt)","tasks":66,"attempts":198,"k":3,"date":"2026-10-01","run":"Vals.ai run: mini-swe-agent on Harbor/Daytona, avg@3 (three full pass@1 runs); reasoning effort not published; some rows include provider fallbacks/refusals noted on the page.","source_kind":"leaderboard","sources":[{"label":"Vals.ai results for GPT-6.1 Sol","url":"https://www.vals.ai/models/openai_gpt-6.1-sol"},{"label":"Vals.ai Terminal-Bench 4.0","url":"https://www.vals.ai/benchmarks/terminal-bench-4"}],"cost_estimated":false},{"id":"hub-6ccd097a","benchmark":"tb4","harness":"Pi","harness_version":"0.85.1","model":"GPT-6 Astra","effort":"xhigh","score":0.55015,"score_published":"181/329","ci95":[0.49613,0.60302],"cost_per_attempt":5.53285,"cost_basis":"Harbor trial cost_usd, mean over 329 of 329 graded attempts","tasks":66,"attempts":329,"k":5,"date":"2026-09-17","run":"Harbor job '20260916-tb4-pi-astra-xhigh-modal' by Philip Sinitsin on Modal; 66 tasks x 5, 329 graded attempts (reward>=1 is a pass, timeouts score 0); Pi 0.85.1; dataset ref sha256:39d9f44b4042.","source_kind":"harbor","sources":[{"label":"Harbor Hub job 6ccd097a","url":"https://hub.harborframework.com/jobs/6ccd097a-747b-4867-be57-a56969df988f"}],"run_ref":"Job 6ccd097a-747b-4867-be57-a56969df988f","cost_estimated":false},{"id":"hub-da64e1b8","benchmark":"tb4","harness":"Claude Code","harness_version":"2.1.273","model":"Claude Fable 5.1","effort":"high","score":0.54545,"score_published":"180/330","ci95":[0.49152,0.59835],"cost_per_attempt":12.0769,"cost_basis":"Harbor trial cost_usd, mean over 330 of 330 graded attempts","tasks":66,"attempts":330,"k":5,"date":"2026-09-17","run":"Harbor job 'Terminal-Bench 4.0.0 · Fable 5.1 / Claude Code · reasoning effort high' by Terminal-Bench; 66 tasks x 5, 330 graded attempts (reward>=1 is a pass, timeouts score 0); Claude Code 2.1.273; dataset ref sha256:39d9f44b4042.","source_kind":"harbor","sources":[{"label":"Harbor Hub job da64e1b8","url":"https://hub.harborframework.com/jobs/da64e1b8-5de1-4246-814d-234f9a6ec09a"},{"label":"Terminal-Bench 4.0 leaderboard row","url":"https://hub.harborframework.com/datasets/terminal-bench/terminal-bench/4/leaderboards/4-0-0/rows/4643b23c-53e8-47ae-a3bd-9abc900e9588"}],"run_ref":"Job da64e1b8-5de1-4246-814d-234f9a6ec09a","cost_estimated":false},{"id":"aa-ed8c1d0b-tb4","benchmark":"tb4","harness":"Codex","model":"GPT-6.1 Sol","effort":"xhigh","score":0.54545,"score_published":"54.5%","ci95":null,"cost_per_attempt":1.60654,"cost_basis":"ESTIMATE: AA publishes only the row's mean cost, $1.04 per attempt; allocated 33.7% to this benchmark by attempts x (input x (1 - 0.9 x cacheHitRate 0.93) + 5 x output tokens), as a share of that mean","cost_estimated":true,"tasks":66,"attempts":198,"k":3,"date":null,"run":"Artificial Analysis per-benchmark mean reward for 'Codex - GPT-6.1 Sol (xhigh)', 198 attempts. No run date published.","source_kind":"aa","sources":[{"label":"Artificial Analysis Coding Agents (no per-run page)","url":"https://artificialanalysis.ai/agents/coding-agents"}]},{"id":"hub-2350fd05","benchmark":"tb4","harness":"Codex","harness_version":"0.151.0","model":"GPT-6 Astra","effort":"medium","score":0.54242,"score_published":"179/330","ci95":[0.48849,0.59538],"cost_per_attempt":5.80243,"cost_basis":"Harbor trial cost_usd, mean over 330 of 330 graded attempts","tasks":66,"attempts":330,"k":5,"date":"2026-09-01","run":"Harbor job 'Terminal-Bench 4.0.0 · GPT-6 Astra / Codex · reasoning effort medium' by Terminal-Bench; 66 tasks x 5, 330 graded attempts (reward>=1 is a pass, timeouts score 0); Codex 0.151.0; dataset ref sha256:39d9f44b4042.","source_kind":"harbor","sources":[{"label":"Harbor Hub job 2350fd05","url":"https://hub.harborframework.com/jobs/2350fd05-ea3b-407e-94ad-07b67c1ed847"},{"label":"Terminal-Bench 4.0 leaderboard row","url":"https://hub.harborframework.com/datasets/terminal-bench/terminal-bench/4/leaderboards/4-0-0/rows/f3c3d5a6-6424-4acb-bfcc-615c3f79f6dd"}],"run_ref":"Job 2350fd05-ea3b-407e-94ad-07b67c1ed847","cost_estimated":false},{"id":"hub-153e8318","benchmark":"tb4","harness":"Claude Code","harness_version":"2.1.273","model":"Claude Opus 5","effort":"xhigh","score":0.53939,"score_published":"178/330","ci95":[0.48547,0.59241],"cost_per_attempt":18.44311,"cost_basis":"Harbor trial cost_usd, mean over 330 of 330 graded attempts","tasks":66,"attempts":330,"k":5,"date":"2026-09-17","run":"Harbor job 'Terminal-Bench 4.0.0 · Opus 5 / Claude Code · reasoning effort xhigh' by Terminal-Bench; 66 tasks x 5, 330 graded attempts (reward>=1 is a pass, timeouts score 0); Claude Code 2.1.273; dataset ref sha256:39d9f44b4042.","source_kind":"harbor","sources":[{"label":"Harbor Hub job 153e8318","url":"https://hub.harborframework.com/jobs/153e8318-9b14-4488-973c-345f4fa4b1ba"},{"label":"Terminal-Bench 4.0 leaderboard row","url":"https://hub.harborframework.com/datasets/terminal-bench/terminal-bench/4/leaderboards/4-0-0/rows/f7844abc-3109-4d5f-91ad-481df7b9fac2"}],"run_ref":"Job 153e8318-9b14-4488-973c-345f4fa4b1ba","cost_estimated":false},{"id":"hub-b6211f84","benchmark":"tb4","harness":"Claude Code","harness_version":"2.1.273","model":"Claude Fable 5.1","effort":"medium","score":0.53939,"score_published":"178/330","ci95":[0.48547,0.59241],"cost_per_attempt":8.58455,"cost_basis":"Harbor trial cost_usd, mean over 330 of 330 graded attempts","tasks":66,"attempts":330,"k":5,"date":"2026-09-17","run":"Harbor job 'Terminal-Bench 4.0.0 · Fable 5.1 / Claude Code · reasoning effort medium' by Terminal-Bench; 66 tasks x 5, 330 graded attempts (reward>=1 is a pass, timeouts score 0); Claude Code 2.1.273; dataset ref sha256:39d9f44b4042.","source_kind":"harbor","sources":[{"label":"Harbor Hub job b6211f84","url":"https://hub.harborframework.com/jobs/b6211f84-8a65-44fd-85ed-d509db5d1861"},{"label":"Terminal-Bench 4.0 leaderboard row","url":"https://hub.harborframework.com/datasets/terminal-bench/terminal-bench/4/leaderboards/4-0-0/rows/83e6bf7e-aeb4-4116-9f30-ca8de96603fb"}],"run_ref":"Job b6211f84-8a65-44fd-85ed-d509db5d1861","cost_estimated":false},{"id":"vals-claude-opus-5","benchmark":"tb4","harness":"mini-swe-agent","model":"Claude Opus 5","effort":null,"score":0.53535,"score_published":"53.53%","ci95":null,"cost_per_attempt":18.59691,"cost_basis":"Vals.ai cost_per_test (mean per task attempt)","tasks":66,"attempts":198,"k":3,"date":"2026-10-01","run":"Vals.ai run: mini-swe-agent on Harbor/Daytona, avg@3 (three full pass@1 runs); reasoning effort not published; some rows include provider fallbacks/refusals noted on the page.","source_kind":"leaderboard","sources":[{"label":"Vals.ai results for Claude Opus 5","url":"https://www.vals.ai/models/anthropic_claude-opus-5"},{"label":"Vals.ai Terminal-Bench 4.0","url":"https://www.vals.ai/benchmarks/terminal-bench-4"}],"cost_estimated":false},{"id":"aa-ea296ce7-tb4","benchmark":"tb4","harness":"Codex","model":"GPT-6.1 Sol","effort":"max","score":0.5303,"score_published":"53.0%","ci95":null,"cost_per_attempt":2.32919,"cost_basis":"ESTIMATE: AA publishes only the row's mean cost, $1.55 per attempt; allocated 32.6% to this benchmark by attempts x (input x (1 - 0.9 x cacheHitRate 0.93) + 5 x output tokens), as a share of that mean","cost_estimated":true,"tasks":66,"attempts":198,"k":3,"date":null,"run":"Artificial Analysis per-benchmark mean reward for 'Codex - GPT-6.1 Sol (max)', 198 attempts. No run date published.","source_kind":"aa","sources":[{"label":"Artificial Analysis Coding Agents (no per-run page)","url":"https://artificialanalysis.ai/agents/coding-agents"}]},{"id":"hub-a1ac63a1","benchmark":"tb4","harness":"Claude Code","harness_version":"2.1.231","model":"Claude Opus 5","effort":"max","score":0.52584,"score_published":"173/329","ci95":[0.47189,0.57918],"cost_per_attempt":18.3425,"cost_basis":"Harbor trial cost_usd, mean over 329 of 329 graded attempts","tasks":66,"attempts":329,"k":5,"date":"2026-08-27","run":"Harbor job 'Terminal-Bench 4.0.0 · Opus 5 / Claude Code' by Terminal-Bench on Docker; 66 tasks x 5, 329 graded attempts (reward>=1 is a pass, timeouts score 0); Claude Code 2.1.231; dataset ref 4.","source_kind":"harbor","sources":[{"label":"Harbor Hub job a1ac63a1","url":"https://hub.harborframework.com/jobs/a1ac63a1-8a9b-4bc7-9906-2b63657ee1c2"},{"label":"Terminal-Bench 4.0 leaderboard row","url":"https://hub.harborframework.com/datasets/terminal-bench/terminal-bench/4/leaderboards/4-0-0/rows/d71ac3d0-da36-49cb-89d9-323136e77111"}],"run_ref":"Job a1ac63a1-8a9b-4bc7-9906-2b63657ee1c2","cost_estimated":false},{"id":"aa-93acd59f-tb4","benchmark":"tb4","harness":"Codex","model":"GPT-6.1 Sol","effort":"medium","score":0.51515,"score_published":"51.5%","ci95":null,"cost_per_attempt":1.11548,"cost_basis":"ESTIMATE: AA publishes only the row's mean cost, $0.70 per attempt; allocated 34.5% to this benchmark by attempts x (input x (1 - 0.9 x cacheHitRate 0.92) + 5 x output tokens), as a share of that mean","cost_estimated":true,"tasks":66,"attempts":198,"k":3,"date":null,"run":"Artificial Analysis per-benchmark mean reward for 'Codex - GPT-6.1 Sol (medium)', 198 attempts. No run date published.","source_kind":"aa","sources":[{"label":"Artificial Analysis Coding Agents (no per-run page)","url":"https://artificialanalysis.ai/agents/coding-agents"}]},{"id":"hub-a538f69d","benchmark":"tb4","harness":"Codex","harness_version":"0.151.0","model":"GPT-6 Astra","effort":"low","score":0.50606,"score_published":"167/330","ci95":[0.45236,0.55962],"cost_per_attempt":4.71909,"cost_basis":"Harbor trial cost_usd, mean over 330 of 330 graded attempts","tasks":66,"attempts":330,"k":5,"date":"2026-09-01","run":"Harbor job 'Terminal-Bench 4.0.0 · GPT-6 Astra / Codex · reasoning effort low' by Terminal-Bench; 66 tasks x 5, 330 graded attempts (reward>=1 is a pass, timeouts score 0); Codex 0.151.0; dataset ref sha256:39d9f44b4042.","source_kind":"harbor","sources":[{"label":"Harbor Hub job a538f69d","url":"https://hub.harborframework.com/jobs/a538f69d-7756-444a-9dfe-43a0ab10b3a8"},{"label":"Terminal-Bench 4.0 leaderboard row","url":"https://hub.harborframework.com/datasets/terminal-bench/terminal-bench/4/leaderboards/4-0-0/rows/b3ad58f3-b311-4d4d-875b-3158cf0309d6"}],"run_ref":"Job a538f69d-7756-444a-9dfe-43a0ab10b3a8","cost_estimated":false},{"id":"hub-cf440c37","benchmark":"tb4","harness":"Claude Code","harness_version":"2.1.273","model":"Claude Opus 5","effort":"high","score":0.50303,"score_published":"166/330","ci95":[0.44936,0.55663],"cost_per_attempt":14.17105,"cost_basis":"Harbor trial cost_usd, mean over 329 of 330 graded attempts","tasks":66,"attempts":330,"k":5,"date":"2026-09-17","run":"Harbor job 'Terminal-Bench 4.0.0 · Opus 5 / Claude Code · reasoning effort high' by Terminal-Bench; 66 tasks x 5, 330 graded attempts (reward>=1 is a pass, timeouts score 0); Claude Code 2.1.273; dataset ref sha256:39d9f44b4042.","source_kind":"harbor","sources":[{"label":"Harbor Hub job cf440c37","url":"https://hub.harborframework.com/jobs/cf440c37-307c-4a9a-9baf-71d52572fab4"},{"label":"Terminal-Bench 4.0 leaderboard row","url":"https://hub.harborframework.com/datasets/terminal-bench/terminal-bench/4/leaderboards/4-0-0/rows/0b2908ee-3035-447c-9876-e68fecb3f413"}],"run_ref":"Job cf440c37-307c-4a9a-9baf-71d52572fab4","cost_estimated":false},{"id":"aa-fa259509-tb4","benchmark":"tb4","harness":"Devin Fusion CLI","model":"GPT-6 Astra + SWE-2 Medium","effort":"xhigh","score":0.5,"score_published":"50.0%","ci95":null,"cost_per_attempt":4.7896,"cost_basis":"ESTIMATE: AA publishes only the row's mean cost, $4.54 per attempt; allocated 23.0% to this benchmark by attempts x (input x (1 - 0.9 x cacheHitRate 0.92) + 5 x output tokens), as a share of that mean","cost_estimated":true,"tasks":66,"attempts":198,"k":3,"date":null,"run":"Artificial Analysis per-benchmark mean reward for 'Devin Fusion CLI - GPT-6 Astra XHigh + SWE-2 Medium', 198 attempts. No run date published.","source_kind":"aa","sources":[{"label":"Artificial Analysis Coding Agents (no per-run page)","url":"https://artificialanalysis.ai/agents/coding-agents"}]},{"id":"aa-4aee8aa2-tb4","benchmark":"tb4","harness":"Codex","model":"GPT-6.1 Sol","effort":"high","score":0.5,"score_published":"50.0%","ci95":null,"cost_per_attempt":1.34841,"cost_basis":"ESTIMATE: AA publishes only the row's mean cost, $0.89 per attempt; allocated 33.0% to this benchmark by attempts x (input x (1 - 0.9 x cacheHitRate 0.93) + 5 x output tokens), as a share of that mean","cost_estimated":true,"tasks":66,"attempts":198,"k":3,"date":null,"run":"Artificial Analysis per-benchmark mean reward for 'Codex - GPT-6.1 Sol (high)', 198 attempts. No run date published.","source_kind":"aa","sources":[{"label":"Artificial Analysis Coding Agents (no per-run page)","url":"https://artificialanalysis.ai/agents/coding-agents"}]},{"id":"hub-a822d728","benchmark":"tb4","harness":"mini-swe-agent","harness_version":"2.4.5","model":"Claude Opus 5","effort":"max","score":0.49231,"score_published":"160/325","ci95":[0.43836,0.54643],"cost_per_attempt":19.72261,"cost_basis":"Harbor trial cost_usd, mean over 325 of 325 graded attempts","tasks":65,"attempts":325,"k":5,"date":"2026-08-26","run":"Harbor job 'Terminal-Bench 4.0.0 · Opus 5 (Max) / mini-SWE-agent' by Ryan Marten on Modal; 65 tasks x 5, 325 graded attempts (reward>=1 is a pass, timeouts score 0); mini-swe-agent 2.4.5; dataset ref sha256:3afcd8602ae9. Pre-release TB 4.0.0 revision (65/66 task content hashes identical to the dataset-ref '4' runs).","source_kind":"harbor","sources":[{"label":"Harbor Hub job a822d728","url":"https://hub.harborframework.com/jobs/a822d728-eb5a-43c6-a19e-7a00030c7770"}],"run_ref":"Job a822d728-eb5a-43c6-a19e-7a00030c7770","cost_estimated":false},{"id":"aa-fa589dad-tb4","benchmark":"tb4","harness":"Codex","model":"GPT-6.1 Sol","effort":"low","score":0.4899,"score_published":"49.0%","ci95":null,"cost_per_attempt":1.00412,"cost_basis":"ESTIMATE: AA publishes only the row's mean cost, $0.50 per attempt; allocated 43.8% to this benchmark by attempts x (input x (1 - 0.9 x cacheHitRate 0.91) + 5 x output tokens), as a share of that mean","cost_estimated":true,"tasks":66,"attempts":198,"k":3,"date":null,"run":"Artificial Analysis per-benchmark mean reward for 'Codex - GPT-6.1 Sol (low)', 198 attempts. No run date published.","source_kind":"aa","sources":[{"label":"Artificial Analysis Coding Agents (no per-run page)","url":"https://artificialanalysis.ai/agents/coding-agents"}]},{"id":"hub-3c35174a","benchmark":"tb4","harness":"Claude Code","harness_version":"2.1.273","model":"Claude Opus 5","effort":"medium","score":0.44848,"score_published":"148/330","ci95":[0.39572,0.50243],"cost_per_attempt":9.6715,"cost_basis":"Harbor trial cost_usd, mean over 330 of 330 graded attempts","tasks":66,"attempts":330,"k":5,"date":"2026-09-17","run":"Harbor job 'Terminal-Bench 4.0.0 · Opus 5 / Claude Code · reasoning effort medium' by Terminal-Bench; 66 tasks x 5, 330 graded attempts (reward>=1 is a pass, timeouts score 0); Claude Code 2.1.273; dataset ref sha256:39d9f44b4042.","source_kind":"harbor","sources":[{"label":"Harbor Hub job 3c35174a","url":"https://hub.harborframework.com/jobs/3c35174a-3a05-4705-bf8e-c09a3d09f013"},{"label":"Terminal-Bench 4.0 leaderboard row","url":"https://hub.harborframework.com/datasets/terminal-bench/terminal-bench/4/leaderboards/4-0-0/rows/8c102a88-0789-4ce7-b503-bd39e84e157b"}],"run_ref":"Job 3c35174a-3a05-4705-bf8e-c09a3d09f013","cost_estimated":false},{"id":"hub-991af5e0","benchmark":"tb4","harness":"Claude Code","harness_version":"2.1.231","model":"Claude Fable 5","effort":"max","score":0.44545,"score_published":"147/330","ci95":[0.39276,0.4994],"cost_per_attempt":22.01518,"cost_basis":"Harbor trial cost_usd, mean over 330 of 330 graded attempts","tasks":66,"attempts":330,"k":5,"date":"2026-08-27","run":"Harbor job 'Terminal-Bench 4.0.0 · Fable 5 / Claude Code' by Terminal-Bench on Docker; 66 tasks x 5, 330 graded attempts (reward>=1 is a pass, timeouts score 0); Claude Code 2.1.231; dataset ref 4.","source_kind":"harbor","sources":[{"label":"Harbor Hub job 991af5e0","url":"https://hub.harborframework.com/jobs/991af5e0-8c5f-4511-9619-a2a46939ebd4"},{"label":"Terminal-Bench 4.0 leaderboard row","url":"https://hub.harborframework.com/datasets/terminal-bench/terminal-bench/4/leaderboards/4-0-0/rows/36c077e0-4879-4444-b315-8532d66401d6"}],"run_ref":"Job 991af5e0-8c5f-4511-9619-a2a46939ebd4","cost_estimated":false},{"id":"vals-gpt-6-sol","benchmark":"tb4","harness":"mini-swe-agent","model":"GPT-6 Sol","effort":null,"score":0.44444,"score_published":"44.44%","ci95":null,"cost_per_attempt":5.79322,"cost_basis":"Vals.ai cost_per_test (mean per task attempt)","tasks":66,"attempts":198,"k":3,"date":"2026-10-01","run":"Vals.ai run: mini-swe-agent on Harbor/Daytona, avg@3 (three full pass@1 runs); reasoning effort not published; some rows include provider fallbacks/refusals noted on the page.","source_kind":"leaderboard","sources":[{"label":"Vals.ai results for GPT-6 Sol","url":"https://www.vals.ai/models/openai_gpt-6-sol"},{"label":"Vals.ai Terminal-Bench 4.0","url":"https://www.vals.ai/benchmarks/terminal-bench-4"}],"cost_estimated":false},{"id":"aa-7a0dfd2a-tb4","benchmark":"tb4","harness":"Codex","model":"GPT-6 Sol","effort":"max","score":0.43434,"score_published":"43.4%","ci95":null,"cost_per_attempt":4.98319,"cost_basis":"ESTIMATE: AA publishes only the row's mean cost, $2.99 per attempt; allocated 36.4% to this benchmark by attempts x (input x (1 - 0.9 x cacheHitRate 0.96) + 5 x output tokens), as a share of that mean","cost_estimated":true,"tasks":66,"attempts":198,"k":3,"date":null,"run":"Artificial Analysis per-benchmark mean reward for 'Codex - GPT-6 Sol (max)', 198 attempts. No run date published.","source_kind":"aa","sources":[{"label":"Artificial Analysis Coding Agents (no per-run page)","url":"https://artificialanalysis.ai/agents/coding-agents"}]},{"id":"hub-b57a324c","benchmark":"tb4","harness":"Claude Code","harness_version":"2.1.273","model":"Claude Fable 5.1","effort":"low","score":0.43333,"score_published":"143/330","ci95":[0.38094,0.48726],"cost_per_attempt":7.14765,"cost_basis":"Harbor trial cost_usd, mean over 330 of 330 graded attempts","tasks":66,"attempts":330,"k":5,"date":"2026-09-17","run":"Harbor job 'Terminal-Bench 4.0.0 · Fable 5.1 / Claude Code · reasoning effort low' by Terminal-Bench; 66 tasks x 5, 330 graded attempts (reward>=1 is a pass, timeouts score 0); Claude Code 2.1.273; dataset ref sha256:39d9f44b4042.","source_kind":"harbor","sources":[{"label":"Harbor Hub job b57a324c","url":"https://hub.harborframework.com/jobs/b57a324c-8076-4954-b452-1121c68b3495"},{"label":"Terminal-Bench 4.0 leaderboard row","url":"https://hub.harborframework.com/datasets/terminal-bench/terminal-bench/4/leaderboards/4-0-0/rows/e18d02ab-0973-4d86-872f-a353c8a7f344"}],"run_ref":"Job b57a324c-8076-4954-b452-1121c68b3495","cost_estimated":false},{"id":"hub-18e4ee29","benchmark":"tb4","harness":"Claude Code","harness_version":"2.1.207","model":"GLM-5.3","effort":"max","score":0.41945,"score_published":"138/329","ci95":[0.36736,0.4734],"cost_per_attempt":8.29066,"cost_basis":"Harbor trial cost_usd, mean over 329 of 329 graded attempts","tasks":66,"attempts":329,"k":5,"date":"2026-08-27","run":"Harbor job 'TB 4.0.0 - GLM-5.3 / Claude Code' by Ryan Marten on Docker; 66 tasks x 5, 329 graded attempts (reward>=1 is a pass, timeouts score 0); Claude Code 2.1.207; dataset ref 4.","source_kind":"harbor","sources":[{"label":"Harbor Hub job 18e4ee29","url":"https://hub.harborframework.com/jobs/18e4ee29-d8b4-516b-b79d-b8bad5bcd5be"},{"label":"Terminal-Bench 4.0 leaderboard row","url":"https://hub.harborframework.com/datasets/terminal-bench/terminal-bench/4/leaderboards/4-0-0/rows/d72e8775-f4f1-4313-99e7-35b1cb499f24"}],"run_ref":"Job 18e4ee29-d8b4-516b-b79d-b8bad5bcd5be","cost_estimated":false},{"id":"aa-33e80f38-tb4","benchmark":"tb4","harness":"Claude Code","model":"Claude Sonnet 5.5","effort":"high","score":0.41919,"score_published":"41.9%","ci95":null,"cost_per_attempt":2.03398,"cost_basis":"ESTIMATE: AA publishes only the row's mean cost, $1.24 per attempt; allocated 35.9% to this benchmark by attempts x (input x (1 - 0.9 x cacheHitRate 0.95) + 5 x output tokens), as a share of that mean","cost_estimated":true,"tasks":66,"attempts":198,"k":3,"date":null,"run":"Artificial Analysis per-benchmark mean reward for 'Claude Code - Sonnet 5.5 (high)', 198 attempts. No run date published.","source_kind":"aa","sources":[{"label":"Artificial Analysis Coding Agents (no per-run page)","url":"https://artificialanalysis.ai/agents/coding-agents"}]},{"id":"vals-claude-fable-5","benchmark":"tb4","harness":"mini-swe-agent","model":"Claude Fable 5","effort":null,"score":0.41414,"score_published":"41.41%","ci95":null,"cost_per_attempt":30.33976,"cost_basis":"Vals.ai cost_per_test (mean per task attempt)","tasks":66,"attempts":198,"k":3,"date":"2026-10-01","run":"Vals.ai run: mini-swe-agent on Harbor/Daytona, avg@3 (three full pass@1 runs); reasoning effort not published; some rows include provider fallbacks/refusals noted on the page.","source_kind":"leaderboard","sources":[{"label":"Vals.ai results for Claude Fable 5","url":"https://www.vals.ai/models/anthropic_claude-fable-5"},{"label":"Vals.ai Terminal-Bench 4.0","url":"https://www.vals.ai/benchmarks/terminal-bench-4"}],"cost_estimated":false},{"id":"aa-45493c27-tb4","benchmark":"tb4","harness":"Opencode","model":"GLM-5.3","effort":"max","score":0.39899,"score_published":"39.9%","ci95":null,"cost_per_attempt":7.78812,"cost_basis":"ESTIMATE: AA publishes only the row's mean cost, $4.24 per attempt; allocated 40.0% to this benchmark by attempts x (input x (1 - 0.9 x cacheHitRate 0.97) + 5 x output tokens), as a share of that mean","cost_estimated":true,"tasks":66,"attempts":198,"k":3,"date":null,"run":"Artificial Analysis per-benchmark mean reward for 'Opencode - GLM-5.3', 198 attempts. No run date published.","source_kind":"aa","sources":[{"label":"Artificial Analysis Coding Agents (no per-run page)","url":"https://artificialanalysis.ai/agents/coding-agents"}]},{"id":"vals-glm-5.3","benchmark":"tb4","harness":"mini-swe-agent","model":"GLM-5.3","effort":null,"score":0.38889,"score_published":"38.89%","ci95":null,"cost_per_attempt":9.37436,"cost_basis":"Vals.ai cost_per_test (mean per task attempt)","tasks":66,"attempts":198,"k":3,"date":"2026-10-01","run":"Vals.ai run: mini-swe-agent on Harbor/Daytona, avg@3 (three full pass@1 runs); reasoning effort not published; some rows include provider fallbacks/refusals noted on the page.","source_kind":"leaderboard","sources":[{"label":"Vals.ai results for GLM-5.3","url":"https://www.vals.ai/models/zai_glm-5.3"},{"label":"Vals.ai Terminal-Bench 4.0","url":"https://www.vals.ai/benchmarks/terminal-bench-4"}],"cost_estimated":false},{"id":"vals-gpt-5.6-sol","benchmark":"tb4","harness":"mini-swe-agent","model":"GPT-5.6 Sol","effort":null,"score":0.37879,"score_published":"37.88%","ci95":null,"cost_per_attempt":7.9836,"cost_basis":"Vals.ai cost_per_test (mean per task attempt)","tasks":66,"attempts":198,"k":3,"date":"2026-10-01","run":"Vals.ai run: mini-swe-agent on Harbor/Daytona, avg@3 (three full pass@1 runs); reasoning effort not published; some rows include provider fallbacks/refusals noted on the page.","source_kind":"leaderboard","sources":[{"label":"Vals.ai results for GPT-5.6 Sol","url":"https://www.vals.ai/models/openai_gpt-5.6-sol"},{"label":"Vals.ai Terminal-Bench 4.0","url":"https://www.vals.ai/benchmarks/terminal-bench-4"}],"cost_estimated":false},{"id":"hub-30225ce2","benchmark":"tb4","harness":"Grok Build","harness_version":"1.0.34","model":"Grok 4.7","effort":"xhigh","score":0.37576,"score_published":"124/330","ci95":[0.32521,0.42916],"cost_per_attempt":11.36817,"cost_basis":"Harbor trial cost_usd, mean over 324 of 330 graded attempts","tasks":66,"attempts":330,"k":5,"date":"2026-09-20","run":"Harbor job 'tb4-grok-4.7-xhigh' by Harbor; 66 tasks x 5, 330 graded attempts (reward>=1 is a pass, timeouts score 0); Grok Build 1.0.34; dataset ref sha256:39d9f44b4042.","source_kind":"harbor","sources":[{"label":"Harbor Hub job 30225ce2","url":"https://hub.harborframework.com/jobs/30225ce2-25f4-44c4-9fc5-67bf80dd4d17"},{"label":"Terminal-Bench 4.0 leaderboard row","url":"https://hub.harborframework.com/datasets/terminal-bench/terminal-bench/4/leaderboards/4-0-0/rows/84b39f56-fe3c-46c3-919f-3b67b73f9b49"}],"run_ref":"Job 30225ce2-25f4-44c4-9fc5-67bf80dd4d17","cost_estimated":false},{"id":"hub-5aebbab5","benchmark":"tb4","harness":"Codex","harness_version":"0.149.1","model":"GPT-5.6 Sol","effort":"max","score":0.37273,"score_published":"123/330","ci95":[0.3223,0.42608],"cost_per_attempt":7.87305,"cost_basis":"Harbor trial cost_usd, mean over 330 of 330 graded attempts","tasks":66,"attempts":330,"k":5,"date":"2026-08-27","run":"Harbor job 'Terminal-Bench 4.0.0 · GPT-5.6 Sol / Codex' by Terminal-Bench on Docker; 66 tasks x 5, 330 graded attempts (reward>=1 is a pass, timeouts score 0); Codex 0.149.1; dataset ref 4.","source_kind":"harbor","sources":[{"label":"Harbor Hub job 5aebbab5","url":"https://hub.harborframework.com/jobs/5aebbab5-1f8f-4cdb-8073-c6ce785a930d"},{"label":"Terminal-Bench 4.0 leaderboard row","url":"https://hub.harborframework.com/datasets/terminal-bench/terminal-bench/4/leaderboards/4-0-0/rows/0e349bde-b264-494d-a853-fada9c696192"}],"run_ref":"Job 5aebbab5-1f8f-4cdb-8073-c6ce785a930d","cost_estimated":false},{"id":"hub-7595dfd2","benchmark":"tb4","harness":"Claude Code","harness_version":"2.1.273","model":"Claude Opus 5","effort":"low","score":0.34848,"score_published":"115/330","ci95":[0.29909,0.40137],"cost_per_attempt":7.29841,"cost_basis":"Harbor trial cost_usd, mean over 328 of 330 graded attempts","tasks":66,"attempts":330,"k":5,"date":"2026-09-17","run":"Harbor job 'Terminal-Bench 4.0.0 · Opus 5 / Claude Code · reasoning effort low' by Terminal-Bench; 66 tasks x 5, 330 graded attempts (reward>=1 is a pass, timeouts score 0); Claude Code 2.1.273; dataset ref sha256:39d9f44b4042.","source_kind":"harbor","sources":[{"label":"Harbor Hub job 7595dfd2","url":"https://hub.harborframework.com/jobs/7595dfd2-22f0-4b03-81a9-c3c22b6ae047"},{"label":"Terminal-Bench 4.0 leaderboard row","url":"https://hub.harborframework.com/datasets/terminal-bench/terminal-bench/4/leaderboards/4-0-0/rows/03fe4f10-b683-427b-a820-61e38145d28b"}],"run_ref":"Job 7595dfd2-22f0-4b03-81a9-c3c22b6ae047","cost_estimated":false},{"id":"vals-qwen3.8-max","benchmark":"tb4","harness":"mini-swe-agent","model":"Qwen3.8 Max","effort":null,"score":0.34343,"score_published":"34.34%","ci95":null,"cost_per_attempt":10.63333,"cost_basis":"Vals.ai cost_per_test (mean per task attempt)","tasks":66,"attempts":198,"k":3,"date":"2026-10-01","run":"Vals.ai run: mini-swe-agent on Harbor/Daytona, avg@3 (three full pass@1 runs); reasoning effort not published; some rows include provider fallbacks/refusals noted on the page.","source_kind":"leaderboard","sources":[{"label":"Vals.ai results for Qwen3.8 Max","url":"https://www.vals.ai/models/alibaba_qwen3.8-max"},{"label":"Vals.ai Terminal-Bench 4.0","url":"https://www.vals.ai/benchmarks/terminal-bench-4"}],"cost_estimated":false},{"id":"aa-f2de0737-tb4","benchmark":"tb4","harness":"Muse Code","model":"Muse Spark 1.3","effort":"max","score":0.31818,"score_published":"31.8%","ci95":null,"cost_per_attempt":12.0982,"cost_basis":"ESTIMATE: AA publishes only the row's mean cost, $3.98 per attempt; allocated 66.3% to this benchmark by attempts x (input x (1 - 0.9 x cacheHitRate 0.97) + 5 x output tokens), as a share of that mean","cost_estimated":true,"tasks":66,"attempts":198,"k":3,"date":null,"run":"Artificial Analysis per-benchmark mean reward for 'Muse Code - Muse Spark 1.3 (max)', 198 attempts. No run date published.","source_kind":"aa","sources":[{"label":"Artificial Analysis Coding Agents (no per-run page)","url":"https://artificialanalysis.ai/agents/coding-agents"}]},{"id":"vals-step-5-preview","benchmark":"tb4","harness":"mini-swe-agent","model":"Step 5 Preview","effort":null,"score":0.31818,"score_published":"31.82%","ci95":null,"cost_per_attempt":3.11322,"cost_basis":"Vals.ai cost_per_test (mean per task attempt)","tasks":66,"attempts":198,"k":3,"date":"2026-10-01","run":"Vals.ai run: mini-swe-agent on Harbor/Daytona, avg@3 (three full pass@1 runs); reasoning effort not published; some rows include provider fallbacks/refusals noted on the page.","source_kind":"leaderboard","sources":[{"label":"Vals.ai results for Step 5 Preview","url":"https://www.vals.ai/models/stepfun_step-5-preview"},{"label":"Vals.ai Terminal-Bench 4.0","url":"https://www.vals.ai/benchmarks/terminal-bench-4"}],"cost_estimated":false},{"id":"vals-mimo-v2.6-pro","benchmark":"tb4","harness":"mini-swe-agent","model":"MiMo V2.6 Pro","effort":null,"score":0.31313,"score_published":"31.31%","ci95":null,"cost_per_attempt":0.4975,"cost_basis":"Vals.ai cost_per_test (mean per task attempt)","tasks":66,"attempts":198,"k":3,"date":"2026-10-01","run":"Vals.ai run: mini-swe-agent on Harbor/Daytona, avg@3 (three full pass@1 runs); reasoning effort not published; some rows include provider fallbacks/refusals noted on the page.","source_kind":"leaderboard","sources":[{"label":"Vals.ai results for MiMo V2.6 Pro","url":"https://www.vals.ai/models/xiaomi_mimo-v2.6-pro"},{"label":"Vals.ai Terminal-Bench 4.0","url":"https://www.vals.ai/benchmarks/terminal-bench-4"}],"cost_estimated":false},{"id":"vals-grok-4.7","benchmark":"tb4","harness":"mini-swe-agent","model":"Grok 4.7","effort":null,"score":0.28788,"score_published":"28.79%","ci95":null,"cost_per_attempt":18.0916,"cost_basis":"Vals.ai cost_per_test (mean per task attempt)","tasks":66,"attempts":198,"k":3,"date":"2026-10-01","run":"Vals.ai run: mini-swe-agent on Harbor/Daytona, avg@3 (three full pass@1 runs); reasoning effort not published; some rows include provider fallbacks/refusals noted on the page.","source_kind":"leaderboard","sources":[{"label":"Vals.ai results for Grok 4.7","url":"https://www.vals.ai/models/grok_grok-4.7"},{"label":"Vals.ai Terminal-Bench 4.0","url":"https://www.vals.ai/benchmarks/terminal-bench-4"}],"cost_estimated":false},{"id":"aa-4d6aa9e9-tb4","benchmark":"tb4","harness":"Claude Code","model":"Claude Sonnet 5.5","effort":"medium","score":0.27273,"score_published":"27.3%","ci95":null,"cost_per_attempt":1.32135,"cost_basis":"ESTIMATE: AA publishes only the row's mean cost, $0.62 per attempt; allocated 46.5% to this benchmark by attempts x (input x (1 - 0.9 x cacheHitRate 0.93) + 5 x output tokens), as a share of that mean","cost_estimated":true,"tasks":66,"attempts":198,"k":3,"date":null,"run":"Artificial Analysis per-benchmark mean reward for 'Claude Code - Sonnet 5.5 (medium)', 198 attempts. No run date published.","source_kind":"aa","sources":[{"label":"Artificial Analysis Coding Agents (no per-run page)","url":"https://artificialanalysis.ai/agents/coding-agents"}]},{"id":"aa-e9f9d3ae-tb4","benchmark":"tb4","harness":"Claude Code","model":"Claude Sonnet 5.5","effort":"low","score":0.25253,"score_published":"25.3%","ci95":null,"cost_per_attempt":1.08415,"cost_basis":"ESTIMATE: AA publishes only the row's mean cost, $0.48 per attempt; allocated 48.9% to this benchmark by attempts x (input x (1 - 0.9 x cacheHitRate 0.93) + 5 x output tokens), as a share of that mean","cost_estimated":true,"tasks":66,"attempts":198,"k":3,"date":null,"run":"Artificial Analysis per-benchmark mean reward for 'Claude Code - Sonnet 5.5 (low)', 198 attempts. No run date published.","source_kind":"aa","sources":[{"label":"Artificial Analysis Coding Agents (no per-run page)","url":"https://artificialanalysis.ai/agents/coding-agents"}]},{"id":"vals-muse_spark_1_3_max","benchmark":"tb4","harness":"mini-swe-agent","model":"Muse Spark 1.3","effort":"max","score":0.24747,"score_published":"24.75%","ci95":null,"cost_per_attempt":6.65456,"cost_basis":"Vals.ai cost_per_test (mean per task attempt)","tasks":66,"attempts":198,"k":3,"date":"2026-10-01","run":"Vals.ai run: mini-swe-agent on Harbor/Daytona, avg@3 (three full pass@1 runs); reasoning effort not published; some rows include provider fallbacks/refusals noted on the page.","source_kind":"leaderboard","sources":[{"label":"Vals.ai results for Muse Spark 1.3","url":"https://www.vals.ai/models/meta_muse_spark_1_3_max"},{"label":"Vals.ai Terminal-Bench 4.0","url":"https://www.vals.ai/benchmarks/terminal-bench-4"}],"cost_estimated":false},{"id":"hub-01fe8c25","benchmark":"tb4","harness":"Ante","harness_version":"0.2.7","model":"DeepSeek V4.1 Flash","effort":null,"score":0.24242,"score_published":"16/66","ci95":[0.15509,0.35809],"cost_per_attempt":null,"cost_basis":"no cost recorded on Harbor trials","tasks":66,"attempts":66,"k":1,"date":"2026-09-30","run":"Harbor job '2026-09-30__05-23-17' by leofoo on Modal; 66 tasks x 1, 66 graded attempts (reward>=1 is a pass, timeouts score 0); Ante 0.2.7; dataset ref sha256:39d9f44b4042.","source_kind":"harbor","sources":[{"label":"Harbor Hub job 01fe8c25","url":"https://hub.harborframework.com/jobs/01fe8c25-9fa7-4d98-8715-512c553fc18a"}],"run_ref":"Job 01fe8c25-9fa7-4d98-8715-512c553fc18a","cost_estimated":false},{"id":"vals-mimo-v2.6-flash","benchmark":"tb4","harness":"mini-swe-agent","model":"MiMo V2.6 Flash","effort":null,"score":0.24242,"score_published":"24.24%","ci95":null,"cost_per_attempt":0.21627,"cost_basis":"Vals.ai cost_per_test (mean per task attempt)","tasks":66,"attempts":198,"k":3,"date":"2026-10-01","run":"Vals.ai run: mini-swe-agent on Harbor/Daytona, avg@3 (three full pass@1 runs); reasoning effort not published; some rows include provider fallbacks/refusals noted on the page.","source_kind":"leaderboard","sources":[{"label":"Vals.ai results for MiMo V2.6 Flash","url":"https://www.vals.ai/models/xiaomi_mimo-v2.6-flash"},{"label":"Vals.ai Terminal-Bench 4.0","url":"https://www.vals.ai/benchmarks/terminal-bench-4"}],"cost_estimated":false},{"id":"hub-10722b22","benchmark":"tb4","harness":"Claude Code","harness_version":"2.1.231","model":"Claude Opus 4.8","effort":"max","score":0.23636,"score_published":"78/330","ci95":[0.19372,0.28507],"cost_per_attempt":19.6402,"cost_basis":"Harbor trial cost_usd, mean over 330 of 330 graded attempts","tasks":66,"attempts":330,"k":5,"date":"2026-08-27","run":"Harbor job 'Terminal-Bench 4.0.0 · Opus 4.8 / Claude Code' by Terminal-Bench on Docker; 66 tasks x 5, 330 graded attempts (reward>=1 is a pass, timeouts score 0); Claude Code 2.1.231; dataset ref 4.","source_kind":"harbor","sources":[{"label":"Harbor Hub job 10722b22","url":"https://hub.harborframework.com/jobs/10722b22-0bbd-4c93-8d27-611fb8dd02a6"},{"label":"Terminal-Bench 4.0 leaderboard row","url":"https://hub.harborframework.com/datasets/terminal-bench/terminal-bench/4/leaderboards/4-0-0/rows/952b4217-421f-4d14-849b-9968fcb2063b"}],"run_ref":"Job 10722b22-0bbd-4c93-8d27-611fb8dd02a6","cost_estimated":false},{"id":"vals-claude-opus-4-8","benchmark":"tb4","harness":"mini-swe-agent","model":"Claude Opus 4.8","effort":null,"score":0.23232,"score_published":"23.23%","ci95":null,"cost_per_attempt":17.14072,"cost_basis":"Vals.ai cost_per_test (mean per task attempt)","tasks":66,"attempts":198,"k":3,"date":"2026-10-01","run":"Vals.ai run: mini-swe-agent on Harbor/Daytona, avg@3 (three full pass@1 runs); reasoning effort not published; some rows include provider fallbacks/refusals noted on the page.","source_kind":"leaderboard","sources":[{"label":"Vals.ai results for Claude Opus 4.8","url":"https://www.vals.ai/models/anthropic_claude-opus-4-8"},{"label":"Vals.ai Terminal-Bench 4.0","url":"https://www.vals.ai/benchmarks/terminal-bench-4"}],"cost_estimated":false},{"id":"vals-gpt-5.6-terra","benchmark":"tb4","harness":"mini-swe-agent","model":"GPT-5.6 Terra","effort":null,"score":0.22727,"score_published":"22.73%","ci95":null,"cost_per_attempt":5.60212,"cost_basis":"Vals.ai cost_per_test (mean per task attempt)","tasks":66,"attempts":198,"k":3,"date":"2026-10-01","run":"Vals.ai run: mini-swe-agent on Harbor/Daytona, avg@3 (three full pass@1 runs); reasoning effort not published; some rows include provider fallbacks/refusals noted on the page.","source_kind":"leaderboard","sources":[{"label":"Vals.ai results for GPT-5.6 Terra","url":"https://www.vals.ai/models/openai_gpt-5.6-terra"},{"label":"Vals.ai Terminal-Bench 4.0","url":"https://www.vals.ai/benchmarks/terminal-bench-4"}],"cost_estimated":false},{"id":"hub-3de0aee9","benchmark":"tb4","harness":"Codex","harness_version":"0.149.1","model":"GPT-5.6 Terra","effort":"max","score":0.21581,"score_published":"71/329","ci95":[0.17477,0.2634],"cost_per_attempt":5.26906,"cost_basis":"Harbor trial cost_usd, mean over 329 of 329 graded attempts","tasks":66,"attempts":329,"k":5,"date":"2026-08-27","run":"Harbor job 'Terminal-Bench 4.0.0 · GPT-5.6 Terra / Codex' by Terminal-Bench on Docker; 66 tasks x 5, 329 graded attempts (reward>=1 is a pass, timeouts score 0); Codex 0.149.1; dataset ref 4.","source_kind":"harbor","sources":[{"label":"Harbor Hub job 3de0aee9","url":"https://hub.harborframework.com/jobs/3de0aee9-6de7-4d94-83c4-ac5e789fe757"},{"label":"Terminal-Bench 4.0 leaderboard row","url":"https://hub.harborframework.com/datasets/terminal-bench/terminal-bench/4/leaderboards/4-0-0/rows/e53da412-5e92-408c-b369-c767924c7c1c"}],"run_ref":"Job 3de0aee9-6de7-4d94-83c4-ac5e789fe757","cost_estimated":false},{"id":"aa-55b029d6-tb4","benchmark":"tb4","harness":"Kimi Code CLI","model":"Kimi K3","effort":null,"score":0.21212,"score_published":"21.2%","ci95":null,"cost_per_attempt":9.59324,"cost_basis":"ESTIMATE: AA publishes only the row's mean cost, $5.05 per attempt; allocated 41.4% to this benchmark by attempts x (input x (1 - 0.9 x cacheHitRate 0.95) + 5 x output tokens), as a share of that mean","cost_estimated":true,"tasks":66,"attempts":198,"k":3,"date":null,"run":"Artificial Analysis per-benchmark mean reward for 'Kimi Code CLI - Kimi K3', 198 attempts. No run date published.","source_kind":"aa","sources":[{"label":"Artificial Analysis Coding Agents (no per-run page)","url":"https://artificialanalysis.ai/agents/coding-agents"}]},{"id":"hub-2c89a14f","benchmark":"tb4","harness":"Grok Build","harness_version":"1.0.5","model":"Grok 4.6","effort":"high","score":0.20365,"score_published":"67/329","ci95":[0.16367,0.25047],"cost_per_attempt":11.00578,"cost_basis":"Harbor trial cost_usd, mean over 326 of 329 graded attempts","tasks":66,"attempts":329,"k":5,"date":"2026-08-27","run":"Harbor job 'Terminal-Bench 4.0.0 · Grok 4.6 / Grok Build' by Terminal-Bench; 66 tasks x 5, 329 graded attempts (reward>=1 is a pass, timeouts score 0); Grok Build 1.0.5; dataset ref 4.","source_kind":"harbor","sources":[{"label":"Harbor Hub job 2c89a14f","url":"https://hub.harborframework.com/jobs/2c89a14f-d14a-4ea8-ad8a-d08d90c67a5d"},{"label":"Terminal-Bench 4.0 leaderboard row","url":"https://hub.harborframework.com/datasets/terminal-bench/terminal-bench/4/leaderboards/4-0-0/rows/26354542-edc0-40cd-8f8d-9fe6fbe92ac3"}],"run_ref":"Job 2c89a14f-d14a-4ea8-ad8a-d08d90c67a5d","cost_estimated":false},{"id":"vals-deepseek-v4.1-flash","benchmark":"tb4","harness":"mini-swe-agent","model":"DeepSeek V4.1 Flash","effort":null,"score":0.19697,"score_published":"19.70%","ci95":null,"cost_per_attempt":0.49947,"cost_basis":"Vals.ai cost_per_test (mean per task attempt)","tasks":66,"attempts":198,"k":3,"date":"2026-10-01","run":"Vals.ai run: mini-swe-agent on Harbor/Daytona, avg@3 (three full pass@1 runs); reasoning effort not published; some rows include provider fallbacks/refusals noted on the page.","source_kind":"leaderboard","sources":[{"label":"Vals.ai results for DeepSeek V4.1 Flash","url":"https://www.vals.ai/models/deepseek_deepseek-v4.1-flash"},{"label":"Vals.ai Terminal-Bench 4.0","url":"https://www.vals.ai/benchmarks/terminal-bench-4"}],"cost_estimated":false},{"id":"vals-ember-1","benchmark":"tb4","harness":"mini-swe-agent","model":"Ember-1","effort":null,"score":0.19697,"score_published":"19.70%","ci95":null,"cost_per_attempt":18.85572,"cost_basis":"Vals.ai cost_per_test (mean per task attempt)","tasks":66,"attempts":198,"k":3,"date":"2026-10-01","run":"Vals.ai run: mini-swe-agent on Harbor/Daytona, avg@3 (three full pass@1 runs); reasoning effort not published; some rows include provider fallbacks/refusals noted on the page.","source_kind":"leaderboard","sources":[{"label":"Vals.ai results for Ember-1","url":"https://www.vals.ai/models/fireworks_ember-1"},{"label":"Vals.ai Terminal-Bench 4.0","url":"https://www.vals.ai/benchmarks/terminal-bench-4"}],"cost_estimated":false},{"id":"vals-gemini-3.8-flash","benchmark":"tb4","harness":"mini-swe-agent","model":"Gemini 3.8 Flash","effort":null,"score":0.19192,"score_published":"19.19%","ci95":null,"cost_per_attempt":8.77433,"cost_basis":"Vals.ai cost_per_test (mean per task attempt)","tasks":66,"attempts":198,"k":3,"date":"2026-10-01","run":"Vals.ai run: mini-swe-agent on Harbor/Daytona, avg@3 (three full pass@1 runs); reasoning effort not published; some rows include provider fallbacks/refusals noted on the page.","source_kind":"leaderboard","sources":[{"label":"Vals.ai results for Gemini 3.8 Flash","url":"https://www.vals.ai/models/google_gemini-3.8-flash"},{"label":"Vals.ai Terminal-Bench 4.0","url":"https://www.vals.ai/benchmarks/terminal-bench-4"}],"cost_estimated":false},{"id":"hub-0e1a4181","benchmark":"tb4","harness":"mini-swe-agent","harness_version":"2.4.6","model":"Gemini 3.8 Flash","effort":"high","score":0.19091,"score_published":"63/330","ci95":[0.15216,0.23677],"cost_per_attempt":5.54172,"cost_basis":"Harbor trial cost_usd, mean over 330 of 330 graded attempts","tasks":66,"attempts":330,"k":5,"date":"2026-09-01","run":"Harbor job 'Terminal-Bench 4.0.0 · Gemini 3.8 Flash / mini-SWE-agent' by Terminal-Bench; 66 tasks x 5, 330 graded attempts (reward>=1 is a pass, timeouts score 0); mini-swe-agent 2.4.6; dataset ref v4.0.0.","source_kind":"harbor","sources":[{"label":"Harbor Hub job 0e1a4181","url":"https://hub.harborframework.com/jobs/0e1a4181-26a9-42de-9d5a-6a36515f13e2"},{"label":"Terminal-Bench 4.0 leaderboard row","url":"https://hub.harborframework.com/datasets/terminal-bench/terminal-bench/4/leaderboards/4-0-0/rows/06850434-507d-4dbd-b74a-09d52519ee35"}],"run_ref":"Job 0e1a4181-26a9-42de-9d5a-6a36515f13e2","cost_estimated":false},{"id":"vals-deepseek-v4-flash-0731","benchmark":"tb4","harness":"mini-swe-agent","model":"DeepSeek V4 Flash 0731","effort":null,"score":0.18687,"score_published":"18.69%","ci95":null,"cost_per_attempt":0.76647,"cost_basis":"Vals.ai cost_per_test (mean per task attempt)","tasks":66,"attempts":198,"k":3,"date":"2026-10-01","run":"Vals.ai run: mini-swe-agent on Harbor/Daytona, avg@3 (three full pass@1 runs); reasoning effort not published; some rows include provider fallbacks/refusals noted on the page.","source_kind":"leaderboard","sources":[{"label":"Vals.ai results for DeepSeek V4 Flash 0731","url":"https://www.vals.ai/models/deepseek_deepseek-v4-flash-0731"},{"label":"Vals.ai Terminal-Bench 4.0","url":"https://www.vals.ai/benchmarks/terminal-bench-4"}],"cost_estimated":false},{"id":"aa-70a5d300-tb4","benchmark":"tb4","harness":"Grok Build","model":"Grok 4.6","effort":"xhigh","score":0.17677,"score_published":"17.7%","ci95":null,"cost_per_attempt":5.68187,"cost_basis":"ESTIMATE: AA publishes only the row's mean cost, $3.57 per attempt; allocated 34.7% to this benchmark by attempts x (input x (1 - 0.9 x cacheHitRate 0.91) + 5 x output tokens), as a share of that mean","cost_estimated":true,"tasks":66,"attempts":198,"k":3,"date":null,"run":"Artificial Analysis per-benchmark mean reward for 'Grok Build - Grok 4.6 (xhigh)', 198 attempts. No run date published.","source_kind":"aa","sources":[{"label":"Artificial Analysis Coding Agents (no per-run page)","url":"https://artificialanalysis.ai/agents/coding-agents"}]},{"id":"hub-b7444ba2","benchmark":"tb4","harness":"Codex","harness_version":"0.149.1","model":"GPT-5.6 Luna","effort":"max","score":0.17273,"score_published":"57/330","ci95":[0.13577,0.21722],"cost_per_attempt":1.05372,"cost_basis":"Harbor trial cost_usd, mean over 329 of 330 graded attempts","tasks":66,"attempts":330,"k":5,"date":"2026-08-27","run":"Harbor job 'Terminal-Bench 4.0.0 · GPT-5.6 Luna / Codex' by Terminal-Bench on Docker; 66 tasks x 5, 330 graded attempts (reward>=1 is a pass, timeouts score 0); Codex 0.149.1; dataset ref 4.","source_kind":"harbor","sources":[{"label":"Harbor Hub job b7444ba2","url":"https://hub.harborframework.com/jobs/b7444ba2-78b7-46a0-a3c1-2196d40a337d"},{"label":"Terminal-Bench 4.0 leaderboard row","url":"https://hub.harborframework.com/datasets/terminal-bench/terminal-bench/4/leaderboards/4-0-0/rows/51c6d76e-5baa-48c1-97b3-88656c620eef"}],"run_ref":"Job b7444ba2-78b7-46a0-a3c1-2196d40a337d","cost_estimated":false},{"id":"vals-grok-4.6","benchmark":"tb4","harness":"mini-swe-agent","model":"Grok 4.6","effort":null,"score":0.17172,"score_published":"17.17%","ci95":null,"cost_per_attempt":5.07111,"cost_basis":"Vals.ai cost_per_test (mean per task attempt)","tasks":66,"attempts":198,"k":3,"date":"2026-10-01","run":"Vals.ai run: mini-swe-agent on Harbor/Daytona, avg@3 (three full pass@1 runs); reasoning effort not published; some rows include provider fallbacks/refusals noted on the page.","source_kind":"leaderboard","sources":[{"label":"Vals.ai results for Grok 4.6","url":"https://www.vals.ai/models/grok_grok-4.6"},{"label":"Vals.ai Terminal-Bench 4.0","url":"https://www.vals.ai/benchmarks/terminal-bench-4"}],"cost_estimated":false},{"id":"vals-kimi-k3","benchmark":"tb4","harness":"mini-swe-agent","model":"Kimi K3","effort":null,"score":0.17172,"score_published":"17.17%","ci95":null,"cost_per_attempt":12.02093,"cost_basis":"Vals.ai cost_per_test (mean per task attempt)","tasks":66,"attempts":198,"k":3,"date":"2026-10-01","run":"Vals.ai run: mini-swe-agent on Harbor/Daytona, avg@3 (three full pass@1 runs); reasoning effort not published; some rows include provider fallbacks/refusals noted on the page.","source_kind":"leaderboard","sources":[{"label":"Vals.ai results for Kimi K3","url":"https://www.vals.ai/models/kimi_kimi-k3"},{"label":"Vals.ai Terminal-Bench 4.0","url":"https://www.vals.ai/benchmarks/terminal-bench-4"}],"cost_estimated":false},{"id":"aa-cd60470c-tb4","benchmark":"tb4","harness":"Muse Code","model":"Muse Spark 1.3","effort":"xhigh","score":0.17172,"score_published":"17.2%","ci95":null,"cost_per_attempt":9.54514,"cost_basis":"ESTIMATE: AA publishes only the row's mean cost, $3.47 per attempt; allocated 60.0% to this benchmark by attempts x (input x (1 - 0.9 x cacheHitRate 0.97) + 5 x output tokens), as a share of that mean","cost_estimated":true,"tasks":66,"attempts":198,"k":3,"date":null,"run":"Artificial Analysis per-benchmark mean reward for 'Muse Code - Muse Spark 1.3 (xhigh)', 198 attempts. No run date published.","source_kind":"aa","sources":[{"label":"Artificial Analysis Coding Agents (no per-run page)","url":"https://artificialanalysis.ai/agents/coding-agents"}]},{"id":"aa-cd4c865d-tb4","benchmark":"tb4","harness":"Claude Code","model":"Qwen3.8 Max","effort":null,"score":0.16667,"score_published":"16.7%","ci95":null,"cost_per_attempt":5.62988,"cost_basis":"ESTIMATE: AA publishes only the row's mean cost, $3.48 per attempt; allocated 35.2% to this benchmark by attempts x (input x (1 - 0.9 x cacheHitRate 0.95) + 5 x output tokens), as a share of that mean","cost_estimated":true,"tasks":66,"attempts":198,"k":3,"date":null,"run":"Artificial Analysis per-benchmark mean reward for 'Claude Code - Qwen3.8 Max', 198 attempts. No run date published.","source_kind":"aa","sources":[{"label":"Artificial Analysis Coding Agents (no per-run page)","url":"https://artificialanalysis.ai/agents/coding-agents"}]},{"id":"hub-e2b5ae5f","benchmark":"tb4","harness":"Ante","harness_version":"0.2.6","model":"MiMo V2.6 Flash","effort":"max","score":0.16667,"score_published":"11/66","ci95":[0.0957,0.27431],"cost_per_attempt":null,"cost_basis":"no cost recorded on Harbor trials","tasks":66,"attempts":66,"k":1,"date":"2026-09-29","run":"Harbor job '2026-09-28__06-39-38' by leofoo on Modal; 66 tasks x 1, 66 graded attempts (reward>=1 is a pass, timeouts score 0); Ante 0.2.6; dataset ref sha256:39d9f44b4042.","source_kind":"harbor","sources":[{"label":"Harbor Hub job e2b5ae5f","url":"https://hub.harborframework.com/jobs/e2b5ae5f-b32f-4488-b0cc-d15a5959c3c5"}],"run_ref":"Job e2b5ae5f-b32f-4488-b0cc-d15a5959c3c5","cost_estimated":false},{"id":"aa-ac8276cd-tb4","benchmark":"tb4","harness":"Codex","model":"GPT-6 Luna","effort":"max","score":0.15152,"score_published":"15.2%","ci95":null,"cost_per_attempt":0.2624,"cost_basis":"ESTIMATE: AA publishes only the row's mean cost, $0.18 per attempt; allocated 32.5% to this benchmark by attempts x (input x (1 - 0.9 x cacheHitRate 0.96) + 5 x output tokens), as a share of that mean","cost_estimated":true,"tasks":66,"attempts":198,"k":3,"date":null,"run":"Artificial Analysis per-benchmark mean reward for 'Codex - GPT-6 Luna (max)', 198 attempts. No run date published.","source_kind":"aa","sources":[{"label":"Artificial Analysis Coding Agents (no per-run page)","url":"https://artificialanalysis.ai/agents/coding-agents"}]},{"id":"aa-d1003684-tb4","benchmark":"tb4","harness":"Antigravity SDK","model":"Gemini 3.8 Flash","effort":"high","score":0.14646,"score_published":"14.6%","ci95":null,"cost_per_attempt":3.12671,"cost_basis":"ESTIMATE: AA publishes only the row's mean cost, $2.47 per attempt; allocated 27.6% to this benchmark by attempts x (input x (1 - 0.9 x cacheHitRate 0.87) + 5 x output tokens), as a share of that mean","cost_estimated":true,"tasks":66,"attempts":198,"k":3,"date":null,"run":"Artificial Analysis per-benchmark mean reward for 'Antigravity SDK - Gemini 3.8 Flash (high)', 198 attempts. No run date published.","source_kind":"aa","sources":[{"label":"Artificial Analysis Coding Agents (no per-run page)","url":"https://artificialanalysis.ai/agents/coding-agents"}]},{"id":"vals-deepseek-v4-pro-0813","benchmark":"tb4","harness":"mini-swe-agent","model":"DeepSeek V4 Pro 0813","effort":null,"score":0.14141,"score_published":"14.14%","ci95":null,"cost_per_attempt":3.31413,"cost_basis":"Vals.ai cost_per_test (mean per task attempt)","tasks":66,"attempts":198,"k":3,"date":"2026-10-01","run":"Vals.ai run: mini-swe-agent on Harbor/Daytona, avg@3 (three full pass@1 runs); reasoning effort not published; some rows include provider fallbacks/refusals noted on the page.","source_kind":"leaderboard","sources":[{"label":"Vals.ai results for DeepSeek V4 Pro 0813","url":"https://www.vals.ai/models/deepseek_deepseek-v4-pro-0813"},{"label":"Vals.ai Terminal-Bench 4.0","url":"https://www.vals.ai/benchmarks/terminal-bench-4"}],"cost_estimated":false},{"id":"vals-gpt-6-luna","benchmark":"tb4","harness":"mini-swe-agent","model":"GPT-6 Luna","effort":null,"score":0.13636,"score_published":"13.64%","ci95":null,"cost_per_attempt":0.35473,"cost_basis":"Vals.ai cost_per_test (mean per task attempt)","tasks":66,"attempts":198,"k":3,"date":"2026-10-01","run":"Vals.ai run: mini-swe-agent on Harbor/Daytona, avg@3 (three full pass@1 runs); reasoning effort not published; some rows include provider fallbacks/refusals noted on the page.","source_kind":"leaderboard","sources":[{"label":"Vals.ai results for GPT-6 Luna","url":"https://www.vals.ai/models/openai_gpt-6-luna"},{"label":"Vals.ai Terminal-Bench 4.0","url":"https://www.vals.ai/benchmarks/terminal-bench-4"}],"cost_estimated":false},{"id":"hub-a5758f1a","benchmark":"tb4","harness":"Claude Code","harness_version":"2.1.231","model":"Claude Sonnet 5","effort":"max","score":0.12462,"score_published":"41/329","ci95":[0.09321,0.1647],"cost_per_attempt":29.19108,"cost_basis":"Harbor trial cost_usd, mean over 329 of 329 graded attempts","tasks":66,"attempts":329,"k":5,"date":"2026-08-27","run":"Harbor job 'Terminal-Bench 4.0.0 · Sonnet 5 / Claude Code' by Terminal-Bench on Docker; 66 tasks x 5, 329 graded attempts (reward>=1 is a pass, timeouts score 0); Claude Code 2.1.231; dataset ref 4.","source_kind":"harbor","sources":[{"label":"Harbor Hub job a5758f1a","url":"https://hub.harborframework.com/jobs/a5758f1a-9ef2-4100-9893-34a99c99bd9c"},{"label":"Terminal-Bench 4.0 leaderboard row","url":"https://hub.harborframework.com/datasets/terminal-bench/terminal-bench/4/leaderboards/4-0-0/rows/8180b9e4-4990-43d0-b906-cd8b43adaaa2"}],"run_ref":"Job a5758f1a-9ef2-4100-9893-34a99c99bd9c","cost_estimated":false},{"id":"hub-51a17348","benchmark":"tb4","harness":"Grok Build","harness_version":"1.0.5","model":"Grok 4.5","effort":"high","score":0.12424,"score_published":"41/330","ci95":[0.09292,0.16421],"cost_per_attempt":6.69044,"cost_basis":"Harbor trial cost_usd, mean over 313 of 330 graded attempts","tasks":66,"attempts":330,"k":5,"date":"2026-08-27","run":"Harbor job 'Terminal-Bench 4.0.0 · Grok 4.5 / Grok Build' by Terminal-Bench; 66 tasks x 5, 330 graded attempts (reward>=1 is a pass, timeouts score 0); Grok Build 1.0.5; dataset ref 4.","source_kind":"harbor","sources":[{"label":"Harbor Hub job 51a17348","url":"https://hub.harborframework.com/jobs/51a17348-a8b8-4c5e-b8d2-73a9a724c3b6"},{"label":"Terminal-Bench 4.0 leaderboard row","url":"https://hub.harborframework.com/datasets/terminal-bench/terminal-bench/4/leaderboards/4-0-0/rows/0eed5a0d-96b6-491b-aeb7-cd5e4700c2d7"}],"run_ref":"Job 51a17348-a8b8-4c5e-b8d2-73a9a724c3b6","cost_estimated":false},{"id":"vals-gemini-3.7-flash","benchmark":"tb4","harness":"mini-swe-agent","model":"Gemini 3.7 Flash","effort":null,"score":0.12121,"score_published":"12.12%","ci95":null,"cost_per_attempt":9.02268,"cost_basis":"Vals.ai cost_per_test (mean per task attempt)","tasks":66,"attempts":198,"k":3,"date":"2026-10-01","run":"Vals.ai run: mini-swe-agent on Harbor/Daytona, avg@3 (three full pass@1 runs); reasoning effort not published; some rows include provider fallbacks/refusals noted on the page.","source_kind":"leaderboard","sources":[{"label":"Vals.ai results for Gemini 3.7 Flash","url":"https://www.vals.ai/models/google_gemini-3.7-flash"},{"label":"Vals.ai Terminal-Bench 4.0","url":"https://www.vals.ai/benchmarks/terminal-bench-4"}],"cost_estimated":false},{"id":"vals-gpt-5.6-luna","benchmark":"tb4","harness":"mini-swe-agent","model":"GPT-5.6 Luna","effort":null,"score":0.11616,"score_published":"11.62%","ci95":null,"cost_per_attempt":0.73165,"cost_basis":"Vals.ai cost_per_test (mean per task attempt)","tasks":66,"attempts":198,"k":3,"date":"2026-10-01","run":"Vals.ai run: mini-swe-agent on Harbor/Daytona, avg@3 (three full pass@1 runs); reasoning effort not published; some rows include provider fallbacks/refusals noted on the page.","source_kind":"leaderboard","sources":[{"label":"Vals.ai results for GPT-5.6 Luna","url":"https://www.vals.ai/models/openai_gpt-5.6-luna"},{"label":"Vals.ai Terminal-Bench 4.0","url":"https://www.vals.ai/benchmarks/terminal-bench-4"}],"cost_estimated":false},{"id":"hub-2d418f81","benchmark":"tb4","harness":"mini-swe-agent","harness_version":"2.4.6","model":"Gemini 3.7 Flash","effort":"high","score":0.11212,"score_published":"37/330","ci95":[0.08245,0.15072],"cost_per_attempt":3.82385,"cost_basis":"Harbor trial cost_usd, mean over 330 of 330 graded attempts","tasks":66,"attempts":330,"k":5,"date":"2026-09-02","run":"Harbor job 'Terminal-Bench 4.0.0 · Gemini 3.7 Flash / mini-SWE-agent' by Terminal-Bench; 66 tasks x 5, 330 graded attempts (reward>=1 is a pass, timeouts score 0); mini-swe-agent 2.4.6; dataset ref sha256:39d9f44b4042.","source_kind":"harbor","sources":[{"label":"Harbor Hub job 2d418f81","url":"https://hub.harborframework.com/jobs/2d418f81-17d6-4f9c-8d16-d104f4ed6caa"},{"label":"Terminal-Bench 4.0 leaderboard row","url":"https://hub.harborframework.com/datasets/terminal-bench/terminal-bench/4/leaderboards/4-0-0/rows/14f4da14-b86a-4dca-92d7-11178d4fe064"}],"run_ref":"Job 2d418f81-17d6-4f9c-8d16-d104f4ed6caa","cost_estimated":false},{"id":"vals-deepseek-v4-pro","benchmark":"tb4","harness":"mini-swe-agent","model":"DeepSeek V4 Pro","effort":null,"score":0.11111,"score_published":"11.11%","ci95":null,"cost_per_attempt":3.37883,"cost_basis":"Vals.ai cost_per_test (mean per task attempt)","tasks":66,"attempts":198,"k":3,"date":"2026-10-01","run":"Vals.ai run: mini-swe-agent on Harbor/Daytona, avg@3 (three full pass@1 runs); reasoning effort not published; some rows include provider fallbacks/refusals noted on the page.","source_kind":"leaderboard","sources":[{"label":"Vals.ai results for DeepSeek V4 Pro","url":"https://www.vals.ai/models/deepseek_deepseek-v4-pro"},{"label":"Vals.ai Terminal-Bench 4.0","url":"https://www.vals.ai/benchmarks/terminal-bench-4"}],"cost_estimated":false},{"id":"aa-28ef6513-tb4","benchmark":"tb4","harness":"Codex","model":"DeepSeek V4 Flash 0731","effort":"max","score":0.10606,"score_published":"10.6%","ci95":null,"cost_per_attempt":0.13016,"cost_basis":"ESTIMATE: AA publishes only the row's mean cost, $0.09 per attempt; allocated 33.3% to this benchmark by attempts x (input x (1 - 0.9 x cacheHitRate 0.99) + 5 x output tokens), as a share of that mean","cost_estimated":true,"tasks":66,"attempts":198,"k":3,"date":null,"run":"Artificial Analysis per-benchmark mean reward for 'Codex - DeepSeek V4 Flash 0731 (max)', 198 attempts. No run date published.","source_kind":"aa","sources":[{"label":"Artificial Analysis Coding Agents (no per-run page)","url":"https://artificialanalysis.ai/agents/coding-agents"}]},{"id":"vals-muse_spark_1_3","benchmark":"tb4","harness":"mini-swe-agent","model":"Muse Spark 1.3","effort":null,"score":0.10606,"score_published":"10.61%","ci95":null,"cost_per_attempt":7.03647,"cost_basis":"Vals.ai cost_per_test (mean per task attempt)","tasks":66,"attempts":198,"k":3,"date":"2026-10-01","run":"Vals.ai run: mini-swe-agent on Harbor/Daytona, avg@3 (three full pass@1 runs); reasoning effort not published; some rows include provider fallbacks/refusals noted on the page.","source_kind":"leaderboard","sources":[{"label":"Vals.ai results for Muse Spark 1.3","url":"https://www.vals.ai/models/meta_muse_spark_1_3"},{"label":"Vals.ai Terminal-Bench 4.0","url":"https://www.vals.ai/benchmarks/terminal-bench-4"}],"cost_estimated":false},{"id":"aa-542590ec-tb4","benchmark":"tb4","harness":"Codex","model":"DeepSeek V4 Pro 0813","effort":"max","score":0.10101,"score_published":"10.1%","ci95":null,"cost_per_attempt":0.53279,"cost_basis":"ESTIMATE: AA publishes only the row's mean cost, $0.24 per attempt; allocated 48.8% to this benchmark by attempts x (input x (1 - 0.9 x cacheHitRate 0.99) + 5 x output tokens), as a share of that mean","cost_estimated":true,"tasks":66,"attempts":198,"k":3,"date":null,"run":"Artificial Analysis per-benchmark mean reward for 'Codex - DeepSeek V4 Pro 0813 (max)', 198 attempts. No run date published.","source_kind":"aa","sources":[{"label":"Artificial Analysis Coding Agents (no per-run page)","url":"https://artificialanalysis.ai/agents/coding-agents"}]},{"id":"vals-claude-sonnet-5","benchmark":"tb4","harness":"mini-swe-agent","model":"Claude Sonnet 5","effort":null,"score":0.09596,"score_published":"9.60%","ci95":null,"cost_per_attempt":26.32556,"cost_basis":"Vals.ai cost_per_test (mean per task attempt)","tasks":66,"attempts":198,"k":3,"date":"2026-10-01","run":"Vals.ai run: mini-swe-agent on Harbor/Daytona, avg@3 (three full pass@1 runs); reasoning effort not published; some rows include provider fallbacks/refusals noted on the page.","source_kind":"leaderboard","sources":[{"label":"Vals.ai results for Claude Sonnet 5","url":"https://www.vals.ai/models/anthropic_claude-sonnet-5"},{"label":"Vals.ai Terminal-Bench 4.0","url":"https://www.vals.ai/benchmarks/terminal-bench-4"}],"cost_estimated":false},{"id":"hub-aa26e8b0","benchmark":"tb4","harness":"Ante","harness_version":"0.2.7","model":"GPT-6 Luna","effort":"max","score":0.09231,"score_published":"6/65","ci95":[0.043,0.18712],"cost_per_attempt":null,"cost_basis":"no cost recorded on Harbor trials","tasks":65,"attempts":65,"k":1,"date":"2026-10-01","run":"Harbor job '2026-10-01__02-11-52' by leofoo on Modal; 65 tasks x 1, 65 graded attempts (reward>=1 is a pass, timeouts score 0); Ante 0.2.7; dataset ref sha256:39d9f44b4042.","source_kind":"harbor","sources":[{"label":"Harbor Hub job aa26e8b0","url":"https://hub.harborframework.com/jobs/aa26e8b0-a780-4f65-9566-ae7d7dce0d5a"}],"run_ref":"Job aa26e8b0-a780-4f65-9566-ae7d7dce0d5a","cost_estimated":false},{"id":"vals-grok-4.5","benchmark":"tb4","harness":"mini-swe-agent","model":"Grok 4.5","effort":null,"score":0.08586,"score_published":"8.59%","ci95":null,"cost_per_attempt":5.79204,"cost_basis":"Vals.ai cost_per_test (mean per task attempt)","tasks":66,"attempts":198,"k":3,"date":"2026-10-01","run":"Vals.ai run: mini-swe-agent on Harbor/Daytona, avg@3 (three full pass@1 runs); reasoning effort not published; some rows include provider fallbacks/refusals noted on the page.","source_kind":"leaderboard","sources":[{"label":"Vals.ai results for Grok 4.5","url":"https://www.vals.ai/models/grok_grok-4.5"},{"label":"Vals.ai Terminal-Bench 4.0","url":"https://www.vals.ai/benchmarks/terminal-bench-4"}],"cost_estimated":false},{"id":"vals-hy4-preview","benchmark":"tb4","harness":"mini-swe-agent","model":"Hy4 Preview","effort":null,"score":0.08081,"score_published":"8.08%","ci95":null,"cost_per_attempt":2.09742,"cost_basis":"Vals.ai cost_per_test (mean per task attempt)","tasks":66,"attempts":198,"k":3,"date":"2026-10-01","run":"Vals.ai run: mini-swe-agent on Harbor/Daytona, avg@3 (three full pass@1 runs); reasoning effort not published; some rows include provider fallbacks/refusals noted on the page.","source_kind":"leaderboard","sources":[{"label":"Vals.ai results for Hy4 Preview","url":"https://www.vals.ai/models/tencent_hy4-preview"},{"label":"Vals.ai Terminal-Bench 4.0","url":"https://www.vals.ai/benchmarks/terminal-bench-4"}],"cost_estimated":false},{"id":"vals-gemini-3.5-flash","benchmark":"tb4","harness":"mini-swe-agent","model":"Gemini 3.5 Flash","effort":null,"score":0.06061,"score_published":"6.06%","ci95":null,"cost_per_attempt":5.98176,"cost_basis":"Vals.ai cost_per_test (mean per task attempt)","tasks":66,"attempts":198,"k":3,"date":"2026-10-01","run":"Vals.ai run: mini-swe-agent on Harbor/Daytona, avg@3 (three full pass@1 runs); reasoning effort not published; some rows include provider fallbacks/refusals noted on the page.","source_kind":"leaderboard","sources":[{"label":"Vals.ai results for Gemini 3.5 Flash","url":"https://www.vals.ai/models/google_gemini-3.5-flash"},{"label":"Vals.ai Terminal-Bench 4.0","url":"https://www.vals.ai/benchmarks/terminal-bench-4"}],"cost_estimated":false},{"id":"vals-muse_spark_1_2","benchmark":"tb4","harness":"mini-swe-agent","model":"Muse Spark 1.2","effort":null,"score":0.06061,"score_published":"6.06%","ci95":null,"cost_per_attempt":4.13554,"cost_basis":"Vals.ai cost_per_test (mean per task attempt)","tasks":66,"attempts":198,"k":3,"date":"2026-10-01","run":"Vals.ai run: mini-swe-agent on Harbor/Daytona, avg@3 (three full pass@1 runs); reasoning effort not published; some rows include provider fallbacks/refusals noted on the page.","source_kind":"leaderboard","sources":[{"label":"Vals.ai results for Muse Spark 1.2","url":"https://www.vals.ai/models/meta_muse_spark_1_2"},{"label":"Vals.ai Terminal-Bench 4.0","url":"https://www.vals.ai/benchmarks/terminal-bench-4"}],"cost_estimated":false},{"id":"vals-gemini-3.1-pro-preview","benchmark":"tb4","harness":"mini-swe-agent","model":"Gemini 3.1 Pro Preview","effort":null,"score":0.02525,"score_published":"2.52%","ci95":null,"cost_per_attempt":4.11196,"cost_basis":"Vals.ai cost_per_test (mean per task attempt)","tasks":66,"attempts":198,"k":3,"date":"2026-10-01","run":"Vals.ai run: mini-swe-agent on Harbor/Daytona, avg@3 (three full pass@1 runs); reasoning effort not published; some rows include provider fallbacks/refusals noted on the page.","source_kind":"leaderboard","sources":[{"label":"Vals.ai results for Gemini 3.1 Pro Preview","url":"https://www.vals.ai/models/google_gemini-3.1-pro-preview"},{"label":"Vals.ai Terminal-Bench 4.0","url":"https://www.vals.ai/benchmarks/terminal-bench-4"}],"cost_estimated":false},{"id":"vals-gpt-5.4-mini-2026-03-17","benchmark":"tb4","harness":"mini-swe-agent","model":"GPT-5.4 Mini","effort":null,"score":0.02525,"score_published":"2.52%","ci95":null,"cost_per_attempt":1.4337,"cost_basis":"Vals.ai cost_per_test (mean per task attempt)","tasks":66,"attempts":198,"k":3,"date":"2026-10-01","run":"Vals.ai run: mini-swe-agent on Harbor/Daytona, avg@3 (three full pass@1 runs); reasoning effort not published; some rows include provider fallbacks/refusals noted on the page.","source_kind":"leaderboard","sources":[{"label":"Vals.ai results for GPT-5.4 Mini","url":"https://www.vals.ai/models/openai_gpt-5.4-mini-2026-03-17"},{"label":"Vals.ai Terminal-Bench 4.0","url":"https://www.vals.ai/benchmarks/terminal-bench-4"}],"cost_estimated":false},{"id":"hub-9dec984a","benchmark":"tb4","harness":"Ante","harness_version":"0.2.7","model":"GPT-6 Luna","effort":null,"score":0.01515,"score_published":"1/66","ci95":[0.00268,0.08096],"cost_per_attempt":null,"cost_basis":"no cost recorded on Harbor trials","tasks":66,"attempts":66,"k":1,"date":"2026-09-30","run":"Harbor job '2026-09-30__05-23-26' by leofoo on Modal; 66 tasks x 1, 66 graded attempts (reward>=1 is a pass, timeouts score 0); Ante 0.2.7; dataset ref sha256:39d9f44b4042.","source_kind":"harbor","sources":[{"label":"Harbor Hub job 9dec984a","url":"https://hub.harborframework.com/jobs/9dec984a-a4ca-4fdc-8d5c-57bbb3b27edd"}],"run_ref":"Job 9dec984a-a4ca-4fdc-8d5c-57bbb3b27edd","cost_estimated":false},{"id":"vals-MiniMax-M3","benchmark":"tb4","harness":"mini-swe-agent","model":"MiniMax-M3","effort":null,"score":0.0101,"score_published":"1.01%","ci95":null,"cost_per_attempt":7.2295,"cost_basis":"Vals.ai cost_per_test (mean per task attempt)","tasks":66,"attempts":198,"k":3,"date":"2026-10-01","run":"Vals.ai run: mini-swe-agent on Harbor/Daytona, avg@3 (three full pass@1 runs); reasoning effort not published; some rows include provider fallbacks/refusals noted on the page.","source_kind":"leaderboard","sources":[{"label":"Vals.ai results for MiniMax-M3","url":"https://www.vals.ai/models/minimax_MiniMax-M3"},{"label":"Vals.ai Terminal-Bench 4.0","url":"https://www.vals.ai/benchmarks/terminal-bench-4"}],"cost_estimated":false},{"id":"vals-inkling","benchmark":"tb4","harness":"mini-swe-agent","model":"Inkling","effort":null,"score":0.00505,"score_published":"0.51%","ci95":null,"cost_per_attempt":2.51049,"cost_basis":"Vals.ai cost_per_test (mean per task attempt)","tasks":66,"attempts":198,"k":3,"date":"2026-10-01","run":"Vals.ai run: mini-swe-agent on Harbor/Daytona, avg@3 (three full pass@1 runs); reasoning effort not published; some rows include provider fallbacks/refusals noted on the page.","source_kind":"leaderboard","sources":[{"label":"Vals.ai results for Inkling","url":"https://www.vals.ai/models/thinkingmachines_inkling"},{"label":"Vals.ai Terminal-Bench 4.0","url":"https://www.vals.ai/benchmarks/terminal-bench-4"}],"cost_estimated":false},{"id":"vals-mimo-v2.5-pro","benchmark":"tb4","harness":"mini-swe-agent","model":"MiMo V2.5 Pro","effort":null,"score":0.00505,"score_published":"0.51%","ci95":null,"cost_per_attempt":0.23303,"cost_basis":"Vals.ai cost_per_test (mean per task attempt)","tasks":66,"attempts":198,"k":3,"date":"2026-10-01","run":"Vals.ai run: mini-swe-agent on Harbor/Daytona, avg@3 (three full pass@1 runs); reasoning effort not published; some rows include provider fallbacks/refusals noted on the page.","source_kind":"leaderboard","sources":[{"label":"Vals.ai results for MiMo V2.5 Pro","url":"https://www.vals.ai/models/xiaomi_mimo-v2.5-pro"},{"label":"Vals.ai Terminal-Bench 4.0","url":"https://www.vals.ai/benchmarks/terminal-bench-4"}],"cost_estimated":false},{"id":"vals-inkling-small","benchmark":"tb4","harness":"mini-swe-agent","model":"Inkling Small","effort":null,"score":0.0,"score_published":"0.00%","ci95":null,"cost_per_attempt":0.32168,"cost_basis":"Vals.ai cost_per_test (mean per task attempt)","tasks":66,"attempts":198,"k":3,"date":"2026-10-01","run":"Vals.ai run: mini-swe-agent on Harbor/Daytona, avg@3 (three full pass@1 runs); reasoning effort not published; some rows include provider fallbacks/refusals noted on the page.","source_kind":"leaderboard","sources":[{"label":"Vals.ai results for Inkling Small","url":"https://www.vals.ai/models/thinkingmachines_inkling-small"},{"label":"Vals.ai Terminal-Bench 4.0","url":"https://www.vals.ai/benchmarks/terminal-bench-4"}],"cost_estimated":false},{"id":"vals-mercury-2.5","benchmark":"tb4","harness":"mini-swe-agent","model":"Mercury 2.5","effort":null,"score":0.0,"score_published":"0.00%","ci95":null,"cost_per_attempt":0.36977,"cost_basis":"Vals.ai cost_per_test (mean per task attempt)","tasks":66,"attempts":198,"k":3,"date":"2026-10-01","run":"Vals.ai run: mini-swe-agent on Harbor/Daytona, avg@3 (three full pass@1 runs); reasoning effort not published; some rows include provider fallbacks/refusals noted on the page.","source_kind":"leaderboard","sources":[{"label":"Vals.ai results for Mercury 2.5","url":"https://www.vals.ai/models/inception_mercury-2.5"},{"label":"Vals.ai Terminal-Bench 4.0","url":"https://www.vals.ai/benchmarks/terminal-bench-4"}],"cost_estimated":false},{"id":"aa-a2c87c06-atlas","benchmark":"atlas","harness":"Claude Code","model":"Claude Sonnet 5.5","effort":"max","score":0.66935,"score_published":"66.9%","ci95":null,"cost_per_attempt":15.87452,"cost_basis":"ESTIMATE: AA publishes only the row's mean cost, $14.19 per attempt; allocated 45.8% to this benchmark by attempts x (input x (1 - 0.9 x cacheHitRate 0.95) + 5 x output tokens), as a share of that mean","cost_estimated":true,"tasks":124,"attempts":372,"k":3,"date":null,"run":"Artificial Analysis per-benchmark mean reward for 'Claude Code - Sonnet 5.5 (max)', 372 attempts. No run date published.","source_kind":"aa","sources":[{"label":"Artificial Analysis Coding Agents (no per-run page)","url":"https://artificialanalysis.ai/agents/coding-agents"}]},{"id":"aa-ab5074e4-atlas","benchmark":"atlas","harness":"Claude Code","model":"Claude Opus 5.5","effort":"max","score":0.66398,"score_published":"66.4%","ci95":null,"cost_per_attempt":10.86018,"cost_basis":"ESTIMATE: AA publishes only the row's mean cost, $13.04 per attempt; allocated 34.1% to this benchmark by attempts x (input x (1 - 0.9 x cacheHitRate 0.95) + 5 x output tokens), as a share of that mean","cost_estimated":true,"tasks":124,"attempts":372,"k":3,"date":null,"run":"Artificial Analysis per-benchmark mean reward for 'Claude Code - Opus 5.5 (max)', 372 attempts. No run date published.","source_kind":"aa","sources":[{"label":"Artificial Analysis Coding Agents (no per-run page)","url":"https://artificialanalysis.ai/agents/coding-agents"}]},{"id":"aa-55b029d6-atlas","benchmark":"atlas","harness":"Kimi Code CLI","model":"Kimi K3","effort":null,"score":0.66129,"score_published":"66.1%","ci95":null,"cost_per_attempt":2.80266,"cost_basis":"ESTIMATE: AA publishes only the row's mean cost, $5.05 per attempt; allocated 22.7% to this benchmark by attempts x (input x (1 - 0.9 x cacheHitRate 0.95) + 5 x output tokens), as a share of that mean","cost_estimated":true,"tasks":124,"attempts":372,"k":3,"date":null,"run":"Artificial Analysis per-benchmark mean reward for 'Kimi Code CLI - Kimi K3', 372 attempts. No run date published.","source_kind":"aa","sources":[{"label":"Artificial Analysis Coding Agents (no per-run page)","url":"https://artificialanalysis.ai/agents/coding-agents"}]},{"id":"aa-34970c96-atlas","benchmark":"atlas","harness":"Devin Fusion CLI","model":"Claude Fable 5.1 + SWE-2 Medium","effort":"xhigh","score":0.6586,"score_published":"65.9%","ci95":null,"cost_per_attempt":4.48172,"cost_basis":"ESTIMATE: AA publishes only the row's mean cost, $7.90 per attempt; allocated 23.3% to this benchmark by attempts x (input x (1 - 0.9 x cacheHitRate 0.96) + 5 x output tokens), as a share of that mean","cost_estimated":true,"tasks":124,"attempts":372,"k":3,"date":null,"run":"Artificial Analysis per-benchmark mean reward for 'Devin Fusion CLI - Claude Fable 5.1 XHigh + SWE-2 Medium', 372 attempts. No run date published.","source_kind":"aa","sources":[{"label":"Artificial Analysis Coding Agents (no per-run page)","url":"https://artificialanalysis.ai/agents/coding-agents"}]},{"id":"aa-29a1cdad-atlas","benchmark":"atlas","harness":"Claude Code","model":"Claude Fable 5.1","effort":"max","score":0.64785,"score_published":"64.8%","ci95":null,"cost_per_attempt":7.83103,"cost_basis":"ESTIMATE: AA publishes only the row's mean cost, $12.39 per attempt; allocated 25.9% to this benchmark by attempts x (input x (1 - 0.9 x cacheHitRate 0.92) + 5 x output tokens), as a share of that mean","cost_estimated":true,"tasks":124,"attempts":372,"k":3,"date":null,"run":"Artificial Analysis per-benchmark mean reward for 'Claude Code - Fable 5.1 (max) (with fallback)', 372 attempts. No run date published. Uses model fallback on refusals.","source_kind":"aa","sources":[{"label":"Artificial Analysis Coding Agents (no per-run page)","url":"https://artificialanalysis.ai/agents/coding-agents"}]},{"id":"aa-f927d9c4-atlas","benchmark":"atlas","harness":"Grok Build","model":"Grok 4.7","effort":"xhigh","score":0.62903,"score_published":"62.9%","ci95":null,"cost_per_attempt":4.76053,"cost_basis":"ESTIMATE: AA publishes only the row's mean cost, $8.82 per attempt; allocated 22.2% to this benchmark by attempts x (input x (1 - 0.9 x cacheHitRate 0.94) + 5 x output tokens), as a share of that mean","cost_estimated":true,"tasks":124,"attempts":372,"k":3,"date":null,"run":"Artificial Analysis per-benchmark mean reward for 'Grok Build - Grok 4.7 (xhigh)', 372 attempts. No run date published.","source_kind":"aa","sources":[{"label":"Artificial Analysis Coding Agents (no per-run page)","url":"https://artificialanalysis.ai/agents/coding-agents"}]},{"id":"aa-cd4c865d-atlas","benchmark":"atlas","harness":"Claude Code","model":"Qwen3.8 Max","effort":null,"score":0.62097,"score_published":"62.1%","ci95":null,"cost_per_attempt":1.91062,"cost_basis":"ESTIMATE: AA publishes only the row's mean cost, $3.48 per attempt; allocated 22.5% to this benchmark by attempts x (input x (1 - 0.9 x cacheHitRate 0.95) + 5 x output tokens), as a share of that mean","cost_estimated":true,"tasks":124,"attempts":372,"k":3,"date":null,"run":"Artificial Analysis per-benchmark mean reward for 'Claude Code - Qwen3.8 Max', 372 attempts. No run date published.","source_kind":"aa","sources":[{"label":"Artificial Analysis Coding Agents (no per-run page)","url":"https://artificialanalysis.ai/agents/coding-agents"}]},{"id":"aa-40951881-atlas","benchmark":"atlas","harness":"Claude Code","model":"Claude Opus 5","effort":"max","score":0.62097,"score_published":"62.1%","ci95":null,"cost_per_attempt":6.51356,"cost_basis":"ESTIMATE: AA publishes only the row's mean cost, $10.79 per attempt; allocated 24.7% to this benchmark by attempts x (input x (1 - 0.9 x cacheHitRate 0.97) + 5 x output tokens), as a share of that mean","cost_estimated":true,"tasks":124,"attempts":372,"k":3,"date":null,"run":"Artificial Analysis per-benchmark mean reward for 'Claude Code - Opus 5 (max)', 372 attempts. No run date published.","source_kind":"aa","sources":[{"label":"Artificial Analysis Coding Agents (no per-run page)","url":"https://artificialanalysis.ai/agents/coding-agents"}]},{"id":"aa-666605c6-atlas","benchmark":"atlas","harness":"Claude Code","model":"Claude Sonnet 5.5","effort":"xhigh","score":0.62097,"score_published":"62.1%","ci95":null,"cost_per_attempt":2.25267,"cost_basis":"ESTIMATE: AA publishes only the row's mean cost, $3.33 per attempt; allocated 27.7% to this benchmark by attempts x (input x (1 - 0.9 x cacheHitRate 0.96) + 5 x output tokens), as a share of that mean","cost_estimated":true,"tasks":124,"attempts":372,"k":3,"date":null,"run":"Artificial Analysis per-benchmark mean reward for 'Claude Code - Sonnet 5.5 (xhigh)', 372 attempts. No run date published.","source_kind":"aa","sources":[{"label":"Artificial Analysis Coding Agents (no per-run page)","url":"https://artificialanalysis.ai/agents/coding-agents"}]},{"id":"aa-9e810ff3-atlas","benchmark":"atlas","harness":"Codex","model":"GPT-6 Astra","effort":"max","score":0.61828,"score_published":"61.8%","ci95":null,"cost_per_attempt":5.50264,"cost_basis":"ESTIMATE: AA publishes only the row's mean cost, $7.47 per attempt; allocated 30.2% to this benchmark by attempts x (input x (1 - 0.9 x cacheHitRate 0.93) + 5 x output tokens), as a share of that mean","cost_estimated":true,"tasks":124,"attempts":372,"k":3,"date":null,"run":"Artificial Analysis per-benchmark mean reward for 'Codex - GPT-6 Astra (max)', 372 attempts. No run date published.","source_kind":"aa","sources":[{"label":"Artificial Analysis Coding Agents (no per-run page)","url":"https://artificialanalysis.ai/agents/coding-agents"}]},{"id":"aa-542590ec-atlas","benchmark":"atlas","harness":"Codex","model":"DeepSeek V4 Pro 0813","effort":"max","score":0.61828,"score_published":"61.8%","ci95":null,"cost_per_attempt":0.09738,"cost_basis":"ESTIMATE: AA publishes only the row's mean cost, $0.24 per attempt; allocated 16.7% to this benchmark by attempts x (input x (1 - 0.9 x cacheHitRate 0.99) + 5 x output tokens), as a share of that mean","cost_estimated":true,"tasks":124,"attempts":372,"k":3,"date":null,"run":"Artificial Analysis per-benchmark mean reward for 'Codex - DeepSeek V4 Pro 0813 (max)', 372 attempts. No run date published.","source_kind":"aa","sources":[{"label":"Artificial Analysis Coding Agents (no per-run page)","url":"https://artificialanalysis.ai/agents/coding-agents"}]},{"id":"aa-ed8c1d0b-atlas","benchmark":"atlas","harness":"Codex","model":"GPT-6.1 Sol","effort":"xhigh","score":0.61022,"score_published":"61.0%","ci95":null,"cost_per_attempt":0.82954,"cost_basis":"ESTIMATE: AA publishes only the row's mean cost, $1.04 per attempt; allocated 32.7% to this benchmark by attempts x (input x (1 - 0.9 x cacheHitRate 0.93) + 5 x output tokens), as a share of that mean","cost_estimated":true,"tasks":124,"attempts":372,"k":3,"date":null,"run":"Artificial Analysis per-benchmark mean reward for 'Codex - GPT-6.1 Sol (xhigh)', 372 attempts. No run date published.","source_kind":"aa","sources":[{"label":"Artificial Analysis Coding Agents (no per-run page)","url":"https://artificialanalysis.ai/agents/coding-agents"}]},{"id":"aa-93acd59f-atlas","benchmark":"atlas","harness":"Codex","model":"GPT-6.1 Sol","effort":"medium","score":0.60753,"score_published":"60.8%","ci95":null,"cost_per_attempt":0.56985,"cost_basis":"ESTIMATE: AA publishes only the row's mean cost, $0.70 per attempt; allocated 33.1% to this benchmark by attempts x (input x (1 - 0.9 x cacheHitRate 0.92) + 5 x output tokens), as a share of that mean","cost_estimated":true,"tasks":124,"attempts":372,"k":3,"date":null,"run":"Artificial Analysis per-benchmark mean reward for 'Codex - GPT-6.1 Sol (medium)', 372 attempts. No run date published.","source_kind":"aa","sources":[{"label":"Artificial Analysis Coding Agents (no per-run page)","url":"https://artificialanalysis.ai/agents/coding-agents"}]},{"id":"hub-11a440fb","benchmark":"atlas","harness":"Codex","harness_version":"0.155.1","model":"GPT-6 Astra","effort":"xhigh","score":0.60484,"score_published":"225/372","ci95":[0.55433,0.6532],"cost_per_attempt":3.70542,"cost_basis":"Harbor trial cost_usd, mean over 366 of 372 graded attempts","tasks":124,"attempts":372,"k":3,"date":"2026-09-21","run":"Harbor job '20260916-atlas-codex-astra-xhigh-modal' by Philip Sinitsin on Modal; 124 tasks x 3, 372 graded attempts (reward>=1 is a pass, timeouts score 0); Codex 0.155.1; dataset ref sha256:0e26bc0313ae.","source_kind":"harbor","sources":[{"label":"Harbor Hub job 11a440fb","url":"https://hub.harborframework.com/jobs/11a440fb-b1c4-473b-98a6-907500b3fdaf"}],"run_ref":"Job 11a440fb-b1c4-473b-98a6-907500b3fdaf","cost_estimated":false},{"id":"aa-4aee8aa2-atlas","benchmark":"atlas","harness":"Codex","model":"GPT-6.1 Sol","effort":"high","score":0.59946,"score_published":"59.9%","ci95":null,"cost_per_attempt":0.71876,"cost_basis":"ESTIMATE: AA publishes only the row's mean cost, $0.89 per attempt; allocated 33.1% to this benchmark by attempts x (input x (1 - 0.9 x cacheHitRate 0.93) + 5 x output tokens), as a share of that mean","cost_estimated":true,"tasks":124,"attempts":372,"k":3,"date":null,"run":"Artificial Analysis per-benchmark mean reward for 'Codex - GPT-6.1 Sol (high)', 372 attempts. No run date published.","source_kind":"aa","sources":[{"label":"Artificial Analysis Coding Agents (no per-run page)","url":"https://artificialanalysis.ai/agents/coding-agents"}]},{"id":"hub-da4ac972","benchmark":"atlas","harness":"Pi","harness_version":"0.86.1","model":"GPT-6 Astra","effort":"xhigh","score":0.59409,"score_published":"221/372","ci95":[0.54347,0.64278],"cost_per_attempt":2.93994,"cost_basis":"Harbor trial cost_usd, mean over 366 of 372 graded attempts","tasks":124,"attempts":372,"k":3,"date":"2026-09-21","run":"Harbor job '20260916-atlas-pi-astra-xhigh-modal' by Philip Sinitsin on Modal; 124 tasks x 3, 372 graded attempts (reward>=1 is a pass, timeouts score 0); Pi 0.86.1; dataset ref sha256:0e26bc0313ae.","source_kind":"harbor","sources":[{"label":"Harbor Hub job da4ac972","url":"https://hub.harborframework.com/jobs/da4ac972-9d5a-4142-b057-85f07375f14e"}],"run_ref":"Job da4ac972-9d5a-4142-b057-85f07375f14e","cost_estimated":false},{"id":"aa-f2de0737-atlas","benchmark":"atlas","harness":"Muse Code","model":"Muse Spark 1.3","effort":"max","score":0.59409,"score_published":"59.4%","ci95":null,"cost_per_attempt":0.80277,"cost_basis":"ESTIMATE: AA publishes only the row's mean cost, $3.98 per attempt; allocated 8.3% to this benchmark by attempts x (input x (1 - 0.9 x cacheHitRate 0.97) + 5 x output tokens), as a share of that mean","cost_estimated":true,"tasks":124,"attempts":372,"k":3,"date":null,"run":"Artificial Analysis per-benchmark mean reward for 'Muse Code - Muse Spark 1.3 (max)', 372 attempts. No run date published.","source_kind":"aa","sources":[{"label":"Artificial Analysis Coding Agents (no per-run page)","url":"https://artificialanalysis.ai/agents/coding-agents"}]},{"id":"aa-fa259509-atlas","benchmark":"atlas","harness":"Devin Fusion CLI","model":"GPT-6 Astra + SWE-2 Medium","effort":"xhigh","score":0.59409,"score_published":"59.4%","ci95":null,"cost_per_attempt":1.65708,"cost_basis":"ESTIMATE: AA publishes only the row's mean cost, $4.54 per attempt; allocated 14.9% to this benchmark by attempts x (input x (1 - 0.9 x cacheHitRate 0.92) + 5 x output tokens), as a share of that mean","cost_estimated":true,"tasks":124,"attempts":372,"k":3,"date":null,"run":"Artificial Analysis per-benchmark mean reward for 'Devin Fusion CLI - GPT-6 Astra XHigh + SWE-2 Medium', 372 attempts. No run date published.","source_kind":"aa","sources":[{"label":"Artificial Analysis Coding Agents (no per-run page)","url":"https://artificialanalysis.ai/agents/coding-agents"}]},{"id":"aa-45493c27-atlas","benchmark":"atlas","harness":"Opencode","model":"GLM-5.3","effort":"max","score":0.59409,"score_published":"59.4%","ci95":null,"cost_per_attempt":1.79822,"cost_basis":"ESTIMATE: AA publishes only the row's mean cost, $4.24 per attempt; allocated 17.4% to this benchmark by attempts x (input x (1 - 0.9 x cacheHitRate 0.97) + 5 x output tokens), as a share of that mean","cost_estimated":true,"tasks":124,"attempts":372,"k":3,"date":null,"run":"Artificial Analysis per-benchmark mean reward for 'Opencode - GLM-5.3', 372 attempts. No run date published.","source_kind":"aa","sources":[{"label":"Artificial Analysis Coding Agents (no per-run page)","url":"https://artificialanalysis.ai/agents/coding-agents"}]},{"id":"aa-70a5d300-atlas","benchmark":"atlas","harness":"Grok Build","model":"Grok 4.6","effort":"xhigh","score":0.58333,"score_published":"58.3%","ci95":null,"cost_per_attempt":1.88661,"cost_basis":"ESTIMATE: AA publishes only the row's mean cost, $3.57 per attempt; allocated 21.6% to this benchmark by attempts x (input x (1 - 0.9 x cacheHitRate 0.91) + 5 x output tokens), as a share of that mean","cost_estimated":true,"tasks":124,"attempts":372,"k":3,"date":null,"run":"Artificial Analysis per-benchmark mean reward for 'Grok Build - Grok 4.6 (xhigh)', 372 attempts. No run date published.","source_kind":"aa","sources":[{"label":"Artificial Analysis Coding Agents (no per-run page)","url":"https://artificialanalysis.ai/agents/coding-agents"}]},{"id":"aa-ea296ce7-atlas","benchmark":"atlas","harness":"Codex","model":"GPT-6.1 Sol","effort":"max","score":0.57796,"score_published":"57.8%","ci95":null,"cost_per_attempt":1.13801,"cost_basis":"ESTIMATE: AA publishes only the row's mean cost, $1.55 per attempt; allocated 30.0% to this benchmark by attempts x (input x (1 - 0.9 x cacheHitRate 0.93) + 5 x output tokens), as a share of that mean","cost_estimated":true,"tasks":124,"attempts":372,"k":3,"date":null,"run":"Artificial Analysis per-benchmark mean reward for 'Codex - GPT-6.1 Sol (max)', 372 attempts. No run date published.","source_kind":"aa","sources":[{"label":"Artificial Analysis Coding Agents (no per-run page)","url":"https://artificialanalysis.ai/agents/coding-agents"}]},{"id":"aa-7a0dfd2a-atlas","benchmark":"atlas","harness":"Codex","model":"GPT-6 Sol","effort":"max","score":0.57527,"score_published":"57.5%","ci95":null,"cost_per_attempt":1.86549,"cost_basis":"ESTIMATE: AA publishes only the row's mean cost, $2.99 per attempt; allocated 25.6% to this benchmark by attempts x (input x (1 - 0.9 x cacheHitRate 0.96) + 5 x output tokens), as a share of that mean","cost_estimated":true,"tasks":124,"attempts":372,"k":3,"date":null,"run":"Artificial Analysis per-benchmark mean reward for 'Codex - GPT-6 Sol (max)', 372 attempts. No run date published.","source_kind":"aa","sources":[{"label":"Artificial Analysis Coding Agents (no per-run page)","url":"https://artificialanalysis.ai/agents/coding-agents"}]},{"id":"aa-ef63fafa-atlas","benchmark":"atlas","harness":"Antigravity CLI","model":"Gemini 4 Argon","effort":"high","score":0.56452,"score_published":"56.5%","ci95":null,"cost_per_attempt":2.95444,"cost_basis":"ESTIMATE: AA publishes only the row's mean cost, $5.84 per attempt; allocated 20.7% to this benchmark by attempts x (input x (1 - 0.9 x cacheHitRate 0.87) + 5 x output tokens), as a share of that mean","cost_estimated":true,"tasks":124,"attempts":371,"k":3,"date":null,"run":"Artificial Analysis per-benchmark mean reward for 'Antigravity CLI - Gemini 4 Argon (high)', 371 attempts. No run date published. Row marked unavailable on AA.","source_kind":"aa","sources":[{"label":"Artificial Analysis Coding Agents (no per-run page)","url":"https://artificialanalysis.ai/agents/coding-agents"}]},{"id":"aa-33e80f38-atlas","benchmark":"atlas","harness":"Claude Code","model":"Claude Sonnet 5.5","effort":"high","score":0.56452,"score_published":"56.5%","ci95":null,"cost_per_attempt":0.76572,"cost_basis":"ESTIMATE: AA publishes only the row's mean cost, $1.24 per attempt; allocated 25.4% to this benchmark by attempts x (input x (1 - 0.9 x cacheHitRate 0.95) + 5 x output tokens), as a share of that mean","cost_estimated":true,"tasks":124,"attempts":372,"k":3,"date":null,"run":"Artificial Analysis per-benchmark mean reward for 'Claude Code - Sonnet 5.5 (high)', 372 attempts. No run date published.","source_kind":"aa","sources":[{"label":"Artificial Analysis Coding Agents (no per-run page)","url":"https://artificialanalysis.ai/agents/coding-agents"}]},{"id":"aa-fa589dad-atlas","benchmark":"atlas","harness":"Codex","model":"GPT-6.1 Sol","effort":"low","score":0.55108,"score_published":"55.1%","ci95":null,"cost_per_attempt":0.36224,"cost_basis":"ESTIMATE: AA publishes only the row's mean cost, $0.50 per attempt; allocated 29.7% to this benchmark by attempts x (input x (1 - 0.9 x cacheHitRate 0.91) + 5 x output tokens), as a share of that mean","cost_estimated":true,"tasks":124,"attempts":372,"k":3,"date":null,"run":"Artificial Analysis per-benchmark mean reward for 'Codex - GPT-6.1 Sol (low)', 372 attempts. No run date published.","source_kind":"aa","sources":[{"label":"Artificial Analysis Coding Agents (no per-run page)","url":"https://artificialanalysis.ai/agents/coding-agents"}]},{"id":"aa-cd60470c-atlas","benchmark":"atlas","harness":"Muse Code","model":"Muse Spark 1.3","effort":"xhigh","score":0.5457,"score_published":"54.6%","ci95":null,"cost_per_attempt":0.73428,"cost_basis":"ESTIMATE: AA publishes only the row's mean cost, $3.47 per attempt; allocated 8.7% to this benchmark by attempts x (input x (1 - 0.9 x cacheHitRate 0.97) + 5 x output tokens), as a share of that mean","cost_estimated":true,"tasks":124,"attempts":372,"k":3,"date":null,"run":"Artificial Analysis per-benchmark mean reward for 'Muse Code - Muse Spark 1.3 (xhigh)', 372 attempts. No run date published.","source_kind":"aa","sources":[{"label":"Artificial Analysis Coding Agents (no per-run page)","url":"https://artificialanalysis.ai/agents/coding-agents"}]},{"id":"aa-709f8012-atlas","benchmark":"atlas","harness":"Codex","model":"GPT-5.6 Sol","effort":"max","score":0.54032,"score_published":"54.0%","ci95":null,"cost_per_attempt":4.83349,"cost_basis":"ESTIMATE: AA publishes only the row's mean cost, $6.35 per attempt; allocated 31.1% to this benchmark by attempts x (input x (1 - 0.9 x cacheHitRate 0.95) + 5 x output tokens), as a share of that mean","cost_estimated":true,"tasks":124,"attempts":372,"k":3,"date":null,"run":"Artificial Analysis per-benchmark mean reward for 'Codex - GPT-5.6 Sol (max)', 372 attempts. No run date published.","source_kind":"aa","sources":[{"label":"Artificial Analysis Coding Agents (no per-run page)","url":"https://artificialanalysis.ai/agents/coding-agents"}]},{"id":"aa-28ef6513-atlas","benchmark":"atlas","harness":"Codex","model":"DeepSeek V4 Flash 0731","effort":"max","score":0.51344,"score_published":"51.3%","ci95":null,"cost_per_attempt":0.04393,"cost_basis":"ESTIMATE: AA publishes only the row's mean cost, $0.09 per attempt; allocated 21.1% to this benchmark by attempts x (input x (1 - 0.9 x cacheHitRate 0.99) + 5 x output tokens), as a share of that mean","cost_estimated":true,"tasks":124,"attempts":372,"k":3,"date":null,"run":"Artificial Analysis per-benchmark mean reward for 'Codex - DeepSeek V4 Flash 0731 (max)', 372 attempts. No run date published.","source_kind":"aa","sources":[{"label":"Artificial Analysis Coding Agents (no per-run page)","url":"https://artificialanalysis.ai/agents/coding-agents"}]},{"id":"aa-b3c56ec6-atlas","benchmark":"atlas","harness":"Codex","model":"GPT-5.6 Luna","effort":"max","score":0.48656,"score_published":"48.7%","ci95":null,"cost_per_attempt":0.22695,"cost_basis":"ESTIMATE: AA publishes only the row's mean cost, $0.44 per attempt; allocated 21.2% to this benchmark by attempts x (input x (1 - 0.9 x cacheHitRate 0.97) + 5 x output tokens), as a share of that mean","cost_estimated":true,"tasks":124,"attempts":372,"k":3,"date":null,"run":"Artificial Analysis per-benchmark mean reward for 'Codex - GPT-5.6 Luna (max)', 372 attempts. No run date published.","source_kind":"aa","sources":[{"label":"Artificial Analysis Coding Agents (no per-run page)","url":"https://artificialanalysis.ai/agents/coding-agents"}]},{"id":"aa-d1003684-atlas","benchmark":"atlas","harness":"Antigravity SDK","model":"Gemini 3.8 Flash","effort":"high","score":0.45161,"score_published":"45.2%","ci95":null,"cost_per_attempt":1.59124,"cost_basis":"ESTIMATE: AA publishes only the row's mean cost, $2.47 per attempt; allocated 26.4% to this benchmark by attempts x (input x (1 - 0.9 x cacheHitRate 0.87) + 5 x output tokens), as a share of that mean","cost_estimated":true,"tasks":124,"attempts":372,"k":3,"date":null,"run":"Artificial Analysis per-benchmark mean reward for 'Antigravity SDK - Gemini 3.8 Flash (high)', 372 attempts. No run date published.","source_kind":"aa","sources":[{"label":"Artificial Analysis Coding Agents (no per-run page)","url":"https://artificialanalysis.ai/agents/coding-agents"}]},{"id":"aa-4d6aa9e9-atlas","benchmark":"atlas","harness":"Claude Code","model":"Claude Sonnet 5.5","effort":"medium","score":0.44892,"score_published":"44.9%","ci95":null,"cost_per_attempt":0.28224,"cost_basis":"ESTIMATE: AA publishes only the row's mean cost, $0.62 per attempt; allocated 18.7% to this benchmark by attempts x (input x (1 - 0.9 x cacheHitRate 0.93) + 5 x output tokens), as a share of that mean","cost_estimated":true,"tasks":124,"attempts":372,"k":3,"date":null,"run":"Artificial Analysis per-benchmark mean reward for 'Claude Code - Sonnet 5.5 (medium)', 372 attempts. No run date published.","source_kind":"aa","sources":[{"label":"Artificial Analysis Coding Agents (no per-run page)","url":"https://artificialanalysis.ai/agents/coding-agents"}]},{"id":"aa-ac8276cd-atlas","benchmark":"atlas","harness":"Codex","model":"GPT-6 Luna","effort":"max","score":0.44355,"score_published":"44.4%","ci95":null,"cost_per_attempt":0.09925,"cost_basis":"ESTIMATE: AA publishes only the row's mean cost, $0.18 per attempt; allocated 23.1% to this benchmark by attempts x (input x (1 - 0.9 x cacheHitRate 0.96) + 5 x output tokens), as a share of that mean","cost_estimated":true,"tasks":124,"attempts":372,"k":3,"date":null,"run":"Artificial Analysis per-benchmark mean reward for 'Codex - GPT-6 Luna (max)', 372 attempts. No run date published.","source_kind":"aa","sources":[{"label":"Artificial Analysis Coding Agents (no per-run page)","url":"https://artificialanalysis.ai/agents/coding-agents"}]},{"id":"aa-e9f9d3ae-atlas","benchmark":"atlas","harness":"Claude Code","model":"Claude Sonnet 5.5","effort":"low","score":0.38978,"score_published":"39.0%","ci95":null,"cost_per_attempt":0.19992,"cost_basis":"ESTIMATE: AA publishes only the row's mean cost, $0.48 per attempt; allocated 16.9% to this benchmark by attempts x (input x (1 - 0.9 x cacheHitRate 0.93) + 5 x output tokens), as a share of that mean","cost_estimated":true,"tasks":124,"attempts":372,"k":3,"date":null,"run":"Artificial Analysis per-benchmark mean reward for 'Claude Code - Sonnet 5.5 (low)', 372 attempts. No run date published.","source_kind":"aa","sources":[{"label":"Artificial Analysis Coding Agents (no per-run page)","url":"https://artificialanalysis.ai/agents/coding-agents"}]}]