{
 "model": "GPT-6 Astra",
 "effort": "xhigh",
 "note": "Same model, same tasks, same graders: every benchmark on all its tasks. DeepSWE uses the grader's own score. Overall cost per attempt is the mean over all 303 tasks, each benchmark's cost per attempt weighted by its task count: Artificial Analysis's cost-per-task basis.",
 "sources": {
  "hub": "https://hub.harborframework.com"
 },
 "overall": [
  {
   "harness": "Reason",
   "ours": true,
   "scores": {
    "atlas": 0.6398,
    "deepswe": 0.7227,
    "tb4": 0.5909
   },
   "index": 0.6511,
   "cost_per_attempt": 3.964,
   "ci95": {
    "atlas": [
     0.5591,
     0.7151
    ],
    "deepswe": [
     0.6431,
     0.7935
    ],
    "tb4": [
     0.4848,
     0.702
    ],
    "index": [
     0.5996,
     0.7021
    ]
   },
   "reason_lead": null
  },
  {
   "harness": "Codex",
   "ours": false,
   "scores": {
    "atlas": 0.6048,
    "deepswe": 0.6903,
    "tb4": 0.5788
   },
   "index": 0.6246,
   "cost_per_attempt": 4.886,
   "ci95": {
    "atlas": [
     0.5242,
     0.6828
    ],
    "deepswe": [
     0.6106,
     0.764
    ],
    "tb4": [
     0.4727,
     0.6848
    ],
    "index": [
     0.5747,
     0.6753
    ]
   },
   "reason_lead": {
    "atlas": [
     0.0349,
     -0.0134,
     0.0833
    ],
    "deepswe": [
     0.0324,
     0.0,
     0.0678
    ],
    "tb4": [
     0.0121,
     -0.0384,
     0.0606
    ],
    "index": [
     0.0265,
     0.0009,
     0.0518
    ]
   }
  },
  {
   "harness": "Pi",
   "ours": false,
   "scores": {
    "atlas": 0.5941,
    "deepswe": 0.6962,
    "tb4": 0.5502
   },
   "index": 0.6135,
   "cost_per_attempt": 4.151,
   "ci95": {
    "atlas": [
     0.5161,
     0.6694
    ],
    "deepswe": [
     0.6195,
     0.7699
    ],
    "tb4": [
     0.4424,
     0.6545
    ],
    "index": [
     0.5631,
     0.6625
    ]
   },
   "reason_lead": {
    "atlas": [
     0.0457,
     -0.0027,
     0.0914
    ],
    "deepswe": [
     0.0265,
     -0.0147,
     0.0708
    ],
    "tb4": [
     0.0424,
     -0.0061,
     0.0949
    ],
    "index": [
     0.0382,
     0.0111,
     0.0657
    ]
   }
  }
 ],
 "benchmarks": [
  {
   "id": "atlas",
   "label": "SWE-Atlas QnA",
   "tasks": 124,
   "rows": [
    {
     "harness": "Reason",
     "ours": true,
     "rate": 0.6398,
     "attempts": 372,
     "tasks": 124,
     "cost_per_attempt": 2.838,
     "total_cost": 1039,
     "cost_note": "11 attempts the provider refused 1 + 10 times score 0; cost is over the 366 attempts with a recorded cost, as for the other rows.",
     "input_per_attempt": 1154392,
     "output_per_attempt": 14075,
     "job": {
      "id": "0ac70bbf-10d8-5f45-8077-c428f10cb1a1",
      "url": "https://hub.harborframework.com/jobs/0ac70bbf-10d8-5f45-8077-c428f10cb1a1",
      "visibility": "public"
     },
     "ci95": [
      0.5591,
      0.7151
     ],
     "reason_lead": null
    },
    {
     "harness": "Codex",
     "ours": false,
     "rate": 0.6048,
     "attempts": 372,
     "tasks": 124,
     "cost_per_attempt": 3.705,
     "total_cost": 1378,
     "cost_note": null,
     "input_per_attempt": 1655511,
     "output_per_attempt": 16799,
     "job": {
      "id": "11a440fb-b1c4-473b-98a6-907500b3fdaf",
      "url": "https://hub.harborframework.com/jobs/11a440fb-b1c4-473b-98a6-907500b3fdaf",
      "visibility": "public"
     },
     "ci95": [
      0.5242,
      0.6828
     ],
     "reason_lead": [
      0.0349,
      -0.0134,
      0.0833
     ]
    },
    {
     "harness": "Pi",
     "ours": false,
     "rate": 0.5941,
     "attempts": 372,
     "tasks": 124,
     "cost_per_attempt": 2.94,
     "total_cost": 1094,
     "cost_note": null,
     "input_per_attempt": 1149726,
     "output_per_attempt": 15071,
     "job": {
      "id": "da4ac972-9d5a-4142-b057-85f07375f14e",
      "url": "https://hub.harborframework.com/jobs/da4ac972-9d5a-4142-b057-85f07375f14e",
      "visibility": "public"
     },
     "ci95": [
      0.5161,
      0.6694
     ],
     "reason_lead": [
      0.0457,
      -0.0027,
      0.0914
     ]
    }
   ]
  },
  {
   "id": "deepswe",
   "label": "DeepSWE v1.1",
   "tasks": 113,
   "rows": [
    {
     "harness": "Reason",
     "ours": true,
     "rate": 0.7227,
     "attempts": 339,
     "tasks": 113,
     "cost_per_attempt": 3.959,
     "total_cost": 1342,
     "cost_note": null,
     "input_per_attempt": 1686903,
     "output_per_attempt": 26543,
     "job": {
      "id": "0ac70bbf-10d8-5f45-8077-c428f10cb1a1",
      "url": "https://hub.harborframework.com/jobs/0ac70bbf-10d8-5f45-8077-c428f10cb1a1",
      "visibility": "public"
     },
     "ci95": [
      0.6431,
      0.7935
     ],
     "reason_lead": null
    },
    {
     "harness": "Codex",
     "ours": false,
     "rate": 0.6903,
     "attempts": 338,
     "tasks": 113,
     "cost_per_attempt": 4.875,
     "total_cost": 1648,
     "cost_note": null,
     "input_per_attempt": 2188918,
     "output_per_attempt": 30280,
     "job": {
      "id": "e20ecafd-695a-417c-90f5-35fd36d2786f",
      "url": "https://hub.harborframework.com/jobs/e20ecafd-695a-417c-90f5-35fd36d2786f",
      "visibility": "public"
     },
     "ci95": [
      0.6106,
      0.764
     ],
     "reason_lead": [
      0.0324,
      0.0,
      0.0678
     ]
    },
    {
     "harness": "Pi",
     "ours": false,
     "rate": 0.6962,
     "attempts": 339,
     "tasks": 113,
     "cost_per_attempt": 4.672,
     "total_cost": 1584,
     "cost_note": null,
     "input_per_attempt": 2130646,
     "output_per_attempt": 30032,
     "job": {
      "id": "cd7d8de6-c152-48de-9840-b22bbea96d99",
      "url": "https://hub.harborframework.com/jobs/cd7d8de6-c152-48de-9840-b22bbea96d99",
      "visibility": "public"
     },
     "ci95": [
      0.6195,
      0.7699
     ],
     "reason_lead": [
      0.0265,
      -0.0147,
      0.0708
     ]
    }
   ]
  },
  {
   "id": "tb4",
   "label": "Terminal-Bench 4.0",
   "tasks": 66,
   "rows": [
    {
     "harness": "Reason",
     "ours": true,
     "rate": 0.5909,
     "attempts": 198,
     "tasks": 66,
     "cost_per_attempt": 6.088,
     "total_cost": 1205,
     "cost_note": null,
     "input_per_attempt": 2916629,
     "output_per_attempt": 37744,
     "job": {
      "id": "0ac70bbf-10d8-5f45-8077-c428f10cb1a1",
      "url": "https://hub.harborframework.com/jobs/0ac70bbf-10d8-5f45-8077-c428f10cb1a1",
      "visibility": "public"
     },
     "ci95": [
      0.4848,
      0.702
     ],
     "reason_lead": null
    },
    {
     "harness": "Codex",
     "ours": false,
     "rate": 0.5788,
     "attempts": 330,
     "tasks": 66,
     "cost_per_attempt": 7.123,
     "total_cost": 2351,
     "cost_note": null,
     "input_per_attempt": 3596840,
     "output_per_attempt": 45304,
     "job": {
      "id": "caf6e433-9fff-4475-b0bc-977f14ac7052",
      "url": "https://hub.harborframework.com/jobs/caf6e433-9fff-4475-b0bc-977f14ac7052",
      "visibility": "public"
     },
     "ci95": [
      0.4727,
      0.6848
     ],
     "reason_lead": [
      0.0121,
      -0.0384,
      0.0606
     ]
    },
    {
     "harness": "Pi",
     "ours": false,
     "rate": 0.5502,
     "attempts": 329,
     "tasks": 66,
     "cost_per_attempt": 5.533,
     "total_cost": 1820,
     "cost_note": null,
     "input_per_attempt": 2743403,
     "output_per_attempt": 34694,
     "job": {
      "id": "6ccd097a-747b-4867-be57-a56969df988f",
      "url": "https://hub.harborframework.com/jobs/6ccd097a-747b-4867-be57-a56969df988f",
      "visibility": "public"
     },
     "ci95": [
      0.4424,
      0.6545
     ],
     "reason_lead": [
      0.0424,
      -0.0061,
      0.0949
     ]
    }
   ]
  }
 ],
 "interval_note": "95% intervals: 2,000 bootstrap resamples of tasks (seed 0), from every attempt in the linked Harbor jobs; refusals, timeouts and attempts without a result score 0. A lead resamples the same tasks for both harnesses; the Index resamples every benchmark's tasks together."
}