{"title": "Terminal-Bench 2 \u2014 Impact of harnesses on the same model", "subtitle": "", "harness": {"label": "Terminal-Bench 2", "baseline": {"key": "pi", "label": "Pi"}, "second": [{"key": "cc", "label": "Claude Code", "symbol": "diamond"}, {"key": "codex", "label": "Codex", "symbol": "triangle-up"}], "models": [{"key": "fable", "label": "Claude Fable 5", "color": "@model-fable", "paired": true}, {"key": "opus", "label": "Claude Opus 4.8", "color": "@model-opus", "paired": true}, {"key": "sonnet", "label": "Claude Sonnet 4.6", "color": "@model-sonnet", "paired": true}, {"key": "haiku", "label": "Claude Haiku 4.5", "color": "@model-haiku", "paired": true}, {"key": "sol", "label": "GPT-5.6 Sol", "color": "@model-sol", "paired": true}, {"key": "luna", "label": "GPT-5.6 Luna", "color": "@model-luna", "paired": true}, {"key": "kimi", "label": "Kimi K3", "color": "@model-kimi", "paired": false}], "rows": [{"kind": "pair", "id": "tb:terminal-bench-v2-common-n30-turncap100-20260906:claude-code-fable vs tb:terminal-bench-v2-common-n30-turncap100-20260906:pi-claude-fable", "model": "fable", "model_label": "Claude Fable 5", "color": "@model-fable", "harness": "cc", "harness_label": "Claude Code", "symbol": "diamond", "baseline": "pi", "baseline_label": "Pi", "a_label": "Claude Fable 5 \u00b7 Claude Code", "b_label": "Claude Fable 5 \u00b7 Pi", "cohort": "terminal-bench-v2-common-n30-turncap100-20260906", "matched": true, "reasons": [], "descriptive": true, "evidence": "descriptive common-n30 estimate; no confirmatory test", "caveats": [], "tasks": 30, "rate": {"mean": 0.044444, "ci": [-0.011111, 0.111111], "p": null, "p_holm": null, "clear": false, "covers_zero": true, "wins_a": 3, "wins_b": 1, "same": 26, "mass_a": 0.055556, "mass_b": 0.011111, "breakdown": [{"task": "adaptive-rejection-sampler", "a": [2, 3], "b": [0, 3], "delta": 0.666667}, {"task": "build-cython-ext", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "build-pmars", "a": [2, 3], "b": [2, 3], "delta": 0.0}, {"task": "code-from-image", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "compile-compcert", "a": [3, 3], "b": [2, 3], "delta": 0.333333}, {"task": "configure-git-webserver", "a": [0, 3], "b": [0, 3], "delta": 0.0}, {"task": "constraints-scheduling", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "feal-linear-cryptanalysis", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "financial-document-processor", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "fix-code-vulnerability", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "fix-git", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "gcode-to-text", "a": [2, 3], "b": [3, 3], "delta": -0.333333}, {"task": "headless-terminal", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "make-doom-for-mips", "a": [0, 3], "b": [0, 3], "delta": 0.0}, {"task": "merge-diff-arc-agi-task", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "multi-source-data-merger", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "nginx-request-logging", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "openssl-selfsigned-cert", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "path-tracing", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "path-tracing-reverse", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "pypi-server", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "query-optimize", "a": [0, 3], "b": [0, 3], "delta": 0.0}, {"task": "regex-log", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "reshard-c4-data", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "sam-cell-seg", "a": [0, 3], "b": [0, 3], "delta": 0.0}, {"task": "sparql-university", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "sqlite-db-truncate", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "torch-tensor-parallelism", "a": [2, 3], "b": [0, 3], "delta": 0.666667}, {"task": "train-fasttext", "a": [0, 3], "b": [0, 3], "delta": 0.0}, {"task": "video-processing", "a": [0, 3], "b": [0, 3], "delta": 0.0}]}, "cost": {"ratio": 1.441, "ratio_ci": [1.0828, 1.9401], "diff": 0.475596, "diff_ci": [0.106592, 0.824546], "clear": false, "costlier": 29, "n": 30, "median": 1.7786, "q1": 1.3261, "q3": 2.4439, "a_mean": 1.554103, "b_mean": 1.078506, "ratios": [{"task": "adaptive-rejection-sampler", "ratio": 3.0592, "a": 2.928855, "b": 0.957392}, {"task": "build-cython-ext", "ratio": 1.2649, "a": 3.14474, "b": 2.486217}, {"task": "build-pmars", "ratio": 3.0939, "a": 0.694183, "b": 0.224375}, {"task": "code-from-image", "ratio": 2.6075, "a": 0.181273, "b": 0.069519}, {"task": "compile-compcert", "ratio": 3.4632, "a": 2.404033, "b": 0.694172}, {"task": "configure-git-webserver", "ratio": 2.9498, "a": 0.648505, "b": 0.21985}, {"task": "constraints-scheduling", "ratio": 1.0418, "a": 0.323264, "b": 0.310281}, {"task": "feal-linear-cryptanalysis", "ratio": 1.0112, "a": 0.938825, "b": 0.928426}, {"task": "financial-document-processor", "ratio": 1.1724, "a": 1.544638, "b": 1.317452}, {"task": "fix-code-vulnerability", "ratio": 1.2542, "a": 0.343508, "b": 0.273893}, {"task": "fix-git", "ratio": 2.1568, "a": 0.340995, "b": 0.158105}, {"task": "gcode-to-text", "ratio": 1.3181, "a": 1.696913, "b": 1.28739}, {"task": "headless-terminal", "ratio": 1.8032, "a": 0.682422, "b": 0.378444}, {"task": "make-doom-for-mips", "ratio": 1.9197, "a": 7.008179, "b": 3.650598}, {"task": "merge-diff-arc-agi-task", "ratio": 2.6231, "a": 0.708009, "b": 0.269908}, {"task": "multi-source-data-merger", "ratio": 1.754, "a": 0.353215, "b": 0.20138}, {"task": "nginx-request-logging", "ratio": 2.4749, "a": 0.391801, "b": 0.158309}, {"task": "openssl-selfsigned-cert", "ratio": 2.056, "a": 0.350218, "b": 0.170339}, {"task": "path-tracing", "ratio": 1.3963, "a": 6.200773, "b": 4.440867}, {"task": "path-tracing-reverse", "ratio": 0.5986, "a": 4.866193, "b": 8.12863}, {"task": "pypi-server", "ratio": 2.3022, "a": 0.310048, "b": 0.134673}, {"task": "query-optimize", "ratio": 1.4851, "a": 0.439554, "b": 0.295982}, {"task": "regex-log", "ratio": 1.1881, "a": 0.626001, "b": 0.526886}, {"task": "reshard-c4-data", "ratio": 1.6538, "a": 1.56367, "b": 0.945478}, {"task": "sam-cell-seg", "ratio": 2.7706, "a": 2.641594, "b": 0.953446}, {"task": "sparql-university", "ratio": 1.3501, "a": 0.75308, "b": 0.557808}, {"task": "sqlite-db-truncate", "ratio": 1.5746, "a": 0.385301, "b": 0.244698}, {"task": "torch-tensor-parallelism", "ratio": 1.8807, "a": 0.813674, "b": 0.432652}, {"task": "train-fasttext", "ratio": 2.3508, "a": 1.619284, "b": 0.688814}, {"task": "video-processing", "ratio": 1.3771, "a": 1.720338, "b": 1.249211}]}}, {"kind": "pair", "id": "tb:terminal-bench-v2-common-n30-turncap100-20260906:claude-code-opus vs tb:terminal-bench-v2-common-n30-turncap100-20260906:pi-claude-opus", "model": "opus", "model_label": "Claude Opus 4.8", "color": "@model-opus", "harness": "cc", "harness_label": "Claude Code", "symbol": "diamond", "baseline": "pi", "baseline_label": "Pi", "a_label": "Claude Opus 4.8 \u00b7 Claude Code", "b_label": "Claude Opus 4.8 \u00b7 Pi", "cohort": "terminal-bench-v2-common-n30-turncap100-20260906", "matched": true, "reasons": [], "descriptive": true, "evidence": "descriptive common-n30 estimate; no confirmatory test", "caveats": [], "tasks": 30, "rate": {"mean": -0.033333, "ci": [-0.133333, 0.055556], "p": null, "p_holm": null, "clear": false, "covers_zero": true, "wins_a": 5, "wins_b": 5, "same": 20, "mass_a": 0.055556, "mass_b": 0.088889, "breakdown": [{"task": "adaptive-rejection-sampler", "a": [0, 3], "b": [0, 3], "delta": 0.0}, {"task": "build-cython-ext", "a": [3, 3], "b": [2, 3], "delta": 0.333333}, {"task": "build-pmars", "a": [3, 3], "b": [2, 3], "delta": 0.333333}, {"task": "code-from-image", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "compile-compcert", "a": [3, 3], "b": [2, 3], "delta": 0.333333}, {"task": "configure-git-webserver", "a": [0, 3], "b": [1, 3], "delta": -0.333333}, {"task": "constraints-scheduling", "a": [3, 3], "b": [2, 3], "delta": 0.333333}, {"task": "feal-linear-cryptanalysis", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "financial-document-processor", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "fix-code-vulnerability", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "fix-git", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "gcode-to-text", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "headless-terminal", "a": [3, 3], "b": [2, 3], "delta": 0.333333}, {"task": "make-doom-for-mips", "a": [0, 3], "b": [0, 3], "delta": 0.0}, {"task": "merge-diff-arc-agi-task", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "multi-source-data-merger", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "nginx-request-logging", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "openssl-selfsigned-cert", "a": [2, 3], "b": [3, 3], "delta": -0.333333}, {"task": "path-tracing", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "path-tracing-reverse", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "pypi-server", "a": [0, 3], "b": [3, 3], "delta": -1.0}, {"task": "query-optimize", "a": [0, 3], "b": [2, 3], "delta": -0.666667}, {"task": "regex-log", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "reshard-c4-data", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "sam-cell-seg", "a": [0, 3], "b": [0, 3], "delta": 0.0}, {"task": "sparql-university", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "sqlite-db-truncate", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "torch-tensor-parallelism", "a": [0, 3], "b": [1, 3], "delta": -0.333333}, {"task": "train-fasttext", "a": [0, 3], "b": [0, 3], "delta": 0.0}, {"task": "video-processing", "a": [0, 3], "b": [0, 3], "delta": 0.0}]}, "cost": {"ratio": 1.1865, "ratio_ci": [0.9444, 1.6952], "diff": 0.141385, "diff_ci": [-0.059341, 0.324035], "clear": false, "costlier": 25, "n": 30, "median": 1.6098, "q1": 1.1776, "q3": 1.9578, "a_mean": 0.899294, "b_mean": 0.757909, "ratios": [{"task": "adaptive-rejection-sampler", "ratio": 1.2961, "a": 1.248524, "b": 0.963275}, {"task": "build-cython-ext", "ratio": 1.9847, "a": 1.654464, "b": 0.833602}, {"task": "build-pmars", "ratio": 1.8049, "a": 0.453585, "b": 0.251302}, {"task": "code-from-image", "ratio": 2.1488, "a": 0.064928, "b": 0.030216}, {"task": "compile-compcert", "ratio": 2.2764, "a": 1.360015, "b": 0.597443}, {"task": "configure-git-webserver", "ratio": 1.5858, "a": 0.282473, "b": 0.17813}, {"task": "constraints-scheduling", "ratio": 1.1637, "a": 0.211958, "b": 0.182148}, {"task": "feal-linear-cryptanalysis", "ratio": 0.7717, "a": 0.823457, "b": 1.067136}, {"task": "financial-document-processor", "ratio": 1.3163, "a": 0.921701, "b": 0.700224}, {"task": "fix-code-vulnerability", "ratio": 1.0203, "a": 0.167536, "b": 0.16421}, {"task": "fix-git", "ratio": 2.181, "a": 0.249244, "b": 0.114282}, {"task": "gcode-to-text", "ratio": 1.6338, "a": 0.493922, "b": 0.302314}, {"task": "headless-terminal", "ratio": 1.667, "a": 0.325972, "b": 0.195547}, {"task": "make-doom-for-mips", "ratio": 0.898, "a": 3.946617, "b": 4.394871}, {"task": "merge-diff-arc-agi-task", "ratio": 2.4669, "a": 0.522005, "b": 0.211603}, {"task": "multi-source-data-merger", "ratio": 1.895, "a": 0.247905, "b": 0.130824}, {"task": "nginx-request-logging", "ratio": 1.7121, "a": 0.235389, "b": 0.137489}, {"task": "openssl-selfsigned-cert", "ratio": 1.5044, "a": 0.18312, "b": 0.121726}, {"task": "path-tracing", "ratio": 0.7186, "a": 1.855827, "b": 2.582508}, {"task": "path-tracing-reverse", "ratio": 0.6836, "a": 3.689278, "b": 5.39712}, {"task": "pypi-server", "ratio": 2.0566, "a": 0.206695, "b": 0.100501}, {"task": "query-optimize", "ratio": 1.7292, "a": 0.284417, "b": 0.164478}, {"task": "regex-log", "ratio": 1.3988, "a": 0.4339, "b": 0.310188}, {"task": "reshard-c4-data", "ratio": 1.5385, "a": 0.985532, "b": 0.640579}, {"task": "sam-cell-seg", "ratio": 1.9788, "a": 1.608669, "b": 0.812953}, {"task": "sparql-university", "ratio": 1.1222, "a": 0.477258, "b": 0.425307}, {"task": "sqlite-db-truncate", "ratio": 0.978, "a": 0.316061, "b": 0.323161}, {"task": "torch-tensor-parallelism", "ratio": 1.2192, "a": 0.301475, "b": 0.24728}, {"task": "train-fasttext", "ratio": 6.454, "a": 1.919001, "b": 0.297335}, {"task": "video-processing", "ratio": 1.7543, "a": 1.507877, "b": 0.859509}]}}, {"kind": "pair", "id": "tb:terminal-bench-v2-common-n30-turncap100-20260906:claude-code-sonnet vs tb:terminal-bench-v2-common-n30-turncap100-20260906:pi-claude-sonnet", "model": "sonnet", "model_label": "Claude Sonnet 4.6", "color": "@model-sonnet", "harness": "cc", "harness_label": "Claude Code", "symbol": "diamond", "baseline": "pi", "baseline_label": "Pi", "a_label": "Claude Sonnet 4.6 \u00b7 Claude Code", "b_label": "Claude Sonnet 4.6 \u00b7 Pi", "cohort": "terminal-bench-v2-common-n30-turncap100-20260906", "matched": true, "reasons": [], "descriptive": true, "evidence": "descriptive common-n30 estimate; no confirmatory test", "caveats": [], "tasks": 30, "rate": {"mean": -0.033333, "ci": [-0.111111, 0.033333], "p": null, "p_holm": null, "clear": false, "covers_zero": true, "wins_a": 3, "wins_b": 5, "same": 22, "mass_a": 0.033333, "mass_b": 0.066667, "breakdown": [{"task": "adaptive-rejection-sampler", "a": [0, 3], "b": [0, 3], "delta": 0.0}, {"task": "build-cython-ext", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "build-pmars", "a": [2, 3], "b": [3, 3], "delta": -0.333333}, {"task": "code-from-image", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "compile-compcert", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "configure-git-webserver", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "constraints-scheduling", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "feal-linear-cryptanalysis", "a": [0, 3], "b": [0, 3], "delta": 0.0}, {"task": "financial-document-processor", "a": [1, 3], "b": [3, 3], "delta": -0.666667}, {"task": "fix-code-vulnerability", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "fix-git", "a": [2, 3], "b": [3, 3], "delta": -0.333333}, {"task": "gcode-to-text", "a": [1, 3], "b": [0, 3], "delta": 0.333333}, {"task": "headless-terminal", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "make-doom-for-mips", "a": [0, 3], "b": [0, 3], "delta": 0.0}, {"task": "merge-diff-arc-agi-task", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "multi-source-data-merger", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "nginx-request-logging", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "openssl-selfsigned-cert", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "path-tracing", "a": [0, 3], "b": [0, 3], "delta": 0.0}, {"task": "path-tracing-reverse", "a": [0, 3], "b": [0, 3], "delta": 0.0}, {"task": "pypi-server", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "query-optimize", "a": [1, 3], "b": [0, 3], "delta": 0.333333}, {"task": "regex-log", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "reshard-c4-data", "a": [3, 3], "b": [2, 3], "delta": 0.333333}, {"task": "sam-cell-seg", "a": [0, 3], "b": [0, 3], "delta": 0.0}, {"task": "sparql-university", "a": [2, 3], "b": [3, 3], "delta": -0.333333}, {"task": "sqlite-db-truncate", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "torch-tensor-parallelism", "a": [2, 3], "b": [3, 3], "delta": -0.333333}, {"task": "train-fasttext", "a": [0, 3], "b": [0, 3], "delta": 0.0}, {"task": "video-processing", "a": [0, 3], "b": [0, 3], "delta": 0.0}]}, "cost": {"ratio": 1.0885, "ratio_ci": [0.9303, 1.3159], "diff": 0.054389, "diff_ci": [-0.048846, 0.162411], "clear": false, "costlier": 21, "n": 30, "median": 1.2248, "q1": 0.889, "q3": 1.7783, "a_mean": 0.668745, "b_mean": 0.614356, "ratios": [{"task": "adaptive-rejection-sampler", "ratio": 1.4051, "a": 0.89602, "b": 0.637714}, {"task": "build-cython-ext", "ratio": 1.4867, "a": 1.202163, "b": 0.80861}, {"task": "build-pmars", "ratio": 3.1733, "a": 0.257201, "b": 0.081052}, {"task": "code-from-image", "ratio": 1.785, "a": 0.052227, "b": 0.02926}, {"task": "compile-compcert", "ratio": 1.8562, "a": 0.903208, "b": 0.486577}, {"task": "configure-git-webserver", "ratio": 1.3912, "a": 0.149432, "b": 0.107412}, {"task": "constraints-scheduling", "ratio": 0.8504, "a": 0.16371, "b": 0.192501}, {"task": "feal-linear-cryptanalysis", "ratio": 2.9606, "a": 1.512274, "b": 0.510794}, {"task": "financial-document-processor", "ratio": 1.5285, "a": 0.316456, "b": 0.207034}, {"task": "fix-code-vulnerability", "ratio": 1.9674, "a": 0.640507, "b": 0.325562}, {"task": "fix-git", "ratio": 1.0438, "a": 0.096413, "b": 0.092368}, {"task": "gcode-to-text", "ratio": 0.7004, "a": 1.110708, "b": 1.585749}, {"task": "headless-terminal", "ratio": 1.0204, "a": 0.210716, "b": 0.206498}, {"task": "make-doom-for-mips", "ratio": 0.869, "a": 1.438197, "b": 1.655095}, {"task": "merge-diff-arc-agi-task", "ratio": 1.7583, "a": 0.327737, "b": 0.186392}, {"task": "multi-source-data-merger", "ratio": 1.0084, "a": 0.115636, "b": 0.114672}, {"task": "nginx-request-logging", "ratio": 2.7397, "a": 0.128771, "b": 0.047002}, {"task": "openssl-selfsigned-cert", "ratio": 1.299, "a": 0.08919, "b": 0.068662}, {"task": "path-tracing", "ratio": 1.0094, "a": 2.002595, "b": 1.983989}, {"task": "path-tracing-reverse", "ratio": 1.04, "a": 2.692608, "b": 2.589087}, {"task": "pypi-server", "ratio": 2.5709, "a": 0.151964, "b": 0.059109}, {"task": "query-optimize", "ratio": 1.8286, "a": 0.302375, "b": 0.165357}, {"task": "regex-log", "ratio": 1.2078, "a": 0.522083, "b": 0.432272}, {"task": "reshard-c4-data", "ratio": 0.8095, "a": 0.618547, "b": 0.764128}, {"task": "sam-cell-seg", "ratio": 0.625, "a": 0.831963, "b": 1.331214}, {"task": "sparql-university", "ratio": 0.9488, "a": 0.31956, "b": 0.336812}, {"task": "sqlite-db-truncate", "ratio": 0.7649, "a": 0.272596, "b": 0.356358}, {"task": "torch-tensor-parallelism", "ratio": 0.6884, "a": 0.164514, "b": 0.238993}, {"task": "train-fasttext", "ratio": 0.5727, "a": 0.806019, "b": 1.407506}, {"task": "video-processing", "ratio": 1.2418, "a": 1.766947, "b": 1.422895}]}}, {"kind": "pair", "id": "tb:terminal-bench-v2-common-n30-turncap100-20260906:claude-code-haiku vs tb:terminal-bench-v2-common-n30-turncap100-20260906:pi-claude-haiku", "model": "haiku", "model_label": "Claude Haiku 4.5", "color": "@model-haiku", "harness": "cc", "harness_label": "Claude Code", "symbol": "diamond", "baseline": "pi", "baseline_label": "Pi", "a_label": "Claude Haiku 4.5 \u00b7 Claude Code", "b_label": "Claude Haiku 4.5 \u00b7 Pi", "cohort": "terminal-bench-v2-common-n30-turncap100-20260906", "matched": true, "reasons": [], "descriptive": true, "evidence": "descriptive common-n30 estimate; no confirmatory test", "caveats": ["4 Claude Code runs hit the 100-turn cap (4 unsolved)"], "tasks": 30, "rate": {"mean": -0.066667, "ci": [-0.133333, 0.0], "p": null, "p_holm": null, "clear": false, "covers_zero": true, "wins_a": 2, "wins_b": 8, "same": 20, "mass_a": 0.022222, "mass_b": 0.088889, "breakdown": [{"task": "adaptive-rejection-sampler", "a": [1, 3], "b": [0, 3], "delta": 0.333333}, {"task": "build-cython-ext", "a": [0, 3], "b": [0, 3], "delta": 0.0}, {"task": "build-pmars", "a": [2, 3], "b": [2, 3], "delta": 0.0}, {"task": "code-from-image", "a": [2, 3], "b": [3, 3], "delta": -0.333333}, {"task": "compile-compcert", "a": [2, 3], "b": [3, 3], "delta": -0.333333}, {"task": "configure-git-webserver", "a": [0, 3], "b": [1, 3], "delta": -0.333333}, {"task": "constraints-scheduling", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "feal-linear-cryptanalysis", "a": [0, 3], "b": [0, 3], "delta": 0.0}, {"task": "financial-document-processor", "a": [1, 3], "b": [0, 3], "delta": 0.333333}, {"task": "fix-code-vulnerability", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "fix-git", "a": [2, 3], "b": [3, 3], "delta": -0.333333}, {"task": "gcode-to-text", "a": [0, 3], "b": [0, 3], "delta": 0.0}, {"task": "headless-terminal", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "make-doom-for-mips", "a": [0, 3], "b": [0, 3], "delta": 0.0}, {"task": "merge-diff-arc-agi-task", "a": [2, 3], "b": [2, 3], "delta": 0.0}, {"task": "multi-source-data-merger", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "nginx-request-logging", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "openssl-selfsigned-cert", "a": [2, 3], "b": [2, 3], "delta": 0.0}, {"task": "path-tracing", "a": [0, 3], "b": [0, 3], "delta": 0.0}, {"task": "path-tracing-reverse", "a": [0, 3], "b": [0, 3], "delta": 0.0}, {"task": "pypi-server", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "query-optimize", "a": [1, 3], "b": [2, 3], "delta": -0.333333}, {"task": "regex-log", "a": [1, 3], "b": [2, 3], "delta": -0.333333}, {"task": "reshard-c4-data", "a": [2, 3], "b": [3, 3], "delta": -0.333333}, {"task": "sam-cell-seg", "a": [0, 3], "b": [0, 3], "delta": 0.0}, {"task": "sparql-university", "a": [1, 3], "b": [2, 3], "delta": -0.333333}, {"task": "sqlite-db-truncate", "a": [0, 3], "b": [0, 3], "delta": 0.0}, {"task": "torch-tensor-parallelism", "a": [0, 3], "b": [0, 3], "delta": 0.0}, {"task": "train-fasttext", "a": [0, 3], "b": [0, 3], "delta": 0.0}, {"task": "video-processing", "a": [0, 3], "b": [0, 3], "delta": 0.0}]}, "cost": {"ratio": 1.0519, "ratio_ci": [0.871, 1.2703], "diff": 0.012956, "diff_ci": [-0.031567, 0.06311], "clear": false, "costlier": 15, "n": 30, "median": 0.9728, "q1": 0.785, "q3": 1.3486, "a_mean": 0.262522, "b_mean": 0.249566, "ratios": [{"task": "adaptive-rejection-sampler", "ratio": 0.9366, "a": 0.658132, "b": 0.7027}, {"task": "build-cython-ext", "ratio": 1.048, "a": 0.462835, "b": 0.441636}, {"task": "build-pmars", "ratio": 1.372, "a": 0.13629, "b": 0.099337}, {"task": "code-from-image", "ratio": 4.5918, "a": 0.091572, "b": 0.019942}, {"task": "compile-compcert", "ratio": 1.7469, "a": 0.488811, "b": 0.279814}, {"task": "configure-git-webserver", "ratio": 0.1326, "a": 0.009477, "b": 0.071448}, {"task": "constraints-scheduling", "ratio": 0.438, "a": 0.052587, "b": 0.12005}, {"task": "feal-linear-cryptanalysis", "ratio": 2.0269, "a": 0.875798, "b": 0.43208}, {"task": "financial-document-processor", "ratio": 0.5985, "a": 0.142011, "b": 0.237262}, {"task": "fix-code-vulnerability", "ratio": 1.7592, "a": 0.257801, "b": 0.146544}, {"task": "fix-git", "ratio": 1.0052, "a": 0.039098, "b": 0.038895}, {"task": "gcode-to-text", "ratio": 0.3255, "a": 0.086104, "b": 0.264532}, {"task": "headless-terminal", "ratio": 1.1509, "a": 0.059978, "b": 0.052115}, {"task": "make-doom-for-mips", "ratio": 0.8859, "a": 0.993393, "b": 1.12135}, {"task": "merge-diff-arc-agi-task", "ratio": 0.583, "a": 0.179323, "b": 0.307612}, {"task": "multi-source-data-merger", "ratio": 0.9068, "a": 0.052411, "b": 0.0578}, {"task": "nginx-request-logging", "ratio": 1.5553, "a": 0.092211, "b": 0.059287}, {"task": "openssl-selfsigned-cert", "ratio": 1.1651, "a": 0.061147, "b": 0.052483}, {"task": "path-tracing", "ratio": 0.8476, "a": 0.793752, "b": 0.936448}, {"task": "path-tracing-reverse", "ratio": 1.4883, "a": 1.01859, "b": 0.684398}, {"task": "pypi-server", "ratio": 1.0499, "a": 0.060954, "b": 0.058057}, {"task": "query-optimize", "ratio": 1.2784, "a": 0.056908, "b": 0.044515}, {"task": "regex-log", "ratio": 1.062, "a": 0.099951, "b": 0.094114}, {"task": "reshard-c4-data", "ratio": 0.7774, "a": 0.189297, "b": 0.243501}, {"task": "sam-cell-seg", "ratio": 0.4255, "a": 0.111441, "b": 0.261889}, {"task": "sparql-university", "ratio": 0.8762, "a": 0.094894, "b": 0.108297}, {"task": "sqlite-db-truncate", "ratio": 0.7551, "a": 0.083287, "b": 0.110297}, {"task": "torch-tensor-parallelism", "ratio": 0.8078, "a": 0.073075, "b": 0.09046}, {"task": "train-fasttext", "ratio": 2.6677, "a": 0.347963, "b": 0.130437}, {"task": "video-processing", "ratio": 0.9403, "a": 0.206568, "b": 0.219677}]}}, {"kind": "pair", "id": "tb:terminal-bench-v2-common-n30-turncap100-20260906:codex-sol vs tb:terminal-bench-v2-common-n30-turncap100-20260906:pi-gpt-sol", "model": "sol", "model_label": "GPT-5.6 Sol", "color": "@model-sol", "harness": "codex", "harness_label": "Codex", "symbol": "triangle-up", "baseline": "pi", "baseline_label": "Pi", "a_label": "GPT-5.6 Sol \u00b7 Codex", "b_label": "GPT-5.6 Sol \u00b7 Pi", "cohort": "terminal-bench-v2-common-n30-turncap100-20260906", "matched": true, "reasons": [], "descriptive": true, "evidence": "descriptive common-n30 estimate; no confirmatory test", "caveats": [], "tasks": 30, "rate": {"mean": -0.044444, "ci": [-0.122222, 0.011111], "p": null, "p_holm": null, "clear": false, "covers_zero": true, "wins_a": 1, "wins_b": 3, "same": 26, "mass_a": 0.011111, "mass_b": 0.055556, "breakdown": [{"task": "adaptive-rejection-sampler", "a": [2, 3], "b": [3, 3], "delta": -0.333333}, {"task": "build-cython-ext", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "build-pmars", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "code-from-image", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "compile-compcert", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "configure-git-webserver", "a": [0, 3], "b": [0, 3], "delta": 0.0}, {"task": "constraints-scheduling", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "feal-linear-cryptanalysis", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "financial-document-processor", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "fix-code-vulnerability", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "fix-git", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "gcode-to-text", "a": [2, 3], "b": [1, 3], "delta": 0.333333}, {"task": "headless-terminal", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "make-doom-for-mips", "a": [0, 3], "b": [1, 3], "delta": -0.333333}, {"task": "merge-diff-arc-agi-task", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "multi-source-data-merger", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "nginx-request-logging", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "openssl-selfsigned-cert", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "path-tracing", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "path-tracing-reverse", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "pypi-server", "a": [0, 3], "b": [3, 3], "delta": -1.0}, {"task": "query-optimize", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "regex-log", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "reshard-c4-data", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "sam-cell-seg", "a": [0, 3], "b": [0, 3], "delta": 0.0}, {"task": "sparql-university", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "sqlite-db-truncate", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "torch-tensor-parallelism", "a": [1, 3], "b": [1, 3], "delta": 0.0}, {"task": "train-fasttext", "a": [2, 3], "b": [2, 3], "delta": 0.0}, {"task": "video-processing", "a": [1, 3], "b": [1, 3], "delta": 0.0}]}, "cost": {"ratio": 1.81, "ratio_ci": [1.4725, 2.274], "diff": 0.340617, "diff_ci": [0.163445, 0.563538], "clear": false, "costlier": 29, "n": 30, "median": 1.693, "q1": 1.4929, "q3": 2.0379, "a_mean": 0.761134, "b_mean": 0.420517, "ratios": [{"task": "adaptive-rejection-sampler", "ratio": 1.4666, "a": 0.781467, "b": 0.532842}, {"task": "build-cython-ext", "ratio": 1.0378, "a": 0.973634, "b": 0.938174}, {"task": "build-pmars", "ratio": 1.6264, "a": 0.591271, "b": 0.363538}, {"task": "code-from-image", "ratio": 2.625, "a": 0.085739, "b": 0.032662}, {"task": "compile-compcert", "ratio": 2.8621, "a": 2.639034, "b": 0.922072}, {"task": "configure-git-webserver", "ratio": 1.6105, "a": 0.43248, "b": 0.268545}, {"task": "constraints-scheduling", "ratio": 2.1871, "a": 0.119009, "b": 0.054414}, {"task": "feal-linear-cryptanalysis", "ratio": 1.5718, "a": 0.719037, "b": 0.457457}, {"task": "financial-document-processor", "ratio": 1.6564, "a": 0.7631, "b": 0.460697}, {"task": "fix-code-vulnerability", "ratio": 1.4207, "a": 0.291509, "b": 0.20519}, {"task": "fix-git", "ratio": 2.3999, "a": 0.245172, "b": 0.102159}, {"task": "gcode-to-text", "ratio": 1.3539, "a": 0.47028, "b": 0.347348}, {"task": "headless-terminal", "ratio": 1.4475, "a": 0.247775, "b": 0.171177}, {"task": "make-doom-for-mips", "ratio": 1.7135, "a": 4.22329, "b": 2.464657}, {"task": "merge-diff-arc-agi-task", "ratio": 2.0412, "a": 0.307245, "b": 0.150521}, {"task": "multi-source-data-merger", "ratio": 2.0128, "a": 0.136442, "b": 0.067789}, {"task": "nginx-request-logging", "ratio": 1.7497, "a": 0.166702, "b": 0.095273}, {"task": "openssl-selfsigned-cert", "ratio": 2.8634, "a": 0.170143, "b": 0.059419}, {"task": "path-tracing", "ratio": 1.6726, "a": 1.56089, "b": 0.933233}, {"task": "path-tracing-reverse", "ratio": 1.1628, "a": 1.049232, "b": 0.902352}, {"task": "pypi-server", "ratio": 2.0282, "a": 0.243162, "b": 0.119891}, {"task": "query-optimize", "ratio": 1.7702, "a": 0.715956, "b": 0.404449}, {"task": "regex-log", "ratio": 1.5829, "a": 0.116922, "b": 0.073865}, {"task": "reshard-c4-data", "ratio": 1.5956, "a": 0.599713, "b": 0.375865}, {"task": "sam-cell-seg", "ratio": 1.862, "a": 0.703906, "b": 0.378046}, {"task": "sparql-university", "ratio": 2.5735, "a": 0.270509, "b": 0.105115}, {"task": "sqlite-db-truncate", "ratio": 1.7844, "a": 0.164914, "b": 0.092419}, {"task": "torch-tensor-parallelism", "ratio": 1.4295, "a": 0.276353, "b": 0.193324}, {"task": "train-fasttext", "ratio": 4.8297, "a": 3.062199, "b": 0.634039}, {"task": "video-processing", "ratio": 0.9971, "a": 0.706942, "b": 0.708983}]}}, {"kind": "pair", "id": "tb:terminal-bench-v2-common-n30-turncap100-20260906:codex-luna vs tb:terminal-bench-v2-common-n30-turncap100-20260906:pi-gpt-luna", "model": "luna", "model_label": "GPT-5.6 Luna", "color": "@model-luna", "harness": "codex", "harness_label": "Codex", "symbol": "triangle-up", "baseline": "pi", "baseline_label": "Pi", "a_label": "GPT-5.6 Luna \u00b7 Codex", "b_label": "GPT-5.6 Luna \u00b7 Pi", "cohort": "terminal-bench-v2-common-n30-turncap100-20260906", "matched": true, "reasons": [], "descriptive": true, "evidence": "descriptive common-n30 estimate; no confirmatory test", "caveats": [], "tasks": 30, "rate": {"mean": -0.044444, "ci": [-0.155556, 0.044444], "p": null, "p_holm": null, "clear": false, "covers_zero": true, "wins_a": 5, "wins_b": 6, "same": 19, "mass_a": 0.055556, "mass_b": 0.1, "breakdown": [{"task": "adaptive-rejection-sampler", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "build-cython-ext", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "build-pmars", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "code-from-image", "a": [2, 3], "b": [3, 3], "delta": -0.333333}, {"task": "compile-compcert", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "configure-git-webserver", "a": [0, 3], "b": [0, 3], "delta": 0.0}, {"task": "constraints-scheduling", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "feal-linear-cryptanalysis", "a": [3, 3], "b": [2, 3], "delta": 0.333333}, {"task": "financial-document-processor", "a": [3, 3], "b": [2, 3], "delta": 0.333333}, {"task": "fix-code-vulnerability", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "fix-git", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "gcode-to-text", "a": [0, 3], "b": [0, 3], "delta": 0.0}, {"task": "headless-terminal", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "make-doom-for-mips", "a": [1, 3], "b": [2, 3], "delta": -0.333333}, {"task": "merge-diff-arc-agi-task", "a": [2, 3], "b": [3, 3], "delta": -0.333333}, {"task": "multi-source-data-merger", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "nginx-request-logging", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "openssl-selfsigned-cert", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "path-tracing", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "path-tracing-reverse", "a": [3, 3], "b": [2, 3], "delta": 0.333333}, {"task": "pypi-server", "a": [0, 3], "b": [3, 3], "delta": -1.0}, {"task": "query-optimize", "a": [1, 3], "b": [3, 3], "delta": -0.666667}, {"task": "regex-log", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "reshard-c4-data", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "sam-cell-seg", "a": [0, 3], "b": [0, 3], "delta": 0.0}, {"task": "sparql-university", "a": [2, 3], "b": [3, 3], "delta": -0.333333}, {"task": "sqlite-db-truncate", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "torch-tensor-parallelism", "a": [2, 3], "b": [1, 3], "delta": 0.333333}, {"task": "train-fasttext", "a": [0, 3], "b": [0, 3], "delta": 0.0}, {"task": "video-processing", "a": [1, 3], "b": [0, 3], "delta": 0.333333}]}, "cost": {"ratio": 1.4109, "ratio_ci": [1.074, 2.0644], "diff": 0.018629, "diff_ci": [0.003461, 0.039301], "clear": false, "costlier": 26, "n": 30, "median": 1.6098, "q1": 1.2141, "q3": 2.0103, "a_mean": 0.06397, "b_mean": 0.045341, "ratios": [{"task": "adaptive-rejection-sampler", "ratio": 1.5601, "a": 0.079611, "b": 0.051028}, {"task": "build-cython-ext", "ratio": 0.8478, "a": 0.066093, "b": 0.077956}, {"task": "build-pmars", "ratio": 1.7738, "a": 0.041162, "b": 0.023206}, {"task": "code-from-image", "ratio": 11.2297, "a": 0.01861, "b": 0.001657}, {"task": "compile-compcert", "ratio": 4.3391, "a": 0.255493, "b": 0.058882}, {"task": "configure-git-webserver", "ratio": 1.4513, "a": 0.02885, "b": 0.019879}, {"task": "constraints-scheduling", "ratio": 2.0648, "a": 0.005464, "b": 0.002646}, {"task": "feal-linear-cryptanalysis", "ratio": 0.8358, "a": 0.069982, "b": 0.083731}, {"task": "financial-document-processor", "ratio": 1.2135, "a": 0.050193, "b": 0.041362}, {"task": "fix-code-vulnerability", "ratio": 2.1019, "a": 0.016076, "b": 0.007648}, {"task": "fix-git", "ratio": 1.8811, "a": 0.012985, "b": 0.006903}, {"task": "gcode-to-text", "ratio": 0.8773, "a": 0.095709, "b": 0.109091}, {"task": "headless-terminal", "ratio": 1.3742, "a": 0.018748, "b": 0.013643}, {"task": "make-doom-for-mips", "ratio": 1.1202, "a": 0.257114, "b": 0.229525}, {"task": "merge-diff-arc-agi-task", "ratio": 1.9284, "a": 0.025686, "b": 0.01332}, {"task": "multi-source-data-merger", "ratio": 1.9645, "a": 0.00873, "b": 0.004444}, {"task": "nginx-request-logging", "ratio": 1.9633, "a": 0.012319, "b": 0.006274}, {"task": "openssl-selfsigned-cert", "ratio": 2.6345, "a": 0.006981, "b": 0.00265}, {"task": "path-tracing", "ratio": 1.1119, "a": 0.18966, "b": 0.170578}, {"task": "path-tracing-reverse", "ratio": 0.7694, "a": 0.170879, "b": 0.222085}, {"task": "pypi-server", "ratio": 2.2003, "a": 0.020608, "b": 0.009366}, {"task": "query-optimize", "ratio": 1.6055, "a": 0.025795, "b": 0.016067}, {"task": "regex-log", "ratio": 2.0255, "a": 0.010866, "b": 0.005365}, {"task": "reshard-c4-data", "ratio": 1.0685, "a": 0.042028, "b": 0.039333}, {"task": "sam-cell-seg", "ratio": 1.614, "a": 0.036406, "b": 0.022557}, {"task": "sparql-university", "ratio": 1.8497, "a": 0.012901, "b": 0.006975}, {"task": "sqlite-db-truncate", "ratio": 1.3783, "a": 0.010895, "b": 0.007904}, {"task": "torch-tensor-parallelism", "ratio": 1.3705, "a": 0.013888, "b": 0.010133}, {"task": "train-fasttext", "ratio": 7.7549, "a": 0.235545, "b": 0.030374}, {"task": "video-processing", "ratio": 1.2158, "a": 0.079831, "b": 0.06566}]}}, {"kind": "single", "id": "tb:terminal-bench-v2-common-n30-turncap100-20260906:pi-kimi-k3", "model": "kimi", "model_label": "Kimi K3", "color": "@model-kimi", "harness": "pi", "harness_label": "Pi", "symbol": "square", "label": "Kimi K3 \u00b7 Pi", "note": "Pi only \u2014 no second harness to compare", "rate": 0.7333333333333333, "cost": 0.38317976666666664}], "frontier_points": [{"id": "tb:terminal-bench-v2-common-n30-turncap100-20260906:claude-code-fable", "label": "cc:fable", "cohort": "terminal-bench-v2-common-n30-turncap100-20260906", "qualifier": null, "sample": "30 unique tasks over 3 repetitions", "model": "fable", "model_label": "Claude Fable 5", "harness": "cc", "harness_label": "Claude Code", "x": 1.554103, "y": 0.755556, "x_ci": [0.992643, 2.235054], "y_ci": [0.611111, 0.888889], "frontier": false, "color": "@model-fable", "symbol": "diamond", "bootstrap": 0.0095, "single_replicate": false, "publication_scope": null, "replicates": 3}, {"id": "tb:terminal-bench-v2-common-n30-turncap100-20260906:claude-code-haiku", "label": "cc:haiku", "cohort": "terminal-bench-v2-common-n30-turncap100-20260906", "qualifier": null, "sample": "30 unique tasks over 3 repetitions \u00b7 4 Claude Code --max-turns 100 cap hits (4 unsolved)", "model": "haiku", "model_label": "Claude Haiku 4.5", "harness": "cc", "harness_label": "Claude Code", "x": 0.262522, "y": 0.411111, "x_ci": [0.162475, 0.377011], "y_ci": [0.277778, 0.555556], "frontier": false, "color": "@model-haiku", "symbol": "diamond", "bootstrap": 0.0, "single_replicate": false, "publication_scope": null, "replicates": 3}, {"id": "tb:terminal-bench-v2-common-n30-turncap100-20260906:claude-code-opus", "label": "cc:opus", "cohort": "terminal-bench-v2-common-n30-turncap100-20260906", "qualifier": null, "sample": "30 unique tasks over 3 repetitions", "model": "opus", "model_label": "Claude Opus 4.8", "harness": "cc", "harness_label": "Claude Code", "x": 0.899294, "y": 0.688889, "x_ci": [0.589994, 1.276792], "y_ci": [0.522222, 0.844444], "frontier": false, "color": "@model-opus", "symbol": "diamond", "bootstrap": 0.0005, "single_replicate": false, "publication_scope": null, "replicates": 3}, {"id": "tb:terminal-bench-v2-common-n30-turncap100-20260906:claude-code-sonnet", "label": "cc:sonnet", "cohort": "terminal-bench-v2-common-n30-turncap100-20260906", "qualifier": null, "sample": "30 unique tasks over 3 repetitions", "model": "sonnet", "model_label": "Claude Sonnet 4.6", "harness": "cc", "harness_label": "Claude Code", "x": 0.668745, "y": 0.622222, "x_ci": [0.446929, 0.917128], "y_ci": [0.466667, 0.766667], "frontier": false, "color": "@model-sonnet", "symbol": "diamond", "bootstrap": 0.0005, "single_replicate": false, "publication_scope": null, "replicates": 3}, {"id": "tb:terminal-bench-v2-common-n30-turncap100-20260906:codex-luna", "label": "codex:luna", "cohort": "terminal-bench-v2-common-n30-turncap100-20260906", "qualifier": null, "sample": "30 unique tasks over 3 repetitions", "model": "luna", "model_label": "GPT-5.6 Luna", "harness": "codex", "harness_label": "Codex", "x": 0.06397, "y": 0.722222, "x_ci": [0.038652, 0.092664], "y_ci": [0.577778, 0.855556], "frontier": false, "color": "@model-luna", "symbol": "triangle-up", "bootstrap": 0.1745, "single_replicate": false, "publication_scope": null, "replicates": 3}, {"id": "tb:terminal-bench-v2-common-n30-turncap100-20260906:codex-sol", "label": "codex:sol", "cohort": "terminal-bench-v2-common-n30-turncap100-20260906", "qualifier": null, "sample": "30 unique tasks over 3 repetitions", "model": "sol", "model_label": "GPT-5.6 Sol", "harness": "codex", "harness_label": "Codex", "x": 0.761134, "y": 0.788889, "x_ci": [0.46322, 1.130819], "y_ci": [0.655556, 0.911111], "frontier": false, "color": "@model-sol", "symbol": "triangle-up", "bootstrap": 0.0645, "single_replicate": false, "publication_scope": null, "replicates": 3}, {"id": "tb:terminal-bench-v2-common-n30-turncap100-20260906:pi-claude-fable", "label": "pi:fable", "cohort": "terminal-bench-v2-common-n30-turncap100-20260906", "qualifier": null, "sample": "30 unique tasks over 3 repetitions", "model": "fable", "model_label": "Claude Fable 5", "harness": "pi", "harness_label": "Pi", "x": 1.078506, "y": 0.711111, "x_ci": [0.571387, 1.74867], "y_ci": [0.544444, 0.855556], "frontier": false, "color": "@model-fable", "symbol": "square", "bootstrap": 0.001, "single_replicate": false, "publication_scope": null, "replicates": 3}, {"id": "tb:terminal-bench-v2-common-n30-turncap100-20260906:pi-claude-haiku", "label": "pi:haiku", "cohort": "terminal-bench-v2-common-n30-turncap100-20260906", "qualifier": null, "sample": "30 unique tasks over 3 repetitions", "model": "haiku", "model_label": "Claude Haiku 4.5", "harness": "pi", "harness_label": "Pi", "x": 0.249566, "y": 0.477778, "x_ci": [0.160236, 0.355113], "y_ci": [0.322222, 0.633333], "frontier": false, "color": "@model-haiku", "symbol": "square", "bootstrap": 0.0, "single_replicate": false, "publication_scope": null, "replicates": 3}, {"id": "tb:terminal-bench-v2-common-n30-turncap100-20260906:pi-claude-opus", "label": "pi:opus", "cohort": "terminal-bench-v2-common-n30-turncap100-20260906", "qualifier": null, "sample": "30 unique tasks over 3 repetitions", "model": "opus", "model_label": "Claude Opus 4.8", "harness": "pi", "harness_label": "Pi", "x": 0.757909, "y": 0.722222, "x_ci": [0.384428, 1.245274], "y_ci": [0.588889, 0.844444], "frontier": false, "color": "@model-opus", "symbol": "square", "bootstrap": 0.0075, "single_replicate": false, "publication_scope": null, "replicates": 3}, {"id": "tb:terminal-bench-v2-common-n30-turncap100-20260906:pi-claude-sonnet", "label": "pi:sonnet", "cohort": "terminal-bench-v2-common-n30-turncap100-20260906", "qualifier": null, "sample": "30 unique tasks over 3 repetitions", "model": "sonnet", "model_label": "Claude Sonnet 4.6", "harness": "pi", "harness_label": "Pi", "x": 0.614356, "y": 0.655556, "x_ci": [0.390826, 0.8692], "y_ci": [0.488889, 0.822222], "frontier": false, "color": "@model-sonnet", "symbol": "square", "bootstrap": 0.012, "single_replicate": false, "publication_scope": null, "replicates": 3}, {"id": "tb:terminal-bench-v2-common-n30-turncap100-20260906:pi-gpt-luna", "label": "pi:luna", "cohort": "terminal-bench-v2-common-n30-turncap100-20260906", "qualifier": null, "sample": "30 unique tasks over 3 repetitions", "model": "luna", "model_label": "GPT-5.6 Luna", "harness": "pi", "harness_label": "Pi", "x": 0.045341, "y": 0.766667, "x_ci": [0.025556, 0.06917], "y_ci": [0.622222, 0.9], "frontier": true, "color": "@model-luna", "symbol": "square", "bootstrap": 0.9995, "single_replicate": false, "publication_scope": null, "replicates": 3}, {"id": "tb:terminal-bench-v2-common-n30-turncap100-20260906:pi-gpt-sol", "label": "pi:sol", "cohort": "terminal-bench-v2-common-n30-turncap100-20260906", "qualifier": null, "sample": "30 unique tasks over 3 repetitions", "model": "sol", "model_label": "GPT-5.6 Sol", "harness": "pi", "harness_label": "Pi", "x": 0.420517, "y": 0.833333, "x_ci": [0.272168, 0.61226], "y_ci": [0.711111, 0.944444], "frontier": true, "color": "@model-sol", "symbol": "square", "bootstrap": 0.9365, "single_replicate": false, "publication_scope": null, "replicates": 3}, {"id": "tb:terminal-bench-v2-common-n30-turncap100-20260906:pi-kimi-k3", "label": "pi:kimi", "cohort": "terminal-bench-v2-common-n30-turncap100-20260906", "qualifier": null, "sample": "30 unique tasks over 3 repetitions", "model": "kimi", "model_label": "Kimi K3", "harness": "pi", "harness_label": "Pi", "x": 0.38318, "y": 0.733333, "x_ci": [0.238695, 0.543259], "y_ci": [0.6, 0.855556], "frontier": false, "color": "@model-kimi", "symbol": "square", "bootstrap": 0.1325, "single_replicate": false, "publication_scope": null, "replicates": 3}, {"id": "tb:arena-cross-r1-r3:kimi:cc:sha256:0dd54cc5bfd048c3aa8edfc5af785146944b2ffffe540beb9258721caebde609", "label": "cc:kimi", "cohort": "Arena cross-family r1-r3", "qualifier": null, "sample": "30 unique tasks over 3 repetitions", "model": "kimi", "model_label": "Kimi K3", "harness": "cc", "harness_label": "Claude Code", "x": 0.521312, "y": 0.666667, "x_ci": [0.355968, 0.707199], "y_ci": [0.522222, 0.8], "frontier": false, "color": "@model-kimi", "symbol": "diamond", "bootstrap": 0.0, "single_replicate": false, "publication_scope": "arena-cross-family-r1-r3", "replicates": 3}, {"id": "tb:arena-cross-r1-r3:luna:cc:sha256:0dd54cc5bfd048c3aa8edfc5af785146944b2ffffe540beb9258721caebde609", "label": "cc:luna", "cohort": "Arena cross-family r1-r3", "qualifier": null, "sample": "30 unique tasks over 3 repetitions", "model": "luna", "model_label": "GPT-5.6 Luna", "harness": "cc", "harness_label": "Claude Code", "x": 0.098445, "y": 0.7, "x_ci": [0.067636, 0.13316], "y_ci": [0.555556, 0.833333], "frontier": false, "color": "@model-luna", "symbol": "diamond", "bootstrap": 0.0795, "single_replicate": false, "publication_scope": "arena-cross-family-r1-r3", "replicates": 3}, {"id": "tb:arena-cross-r1-r3:sol:cc:sha256:0dd54cc5bfd048c3aa8edfc5af785146944b2ffffe540beb9258721caebde609", "label": "cc:sol", "cohort": "Arena cross-family r1-r3", "qualifier": null, "sample": "30 unique tasks over 3 repetitions", "model": "sol", "model_label": "GPT-5.6 Sol", "harness": "cc", "harness_label": "Claude Code", "x": 1.354545, "y": 0.711111, "x_ci": [0.94836, 1.852551], "y_ci": [0.555556, 0.855556], "frontier": false, "color": "@model-sol", "symbol": "diamond", "bootstrap": 0.007, "single_replicate": false, "publication_scope": "arena-cross-family-r1-r3", "replicates": 3}, {"id": "tb:arena-cross-r1-r3:fable:codex:sha256:0dd54cc5bfd048c3aa8edfc5af785146944b2ffffe540beb9258721caebde609", "label": "codex:fable", "cohort": "Arena cross-family r1-r3", "qualifier": null, "sample": "30 unique tasks over 3 repetitions", "model": "fable", "model_label": "Claude Fable 5", "harness": "codex", "harness_label": "Codex", "x": 0.976038, "y": 0.722222, "x_ci": [0.631696, 1.406231], "y_ci": [0.566667, 0.866667], "frontier": false, "color": "@model-fable", "symbol": "triangle-up", "bootstrap": 0.013, "single_replicate": false, "publication_scope": "arena-cross-family-r1-r3", "replicates": 3}, {"id": "tb:arena-cross-r1-r3:haiku:codex:sha256:0dd54cc5bfd048c3aa8edfc5af785146944b2ffffe540beb9258721caebde609", "label": "codex:haiku", "cohort": "Arena cross-family r1-r3", "qualifier": null, "sample": "30 unique tasks over 3 repetitions", "model": "haiku", "model_label": "Claude Haiku 4.5", "harness": "codex", "harness_label": "Codex", "x": 0.213731, "y": 0.311111, "x_ci": [0.142742, 0.293352], "y_ci": [0.155556, 0.477778], "frontier": false, "color": "@model-haiku", "symbol": "triangle-up", "bootstrap": 0.0, "single_replicate": false, "publication_scope": "arena-cross-family-r1-r3", "replicates": 3}, {"id": "tb:arena-cross-r1-r3:kimi:codex:sha256:0dd54cc5bfd048c3aa8edfc5af785146944b2ffffe540beb9258721caebde609", "label": "codex:kimi", "cohort": "Arena cross-family r1-r3", "qualifier": null, "sample": "30 unique tasks over 3 repetitions", "model": "kimi", "model_label": "Kimi K3", "harness": "codex", "harness_label": "Codex", "x": 0.449972, "y": 0.7, "x_ci": [0.283386, 0.656948], "y_ci": [0.555556, 0.833333], "frontier": false, "color": "@model-kimi", "symbol": "triangle-up", "bootstrap": 0.002, "single_replicate": false, "publication_scope": "arena-cross-family-r1-r3", "replicates": 3}, {"id": "tb:arena-cross-r1-r3:opus:codex:sha256:0dd54cc5bfd048c3aa8edfc5af785146944b2ffffe540beb9258721caebde609", "label": "codex:opus", "cohort": "Arena cross-family r1-r3", "qualifier": null, "sample": "30 unique tasks over 3 repetitions", "model": "opus", "model_label": "Claude Opus 4.8", "harness": "codex", "harness_label": "Codex", "x": 0.847954, "y": 0.722222, "x_ci": [0.507623, 1.269417], "y_ci": [0.566667, 0.866667], "frontier": false, "color": "@model-opus", "symbol": "triangle-up", "bootstrap": 0.0125, "single_replicate": false, "publication_scope": "arena-cross-family-r1-r3", "replicates": 3}, {"id": "tb:arena-cross-r1-r3:sonnet:codex:sha256:0dd54cc5bfd048c3aa8edfc5af785146944b2ffffe540beb9258721caebde609", "label": "codex:sonnet", "cohort": "Arena cross-family r1-r3", "qualifier": null, "sample": "30 unique tasks over 3 repetitions", "model": "sonnet", "model_label": "Claude Sonnet 4.6", "harness": "codex", "harness_label": "Codex", "x": 0.552243, "y": 0.633333, "x_ci": [0.38034, 0.741388], "y_ci": [0.466667, 0.788889], "frontier": false, "color": "@model-sonnet", "symbol": "triangle-up", "bootstrap": 0.002, "single_replicate": false, "publication_scope": "arena-cross-family-r1-r3", "replicates": 3}]}, "plotly": {"data": [], "layout": {}}}