{"title": "Terminal-Bench 2 \u2014 performance-cost Pareto frontier", "subtitle": null, "frontier": {"x_label": "cost per rollout (USD, log scale)", "x_scale": "log", "y_label": "success rate", "marker_size": {"frontier": 18, "dominated": 12}, "points": [{"id": "tb:terminal-bench-v2-common-n30-turncap100-20260906:claude-code-fable", "label": "cc:fable", "cohort": "terminal-bench-v2-common-n30-turncap100-20260906", "qualifier": null, "sample": "30 unique tasks over 3 repetitions", "model": "fable", "model_label": "Claude Fable 5", "harness": "cc", "harness_label": "Claude Code", "x": 1.554103, "y": 0.755556, "x_ci": [0.992643, 2.235054], "y_ci": [0.611111, 0.888889], "frontier": false, "color": "@model-fable", "symbol": "diamond", "bootstrap": 0.0095, "single_replicate": false, "publication_scope": null, "replicates": 3}, {"id": "tb:terminal-bench-v2-common-n30-turncap100-20260906:claude-code-haiku", "label": "cc:haiku", "cohort": "terminal-bench-v2-common-n30-turncap100-20260906", "qualifier": null, "sample": "30 unique tasks over 3 repetitions \u00b7 4 Claude Code --max-turns 100 cap hits (4 unsolved)", "model": "haiku", "model_label": "Claude Haiku 4.5", "harness": "cc", "harness_label": "Claude Code", "x": 0.262522, "y": 0.411111, "x_ci": [0.162475, 0.377011], "y_ci": [0.277778, 0.555556], "frontier": false, "color": "@model-haiku", "symbol": "diamond", "bootstrap": 0.0, "single_replicate": false, "publication_scope": null, "replicates": 3}, {"id": "tb:terminal-bench-v2-common-n30-turncap100-20260906:claude-code-opus", "label": "cc:opus", "cohort": "terminal-bench-v2-common-n30-turncap100-20260906", "qualifier": null, "sample": "30 unique tasks over 3 repetitions", "model": "opus", "model_label": "Claude Opus 4.8", "harness": "cc", "harness_label": "Claude Code", "x": 0.899294, "y": 0.688889, "x_ci": [0.589994, 1.276792], "y_ci": [0.522222, 0.844444], "frontier": false, "color": "@model-opus", "symbol": "diamond", "bootstrap": 0.0005, "single_replicate": false, "publication_scope": null, "replicates": 3}, {"id": "tb:terminal-bench-v2-common-n30-turncap100-20260906:claude-code-sonnet", "label": "cc:sonnet", "cohort": "terminal-bench-v2-common-n30-turncap100-20260906", "qualifier": null, "sample": "30 unique tasks over 3 repetitions", "model": "sonnet", "model_label": "Claude Sonnet 4.6", "harness": "cc", "harness_label": "Claude Code", "x": 0.668745, "y": 0.622222, "x_ci": [0.446929, 0.917128], "y_ci": [0.466667, 0.766667], "frontier": false, "color": "@model-sonnet", "symbol": "diamond", "bootstrap": 0.0005, "single_replicate": false, "publication_scope": null, "replicates": 3}, {"id": "tb:terminal-bench-v2-common-n30-turncap100-20260906:codex-luna", "label": "codex:luna", "cohort": "terminal-bench-v2-common-n30-turncap100-20260906", "qualifier": null, "sample": "30 unique tasks over 3 repetitions", "model": "luna", "model_label": "GPT-5.6 Luna", "harness": "codex", "harness_label": "Codex", "x": 0.06397, "y": 0.722222, "x_ci": [0.038652, 0.092664], "y_ci": [0.577778, 0.855556], "frontier": false, "color": "@model-luna", "symbol": "triangle-up", "bootstrap": 0.1745, "single_replicate": false, "publication_scope": null, "replicates": 3}, {"id": "tb:terminal-bench-v2-common-n30-turncap100-20260906:codex-sol", "label": "codex:sol", "cohort": "terminal-bench-v2-common-n30-turncap100-20260906", "qualifier": null, "sample": "30 unique tasks over 3 repetitions", "model": "sol", "model_label": "GPT-5.6 Sol", "harness": "codex", "harness_label": "Codex", "x": 0.761134, "y": 0.788889, "x_ci": [0.46322, 1.130819], "y_ci": [0.655556, 0.911111], "frontier": false, "color": "@model-sol", "symbol": "triangle-up", "bootstrap": 0.0645, "single_replicate": false, "publication_scope": null, "replicates": 3}, {"id": "tb:terminal-bench-v2-common-n30-turncap100-20260906:pi-claude-fable", "label": "pi:fable", "cohort": "terminal-bench-v2-common-n30-turncap100-20260906", "qualifier": null, "sample": "30 unique tasks over 3 repetitions", "model": "fable", "model_label": "Claude Fable 5", "harness": "pi", "harness_label": "Pi", "x": 1.078506, "y": 0.711111, "x_ci": [0.571387, 1.74867], "y_ci": [0.544444, 0.855556], "frontier": false, "color": "@model-fable", "symbol": "square", "bootstrap": 0.001, "single_replicate": false, "publication_scope": null, "replicates": 3}, {"id": "tb:terminal-bench-v2-common-n30-turncap100-20260906:pi-claude-haiku", "label": "pi:haiku", "cohort": "terminal-bench-v2-common-n30-turncap100-20260906", "qualifier": null, "sample": "30 unique tasks over 3 repetitions", "model": "haiku", "model_label": "Claude Haiku 4.5", "harness": "pi", "harness_label": "Pi", "x": 0.249566, "y": 0.477778, "x_ci": [0.160236, 0.355113], "y_ci": [0.322222, 0.633333], "frontier": false, "color": "@model-haiku", "symbol": "square", "bootstrap": 0.0, "single_replicate": false, "publication_scope": null, "replicates": 3}, {"id": "tb:terminal-bench-v2-common-n30-turncap100-20260906:pi-claude-opus", "label": "pi:opus", "cohort": "terminal-bench-v2-common-n30-turncap100-20260906", "qualifier": null, "sample": "30 unique tasks over 3 repetitions", "model": "opus", "model_label": "Claude Opus 4.8", "harness": "pi", "harness_label": "Pi", "x": 0.757909, "y": 0.722222, "x_ci": [0.384428, 1.245274], "y_ci": [0.588889, 0.844444], "frontier": false, "color": "@model-opus", "symbol": "square", "bootstrap": 0.0075, "single_replicate": false, "publication_scope": null, "replicates": 3}, {"id": "tb:terminal-bench-v2-common-n30-turncap100-20260906:pi-claude-sonnet", "label": "pi:sonnet", "cohort": "terminal-bench-v2-common-n30-turncap100-20260906", "qualifier": null, "sample": "30 unique tasks over 3 repetitions", "model": "sonnet", "model_label": "Claude Sonnet 4.6", "harness": "pi", "harness_label": "Pi", "x": 0.614356, "y": 0.655556, "x_ci": [0.390826, 0.8692], "y_ci": [0.488889, 0.822222], "frontier": false, "color": "@model-sonnet", "symbol": "square", "bootstrap": 0.012, "single_replicate": false, "publication_scope": null, "replicates": 3}, {"id": "tb:terminal-bench-v2-common-n30-turncap100-20260906:pi-gpt-luna", "label": "pi:luna", "cohort": "terminal-bench-v2-common-n30-turncap100-20260906", "qualifier": null, "sample": "30 unique tasks over 3 repetitions", "model": "luna", "model_label": "GPT-5.6 Luna", "harness": "pi", "harness_label": "Pi", "x": 0.045341, "y": 0.766667, "x_ci": [0.025556, 0.06917], "y_ci": [0.622222, 0.9], "frontier": true, "color": "@model-luna", "symbol": "square", "bootstrap": 0.9995, "single_replicate": false, "publication_scope": null, "replicates": 3}, {"id": "tb:terminal-bench-v2-common-n30-turncap100-20260906:pi-gpt-sol", "label": "pi:sol", "cohort": "terminal-bench-v2-common-n30-turncap100-20260906", "qualifier": null, "sample": "30 unique tasks over 3 repetitions", "model": "sol", "model_label": "GPT-5.6 Sol", "harness": "pi", "harness_label": "Pi", "x": 0.420517, "y": 0.833333, "x_ci": [0.272168, 0.61226], "y_ci": [0.711111, 0.944444], "frontier": true, "color": "@model-sol", "symbol": "square", "bootstrap": 0.9365, "single_replicate": false, "publication_scope": null, "replicates": 3}, {"id": "tb:terminal-bench-v2-common-n30-turncap100-20260906:pi-kimi-k3", "label": "pi:kimi", "cohort": "terminal-bench-v2-common-n30-turncap100-20260906", "qualifier": null, "sample": "30 unique tasks over 3 repetitions", "model": "kimi", "model_label": "Kimi K3", "harness": "pi", "harness_label": "Pi", "x": 0.38318, "y": 0.733333, "x_ci": [0.238695, 0.543259], "y_ci": [0.6, 0.855556], "frontier": false, "color": "@model-kimi", "symbol": "square", "bootstrap": 0.1325, "single_replicate": false, "publication_scope": null, "replicates": 3}, {"id": "tb:arena-cross-r1-r3:kimi:cc:sha256:0dd54cc5bfd048c3aa8edfc5af785146944b2ffffe540beb9258721caebde609", "label": "cc:kimi", "cohort": "Arena cross-family r1-r3", "qualifier": null, "sample": "30 unique tasks over 3 repetitions", "model": "kimi", "model_label": "Kimi K3", "harness": "cc", "harness_label": "Claude Code", "x": 0.521312, "y": 0.666667, "x_ci": [0.355968, 0.707199], "y_ci": [0.522222, 0.8], "frontier": false, "color": "@model-kimi", "symbol": "diamond", "bootstrap": 0.0, "single_replicate": false, "publication_scope": "arena-cross-family-r1-r3", "replicates": 3}, {"id": "tb:arena-cross-r1-r3:luna:cc:sha256:0dd54cc5bfd048c3aa8edfc5af785146944b2ffffe540beb9258721caebde609", "label": "cc:luna", "cohort": "Arena cross-family r1-r3", "qualifier": null, "sample": "30 unique tasks over 3 repetitions", "model": "luna", "model_label": "GPT-5.6 Luna", "harness": "cc", "harness_label": "Claude Code", "x": 0.098445, "y": 0.7, "x_ci": [0.067636, 0.13316], "y_ci": [0.555556, 0.833333], "frontier": false, "color": "@model-luna", "symbol": "diamond", "bootstrap": 0.0795, "single_replicate": false, "publication_scope": "arena-cross-family-r1-r3", "replicates": 3}, {"id": "tb:arena-cross-r1-r3:sol:cc:sha256:0dd54cc5bfd048c3aa8edfc5af785146944b2ffffe540beb9258721caebde609", "label": "cc:sol", "cohort": "Arena cross-family r1-r3", "qualifier": null, "sample": "30 unique tasks over 3 repetitions", "model": "sol", "model_label": "GPT-5.6 Sol", "harness": "cc", "harness_label": "Claude Code", "x": 1.354545, "y": 0.711111, "x_ci": [0.94836, 1.852551], "y_ci": [0.555556, 0.855556], "frontier": false, "color": "@model-sol", "symbol": "diamond", "bootstrap": 0.007, "single_replicate": false, "publication_scope": "arena-cross-family-r1-r3", "replicates": 3}, {"id": "tb:arena-cross-r1-r3:fable:codex:sha256:0dd54cc5bfd048c3aa8edfc5af785146944b2ffffe540beb9258721caebde609", "label": "codex:fable", "cohort": "Arena cross-family r1-r3", "qualifier": null, "sample": "30 unique tasks over 3 repetitions", "model": "fable", "model_label": "Claude Fable 5", "harness": "codex", "harness_label": "Codex", "x": 0.976038, "y": 0.722222, "x_ci": [0.631696, 1.406231], "y_ci": [0.566667, 0.866667], "frontier": false, "color": "@model-fable", "symbol": "triangle-up", "bootstrap": 0.013, "single_replicate": false, "publication_scope": "arena-cross-family-r1-r3", "replicates": 3}, {"id": "tb:arena-cross-r1-r3:haiku:codex:sha256:0dd54cc5bfd048c3aa8edfc5af785146944b2ffffe540beb9258721caebde609", "label": "codex:haiku", "cohort": "Arena cross-family r1-r3", "qualifier": null, "sample": "30 unique tasks over 3 repetitions", "model": "haiku", "model_label": "Claude Haiku 4.5", "harness": "codex", "harness_label": "Codex", "x": 0.213731, "y": 0.311111, "x_ci": [0.142742, 0.293352], "y_ci": [0.155556, 0.477778], "frontier": false, "color": "@model-haiku", "symbol": "triangle-up", "bootstrap": 0.0, "single_replicate": false, "publication_scope": "arena-cross-family-r1-r3", "replicates": 3}, {"id": "tb:arena-cross-r1-r3:kimi:codex:sha256:0dd54cc5bfd048c3aa8edfc5af785146944b2ffffe540beb9258721caebde609", "label": "codex:kimi", "cohort": "Arena cross-family r1-r3", "qualifier": null, "sample": "30 unique tasks over 3 repetitions", "model": "kimi", "model_label": "Kimi K3", "harness": "codex", "harness_label": "Codex", "x": 0.449972, "y": 0.7, "x_ci": [0.283386, 0.656948], "y_ci": [0.555556, 0.833333], "frontier": false, "color": "@model-kimi", "symbol": "triangle-up", "bootstrap": 0.002, "single_replicate": false, "publication_scope": "arena-cross-family-r1-r3", "replicates": 3}, {"id": "tb:arena-cross-r1-r3:opus:codex:sha256:0dd54cc5bfd048c3aa8edfc5af785146944b2ffffe540beb9258721caebde609", "label": "codex:opus", "cohort": "Arena cross-family r1-r3", "qualifier": null, "sample": "30 unique tasks over 3 repetitions", "model": "opus", "model_label": "Claude Opus 4.8", "harness": "codex", "harness_label": "Codex", "x": 0.847954, "y": 0.722222, "x_ci": [0.507623, 1.269417], "y_ci": [0.566667, 0.866667], "frontier": false, "color": "@model-opus", "symbol": "triangle-up", "bootstrap": 0.0125, "single_replicate": false, "publication_scope": "arena-cross-family-r1-r3", "replicates": 3}, {"id": "tb:arena-cross-r1-r3:sonnet:codex:sha256:0dd54cc5bfd048c3aa8edfc5af785146944b2ffffe540beb9258721caebde609", "label": "codex:sonnet", "cohort": "Arena cross-family r1-r3", "qualifier": null, "sample": "30 unique tasks over 3 repetitions", "model": "sonnet", "model_label": "Claude Sonnet 4.6", "harness": "codex", "harness_label": "Codex", "x": 0.552243, "y": 0.633333, "x_ci": [0.38034, 0.741388], "y_ci": [0.466667, 0.788889], "frontier": false, "color": "@model-sonnet", "symbol": "triangle-up", "bootstrap": 0.002, "single_replicate": false, "publication_scope": "arena-cross-family-r1-r3", "replicates": 3}], "path": ["tb:terminal-bench-v2-common-n30-turncap100-20260906:pi-gpt-luna", "tb:terminal-bench-v2-common-n30-turncap100-20260906:pi-gpt-sol"], "models": [{"key": "fable", "label": "Claude Fable 5", "color": "@model-fable"}, {"key": "opus", "label": "Claude Opus 4.8", "color": "@model-opus"}, {"key": "sonnet", "label": "Claude Sonnet 4.6", "color": "@model-sonnet"}, {"key": "haiku", "label": "Claude Haiku 4.5", "color": "@model-haiku"}, {"key": "sol", "label": "GPT-5.6 Sol", "color": "@model-sol"}, {"key": "luna", "label": "GPT-5.6 Luna", "color": "@model-luna"}, {"key": "kimi", "label": "Kimi K3", "color": "@model-kimi"}], "harnesses": [{"key": "pi", "label": "Pi", "symbol": "square"}, {"key": "codex", "label": "Codex", "symbol": "triangle-up"}, {"key": "cc", "label": "Claude Code", "symbol": "diamond"}], "subsets": [{"model": null, "harness": "pi", "path": ["tb:terminal-bench-v2-common-n30-turncap100-20260906:pi-gpt-luna", "tb:terminal-bench-v2-common-n30-turncap100-20260906:pi-gpt-sol"]}, {"model": null, "harness": "codex", "path": ["tb:terminal-bench-v2-common-n30-turncap100-20260906:codex-luna", "tb:terminal-bench-v2-common-n30-turncap100-20260906:codex-sol"]}, {"model": null, "harness": "cc", "path": ["tb:arena-cross-r1-r3:luna:cc:sha256:0dd54cc5bfd048c3aa8edfc5af785146944b2ffffe540beb9258721caebde609", "tb:arena-cross-r1-r3:sol:cc:sha256:0dd54cc5bfd048c3aa8edfc5af785146944b2ffffe540beb9258721caebde609", "tb:terminal-bench-v2-common-n30-turncap100-20260906:claude-code-fable"]}, {"model": "fable", "harness": null, "path": ["tb:arena-cross-r1-r3:fable:codex:sha256:0dd54cc5bfd048c3aa8edfc5af785146944b2ffffe540beb9258721caebde609", "tb:terminal-bench-v2-common-n30-turncap100-20260906:claude-code-fable"]}, {"model": "fable", "harness": "pi", "path": ["tb:terminal-bench-v2-common-n30-turncap100-20260906:pi-claude-fable"]}, {"model": "fable", "harness": "codex", "path": ["tb:arena-cross-r1-r3:fable:codex:sha256:0dd54cc5bfd048c3aa8edfc5af785146944b2ffffe540beb9258721caebde609"]}, {"model": "fable", "harness": "cc", "path": ["tb:terminal-bench-v2-common-n30-turncap100-20260906:claude-code-fable"]}, {"model": "haiku", "harness": null, "path": ["tb:arena-cross-r1-r3:haiku:codex:sha256:0dd54cc5bfd048c3aa8edfc5af785146944b2ffffe540beb9258721caebde609", "tb:terminal-bench-v2-common-n30-turncap100-20260906:pi-claude-haiku"]}, {"model": "haiku", "harness": "pi", "path": ["tb:terminal-bench-v2-common-n30-turncap100-20260906:pi-claude-haiku"]}, {"model": "haiku", "harness": "codex", "path": ["tb:arena-cross-r1-r3:haiku:codex:sha256:0dd54cc5bfd048c3aa8edfc5af785146944b2ffffe540beb9258721caebde609"]}, {"model": "haiku", "harness": "cc", "path": ["tb:terminal-bench-v2-common-n30-turncap100-20260906:claude-code-haiku"]}, {"model": "opus", "harness": null, "path": ["tb:terminal-bench-v2-common-n30-turncap100-20260906:pi-claude-opus"]}, {"model": "opus", "harness": "pi", "path": ["tb:terminal-bench-v2-common-n30-turncap100-20260906:pi-claude-opus"]}, {"model": "opus", "harness": "codex", "path": ["tb:arena-cross-r1-r3:opus:codex:sha256:0dd54cc5bfd048c3aa8edfc5af785146944b2ffffe540beb9258721caebde609"]}, {"model": "opus", "harness": "cc", "path": ["tb:terminal-bench-v2-common-n30-turncap100-20260906:claude-code-opus"]}, {"model": "sonnet", "harness": null, "path": ["tb:arena-cross-r1-r3:sonnet:codex:sha256:0dd54cc5bfd048c3aa8edfc5af785146944b2ffffe540beb9258721caebde609", "tb:terminal-bench-v2-common-n30-turncap100-20260906:pi-claude-sonnet"]}, {"model": "sonnet", "harness": "pi", "path": ["tb:terminal-bench-v2-common-n30-turncap100-20260906:pi-claude-sonnet"]}, {"model": "sonnet", "harness": "codex", "path": ["tb:arena-cross-r1-r3:sonnet:codex:sha256:0dd54cc5bfd048c3aa8edfc5af785146944b2ffffe540beb9258721caebde609"]}, {"model": "sonnet", "harness": "cc", "path": ["tb:terminal-bench-v2-common-n30-turncap100-20260906:claude-code-sonnet"]}, {"model": "luna", "harness": null, "path": ["tb:terminal-bench-v2-common-n30-turncap100-20260906:pi-gpt-luna"]}, {"model": "luna", "harness": "pi", "path": ["tb:terminal-bench-v2-common-n30-turncap100-20260906:pi-gpt-luna"]}, {"model": "luna", "harness": "codex", "path": ["tb:terminal-bench-v2-common-n30-turncap100-20260906:codex-luna"]}, {"model": "luna", "harness": "cc", "path": ["tb:arena-cross-r1-r3:luna:cc:sha256:0dd54cc5bfd048c3aa8edfc5af785146944b2ffffe540beb9258721caebde609"]}, {"model": "sol", "harness": null, "path": ["tb:terminal-bench-v2-common-n30-turncap100-20260906:pi-gpt-sol"]}, {"model": "sol", "harness": "pi", "path": ["tb:terminal-bench-v2-common-n30-turncap100-20260906:pi-gpt-sol"]}, {"model": "sol", "harness": "codex", "path": ["tb:terminal-bench-v2-common-n30-turncap100-20260906:codex-sol"]}, {"model": "sol", "harness": "cc", "path": ["tb:arena-cross-r1-r3:sol:cc:sha256:0dd54cc5bfd048c3aa8edfc5af785146944b2ffffe540beb9258721caebde609"]}, {"model": "kimi", "harness": null, "path": ["tb:terminal-bench-v2-common-n30-turncap100-20260906:pi-kimi-k3"]}, {"model": "kimi", "harness": "pi", "path": ["tb:terminal-bench-v2-common-n30-turncap100-20260906:pi-kimi-k3"]}, {"model": "kimi", "harness": "codex", "path": ["tb:arena-cross-r1-r3:kimi:codex:sha256:0dd54cc5bfd048c3aa8edfc5af785146944b2ffffe540beb9258721caebde609"]}, {"model": "kimi", "harness": "cc", "path": ["tb:arena-cross-r1-r3:kimi:cc:sha256:0dd54cc5bfd048c3aa8edfc5af785146944b2ffffe540beb9258721caebde609"]}], "notes": "exact hybrid 21-cell profile; cross-family cells use Arena r1-r3 and reused cells retain their original validated route/epoch \u00b7 2/21 observations are nondominated on the empirical means \u00b7 large filled/labeled = frontier \u00b7 small open = dominated \u00b7 icon = harness \u00b7 Pareto membership is descriptive; intervals show uncertainty and can overlap \u00b7 cost is token usage repriced under one frozen price schedule (agentbrane-research-2026-09-01-v3) so systems billed through different routes stay comparable; it need not equal what was actually paid \u00b7 bootstrap frequency is not a posterior probability \u00b7 The cost axis is logarithmic because the included systems span more than three orders of magnitude. The Arena cross-family cells (Claude Fable 5 \u00b7 Codex, Claude Haiku 4.5 \u00b7 Codex, Claude Opus 4.8 \u00b7 Codex, Claude Sonnet 4.6 \u00b7 Codex, GPT-5.6 Luna \u00b7 Claude Code, GPT-5.6 Sol \u00b7 Claude Code, Kimi K3 \u00b7 Claude Code, Kimi K3 \u00b7 Codex) contain 3 repetitions run through the Arena serving route in a later execution epoch than the existing cells. They are drawn, labelled and hovered exactly like every other system; their success-rate and cost intervals are the same task-cluster percentile bootstrap (10,000 draws, repetitions averaged within task) every other cell carries, computed by the dashboard from the published task outcomes. The hybrid publication plan makes no test or ranking claim for them, and route or execution epoch may confound comparisons with the existing cells."}, "plotly": {"data": [{"type": "scatter", "mode": "lines", "x": [0.04534, 0.42052], "y": [0.76667, 0.83333], "name": "empirical frontier", "showlegend": false, "line": {"color": "@ink2", "width": 2, "shape": "hv"}, "hoverinfo": "skip"}, {"type": "scatter", "meta": {"model": "fable", "harness": "pi", "frontier": false, "ids": ["tb:terminal-bench-v2-common-n30-turncap100-20260906:pi-claude-fable"]}, "showlegend": false, "mode": "markers", "x": [1.07851], "y": [0.71111], "customdata": [["Claude Fable 5 \u00b7 Pi", "54.4%\u201385.6%", "$0.571\u2013$1.749", "0%", "30 unique tasks over 3 repetitions"]], "name": "fable \u00b7 Pi \u00b7 dominated", "legendgroup": "fable", "marker": {"size": [12], "color": "@model-fable", "symbol": ["square-open"], "opacity": 0.42, "line": {"color": "@model-fable", "width": 2}}, "error_y": {"type": "data", "thickness": 1.1, "width": 3, "color": "@model-fable", "array": [0.14444], "arrayminus": [0.16667]}, "error_x": {"type": "data", "thickness": 1.1, "width": 3, "color": "@model-fable", "array": [0.67016], "arrayminus": [0.50712]}, "hovertemplate": "<b>%{customdata[0]}</b><br><b>%{y:.1%}</b> resolved (95% CI %{customdata[1]})<br><b>$%{x:.3f}</b> standardized cost per rollout (95% CI %{customdata[2]})<br>bootstrap frontier %{customdata[3]}<br>%{customdata[4]}<extra></extra>"}, {"type": "scatter", "meta": {"model": "fable", "harness": "codex", "frontier": false, "ids": ["tb:arena-cross-r1-r3:fable:codex:sha256:0dd54cc5bfd048c3aa8edfc5af785146944b2ffffe540beb9258721caebde609"]}, "showlegend": false, "mode": "markers", "x": [0.97604], "y": [0.72222], "customdata": [["Claude Fable 5 \u00b7 Codex", "56.7%\u201386.7%", "$0.632\u2013$1.406", "1%", "30 unique tasks over 3 repetitions"]], "name": "fable \u00b7 Codex \u00b7 dominated", "legendgroup": "fable", "marker": {"size": [12], "color": "@model-fable", "symbol": ["triangle-up-open"], "opacity": 0.42, "line": {"color": "@model-fable", "width": 2}}, "error_y": {"type": "data", "thickness": 1.1, "width": 3, "color": "@model-fable", "array": [0.14444], "arrayminus": [0.15556]}, "error_x": {"type": "data", "thickness": 1.1, "width": 3, "color": "@model-fable", "array": [0.43019], "arrayminus": [0.34434]}, "hovertemplate": "<b>%{customdata[0]}</b><br><b>%{y:.1%}</b> resolved (95% CI %{customdata[1]})<br><b>$%{x:.3f}</b> standardized cost per rollout (95% CI %{customdata[2]})<br>bootstrap frontier %{customdata[3]}<br>%{customdata[4]}<extra></extra>"}, {"type": "scatter", "meta": {"model": "fable", "harness": "cc", "frontier": false, "ids": ["tb:terminal-bench-v2-common-n30-turncap100-20260906:claude-code-fable"]}, "showlegend": false, "mode": "markers", "x": [1.5541], "y": [0.75556], "customdata": [["Claude Fable 5 \u00b7 Claude Code", "61.1%\u201388.9%", "$0.993\u2013$2.235", "1%", "30 unique tasks over 3 repetitions"]], "name": "fable \u00b7 Claude Code \u00b7 dominated", "legendgroup": "fable", "marker": {"size": [12], "color": "@model-fable", "symbol": ["diamond-open"], "opacity": 0.42, "line": {"color": "@model-fable", "width": 2}}, "error_y": {"type": "data", "thickness": 1.1, "width": 3, "color": "@model-fable", "array": [0.13333], "arrayminus": [0.14444]}, "error_x": {"type": "data", "thickness": 1.1, "width": 3, "color": "@model-fable", "array": [0.68095], "arrayminus": [0.56146]}, "hovertemplate": "<b>%{customdata[0]}</b><br><b>%{y:.1%}</b> resolved (95% CI %{customdata[1]})<br><b>$%{x:.3f}</b> standardized cost per rollout (95% CI %{customdata[2]})<br>bootstrap frontier %{customdata[3]}<br>%{customdata[4]}<extra></extra>"}, {"type": "scatter", "meta": {"model": "haiku", "harness": "pi", "frontier": false, "ids": ["tb:terminal-bench-v2-common-n30-turncap100-20260906:pi-claude-haiku"]}, "showlegend": false, "mode": "markers", "x": [0.24957], "y": [0.47778], "customdata": [["Claude Haiku 4.5 \u00b7 Pi", "32.2%\u201363.3%", "$0.160\u2013$0.355", "0%", "30 unique tasks over 3 repetitions"]], "name": "haiku \u00b7 Pi \u00b7 dominated", "legendgroup": "haiku", "marker": {"size": [12], "color": "@model-haiku", "symbol": ["square-open"], "opacity": 0.42, "line": {"color": "@model-haiku", "width": 2}}, "error_y": {"type": "data", "thickness": 1.1, "width": 3, "color": "@model-haiku", "array": [0.15556], "arrayminus": [0.15556]}, "error_x": {"type": "data", "thickness": 1.1, "width": 3, "color": "@model-haiku", "array": [0.10555], "arrayminus": [0.08933]}, "hovertemplate": "<b>%{customdata[0]}</b><br><b>%{y:.1%}</b> resolved (95% CI %{customdata[1]})<br><b>$%{x:.3f}</b> standardized cost per rollout (95% CI %{customdata[2]})<br>bootstrap frontier %{customdata[3]}<br>%{customdata[4]}<extra></extra>"}, {"type": "scatter", "meta": {"model": "haiku", "harness": "codex", "frontier": false, "ids": ["tb:arena-cross-r1-r3:haiku:codex:sha256:0dd54cc5bfd048c3aa8edfc5af785146944b2ffffe540beb9258721caebde609"]}, "showlegend": false, "mode": "markers", "x": [0.21373], "y": [0.31111], "customdata": [["Claude Haiku 4.5 \u00b7 Codex", "15.6%\u201347.8%", "$0.143\u2013$0.293", "0%", "30 unique tasks over 3 repetitions"]], "name": "haiku \u00b7 Codex \u00b7 dominated", "legendgroup": "haiku", "marker": {"size": [12], "color": "@model-haiku", "symbol": ["triangle-up-open"], "opacity": 0.42, "line": {"color": "@model-haiku", "width": 2}}, "error_y": {"type": "data", "thickness": 1.1, "width": 3, "color": "@model-haiku", "array": [0.16667], "arrayminus": [0.15556]}, "error_x": {"type": "data", "thickness": 1.1, "width": 3, "color": "@model-haiku", "array": [0.07962], "arrayminus": [0.07099]}, "hovertemplate": "<b>%{customdata[0]}</b><br><b>%{y:.1%}</b> resolved (95% CI %{customdata[1]})<br><b>$%{x:.3f}</b> standardized cost per rollout (95% CI %{customdata[2]})<br>bootstrap frontier %{customdata[3]}<br>%{customdata[4]}<extra></extra>"}, {"type": "scatter", "meta": {"model": "haiku", "harness": "cc", "frontier": false, "ids": ["tb:terminal-bench-v2-common-n30-turncap100-20260906:claude-code-haiku"]}, "showlegend": false, "mode": "markers", "x": [0.26252], "y": [0.41111], "customdata": [["Claude Haiku 4.5 \u00b7 Claude Code", "27.8%\u201355.6%", "$0.162\u2013$0.377", "0%", "30 unique tasks over 3 repetitions \u00b7 4 Claude Code --max-turns 100 cap hits (4 unsolved)"]], "name": "haiku \u00b7 Claude Code \u00b7 dominated", "legendgroup": "haiku", "marker": {"size": [12], "color": "@model-haiku", "symbol": ["diamond-open"], "opacity": 0.42, "line": {"color": "@model-haiku", "width": 2}}, "error_y": {"type": "data", "thickness": 1.1, "width": 3, "color": "@model-haiku", "array": [0.14444], "arrayminus": [0.13333]}, "error_x": {"type": "data", "thickness": 1.1, "width": 3, "color": "@model-haiku", "array": [0.11449], "arrayminus": [0.10005]}, "hovertemplate": "<b>%{customdata[0]}</b><br><b>%{y:.1%}</b> resolved (95% CI %{customdata[1]})<br><b>$%{x:.3f}</b> standardized cost per rollout (95% CI %{customdata[2]})<br>bootstrap frontier %{customdata[3]}<br>%{customdata[4]}<extra></extra>"}, {"type": "scatter", "meta": {"model": "opus", "harness": "pi", "frontier": false, "ids": ["tb:terminal-bench-v2-common-n30-turncap100-20260906:pi-claude-opus"]}, "showlegend": false, "mode": "markers", "x": [0.75791], "y": [0.72222], "customdata": [["Claude Opus 4.8 \u00b7 Pi", "58.9%\u201384.4%", "$0.384\u2013$1.245", "1%", "30 unique tasks over 3 repetitions"]], "name": "opus \u00b7 Pi \u00b7 dominated", "legendgroup": "opus", "marker": {"size": [12], "color": "@model-opus", "symbol": ["square-open"], "opacity": 0.42, "line": {"color": "@model-opus", "width": 2}}, "error_y": {"type": "data", "thickness": 1.1, "width": 3, "color": "@model-opus", "array": [0.12222], "arrayminus": [0.13333]}, "error_x": {"type": "data", "thickness": 1.1, "width": 3, "color": "@model-opus", "array": [0.48737], "arrayminus": [0.37348]}, "hovertemplate": "<b>%{customdata[0]}</b><br><b>%{y:.1%}</b> resolved (95% CI %{customdata[1]})<br><b>$%{x:.3f}</b> standardized cost per rollout (95% CI %{customdata[2]})<br>bootstrap frontier %{customdata[3]}<br>%{customdata[4]}<extra></extra>"}, {"type": "scatter", "meta": {"model": "opus", "harness": "codex", "frontier": false, "ids": ["tb:arena-cross-r1-r3:opus:codex:sha256:0dd54cc5bfd048c3aa8edfc5af785146944b2ffffe540beb9258721caebde609"]}, "showlegend": false, "mode": "markers", "x": [0.84795], "y": [0.72222], "customdata": [["Claude Opus 4.8 \u00b7 Codex", "56.7%\u201386.7%", "$0.508\u2013$1.269", "1%", "30 unique tasks over 3 repetitions"]], "name": "opus \u00b7 Codex \u00b7 dominated", "legendgroup": "opus", "marker": {"size": [12], "color": "@model-opus", "symbol": ["triangle-up-open"], "opacity": 0.42, "line": {"color": "@model-opus", "width": 2}}, "error_y": {"type": "data", "thickness": 1.1, "width": 3, "color": "@model-opus", "array": [0.14444], "arrayminus": [0.15556]}, "error_x": {"type": "data", "thickness": 1.1, "width": 3, "color": "@model-opus", "array": [0.42146], "arrayminus": [0.34033]}, "hovertemplate": "<b>%{customdata[0]}</b><br><b>%{y:.1%}</b> resolved (95% CI %{customdata[1]})<br><b>$%{x:.3f}</b> standardized cost per rollout (95% CI %{customdata[2]})<br>bootstrap frontier %{customdata[3]}<br>%{customdata[4]}<extra></extra>"}, {"type": "scatter", "meta": {"model": "opus", "harness": "cc", "frontier": false, "ids": ["tb:terminal-bench-v2-common-n30-turncap100-20260906:claude-code-opus"]}, "showlegend": false, "mode": "markers", "x": [0.89929], "y": [0.68889], "customdata": [["Claude Opus 4.8 \u00b7 Claude Code", "52.2%\u201384.4%", "$0.590\u2013$1.277", "0%", "30 unique tasks over 3 repetitions"]], "name": "opus \u00b7 Claude Code \u00b7 dominated", "legendgroup": "opus", "marker": {"size": [12], "color": "@model-opus", "symbol": ["diamond-open"], "opacity": 0.42, "line": {"color": "@model-opus", "width": 2}}, "error_y": {"type": "data", "thickness": 1.1, "width": 3, "color": "@model-opus", "array": [0.15556], "arrayminus": [0.16667]}, "error_x": {"type": "data", "thickness": 1.1, "width": 3, "color": "@model-opus", "array": [0.3775], "arrayminus": [0.3093]}, "hovertemplate": "<b>%{customdata[0]}</b><br><b>%{y:.1%}</b> resolved (95% CI %{customdata[1]})<br><b>$%{x:.3f}</b> standardized cost per rollout (95% CI %{customdata[2]})<br>bootstrap frontier %{customdata[3]}<br>%{customdata[4]}<extra></extra>"}, {"type": "scatter", "meta": {"model": "sonnet", "harness": "pi", "frontier": false, "ids": ["tb:terminal-bench-v2-common-n30-turncap100-20260906:pi-claude-sonnet"]}, "showlegend": false, "mode": "markers", "x": [0.61436], "y": [0.65556], "customdata": [["Claude Sonnet 4.6 \u00b7 Pi", "48.9%\u201382.2%", "$0.391\u2013$0.869", "1%", "30 unique tasks over 3 repetitions"]], "name": "sonnet \u00b7 Pi \u00b7 dominated", "legendgroup": "sonnet", "marker": {"size": [12], "color": "@model-sonnet", "symbol": ["square-open"], "opacity": 0.42, "line": {"color": "@model-sonnet", "width": 2}}, "error_y": {"type": "data", "thickness": 1.1, "width": 3, "color": "@model-sonnet", "array": [0.16667], "arrayminus": [0.16667]}, "error_x": {"type": "data", "thickness": 1.1, "width": 3, "color": "@model-sonnet", "array": [0.25484], "arrayminus": [0.22353]}, "hovertemplate": "<b>%{customdata[0]}</b><br><b>%{y:.1%}</b> resolved (95% CI %{customdata[1]})<br><b>$%{x:.3f}</b> standardized cost per rollout (95% CI %{customdata[2]})<br>bootstrap frontier %{customdata[3]}<br>%{customdata[4]}<extra></extra>"}, {"type": "scatter", "meta": {"model": "sonnet", "harness": "codex", "frontier": false, "ids": ["tb:arena-cross-r1-r3:sonnet:codex:sha256:0dd54cc5bfd048c3aa8edfc5af785146944b2ffffe540beb9258721caebde609"]}, "showlegend": false, "mode": "markers", "x": [0.55224], "y": [0.63333], "customdata": [["Claude Sonnet 4.6 \u00b7 Codex", "46.7%\u201378.9%", "$0.380\u2013$0.741", "0%", "30 unique tasks over 3 repetitions"]], "name": "sonnet \u00b7 Codex \u00b7 dominated", "legendgroup": "sonnet", "marker": {"size": [12], "color": "@model-sonnet", "symbol": ["triangle-up-open"], "opacity": 0.42, "line": {"color": "@model-sonnet", "width": 2}}, "error_y": {"type": "data", "thickness": 1.1, "width": 3, "color": "@model-sonnet", "array": [0.15556], "arrayminus": [0.16667]}, "error_x": {"type": "data", "thickness": 1.1, "width": 3, "color": "@model-sonnet", "array": [0.18915], "arrayminus": [0.1719]}, "hovertemplate": "<b>%{customdata[0]}</b><br><b>%{y:.1%}</b> resolved (95% CI %{customdata[1]})<br><b>$%{x:.3f}</b> standardized cost per rollout (95% CI %{customdata[2]})<br>bootstrap frontier %{customdata[3]}<br>%{customdata[4]}<extra></extra>"}, {"type": "scatter", "meta": {"model": "sonnet", "harness": "cc", "frontier": false, "ids": ["tb:terminal-bench-v2-common-n30-turncap100-20260906:claude-code-sonnet"]}, "showlegend": false, "mode": "markers", "x": [0.66874], "y": [0.62222], "customdata": [["Claude Sonnet 4.6 \u00b7 Claude Code", "46.7%\u201376.7%", "$0.447\u2013$0.917", "0%", "30 unique tasks over 3 repetitions"]], "name": "sonnet \u00b7 Claude Code \u00b7 dominated", "legendgroup": "sonnet", "marker": {"size": [12], "color": "@model-sonnet", "symbol": ["diamond-open"], "opacity": 0.42, "line": {"color": "@model-sonnet", "width": 2}}, "error_y": {"type": "data", "thickness": 1.1, "width": 3, "color": "@model-sonnet", "array": [0.14444], "arrayminus": [0.15556]}, "error_x": {"type": "data", "thickness": 1.1, "width": 3, "color": "@model-sonnet", "array": [0.24838], "arrayminus": [0.22182]}, "hovertemplate": "<b>%{customdata[0]}</b><br><b>%{y:.1%}</b> resolved (95% CI %{customdata[1]})<br><b>$%{x:.3f}</b> standardized cost per rollout (95% CI %{customdata[2]})<br>bootstrap frontier %{customdata[3]}<br>%{customdata[4]}<extra></extra>"}, {"type": "scatter", "meta": {"model": "luna", "harness": "pi", "frontier": true, "ids": ["tb:terminal-bench-v2-common-n30-turncap100-20260906:pi-gpt-luna"]}, "showlegend": false, "mode": "markers", "x": [0.04534], "y": [0.76667], "customdata": [["GPT-5.6 Luna \u00b7 Pi", "62.2%\u201390.0%", "$0.026\u2013$0.069", "100%", "30 unique tasks over 3 repetitions"]], "name": "luna \u00b7 Pi \u00b7 frontier", "legendgroup": "luna", "marker": {"size": [18], "color": "@model-luna", "symbol": ["square"], "opacity": 1, "line": {"color": "@surface", "width": 2}}, "error_y": {"type": "data", "thickness": 1.1, "width": 3, "color": "@model-luna", "array": [0.13333], "arrayminus": [0.14444]}, "error_x": {"type": "data", "thickness": 1.1, "width": 3, "color": "@model-luna", "array": [0.02383], "arrayminus": [0.01978]}, "hovertemplate": "<b>%{customdata[0]}</b><br><b>%{y:.1%}</b> resolved (95% CI %{customdata[1]})<br><b>$%{x:.3f}</b> standardized cost per rollout (95% CI %{customdata[2]})<br>bootstrap frontier %{customdata[3]}<br>%{customdata[4]}<extra></extra>"}, {"type": "scatter", "meta": {"model": "luna", "harness": "codex", "frontier": false, "ids": ["tb:terminal-bench-v2-common-n30-turncap100-20260906:codex-luna"]}, "showlegend": false, "mode": "markers", "x": [0.06397], "y": [0.72222], "customdata": [["GPT-5.6 Luna \u00b7 Codex", "57.8%\u201385.6%", "$0.039\u2013$0.093", "17%", "30 unique tasks over 3 repetitions"]], "name": "luna \u00b7 Codex \u00b7 dominated", "legendgroup": "luna", "marker": {"size": [12], "color": "@model-luna", "symbol": ["triangle-up-open"], "opacity": 0.42, "line": {"color": "@model-luna", "width": 2}}, "error_y": {"type": "data", "thickness": 1.1, "width": 3, "color": "@model-luna", "array": [0.13333], "arrayminus": [0.14444]}, "error_x": {"type": "data", "thickness": 1.1, "width": 3, "color": "@model-luna", "array": [0.02869], "arrayminus": [0.02532]}, "hovertemplate": "<b>%{customdata[0]}</b><br><b>%{y:.1%}</b> resolved (95% CI %{customdata[1]})<br><b>$%{x:.3f}</b> standardized cost per rollout (95% CI %{customdata[2]})<br>bootstrap frontier %{customdata[3]}<br>%{customdata[4]}<extra></extra>"}, {"type": "scatter", "meta": {"model": "luna", "harness": "cc", "frontier": false, "ids": ["tb:arena-cross-r1-r3:luna:cc:sha256:0dd54cc5bfd048c3aa8edfc5af785146944b2ffffe540beb9258721caebde609"]}, "showlegend": false, "mode": "markers", "x": [0.09845], "y": [0.7], "customdata": [["GPT-5.6 Luna \u00b7 Claude Code", "55.6%\u201383.3%", "$0.068\u2013$0.133", "8%", "30 unique tasks over 3 repetitions"]], "name": "luna \u00b7 Claude Code \u00b7 dominated", "legendgroup": "luna", "marker": {"size": [12], "color": "@model-luna", "symbol": ["diamond-open"], "opacity": 0.42, "line": {"color": "@model-luna", "width": 2}}, "error_y": {"type": "data", "thickness": 1.1, "width": 3, "color": "@model-luna", "array": [0.13333], "arrayminus": [0.14444]}, "error_x": {"type": "data", "thickness": 1.1, "width": 3, "color": "@model-luna", "array": [0.03472], "arrayminus": [0.03081]}, "hovertemplate": "<b>%{customdata[0]}</b><br><b>%{y:.1%}</b> resolved (95% CI %{customdata[1]})<br><b>$%{x:.3f}</b> standardized cost per rollout (95% CI %{customdata[2]})<br>bootstrap frontier %{customdata[3]}<br>%{customdata[4]}<extra></extra>"}, {"type": "scatter", "meta": {"model": "sol", "harness": "pi", "frontier": true, "ids": ["tb:terminal-bench-v2-common-n30-turncap100-20260906:pi-gpt-sol"]}, "showlegend": false, "mode": "markers", "x": [0.42052], "y": [0.83333], "customdata": [["GPT-5.6 Sol \u00b7 Pi", "71.1%\u201394.4%", "$0.272\u2013$0.612", "94%", "30 unique tasks over 3 repetitions"]], "name": "sol \u00b7 Pi \u00b7 frontier", "legendgroup": "sol", "marker": {"size": [18], "color": "@model-sol", "symbol": ["square"], "opacity": 1, "line": {"color": "@surface", "width": 2}}, "error_y": {"type": "data", "thickness": 1.1, "width": 3, "color": "@model-sol", "array": [0.11111], "arrayminus": [0.12222]}, "error_x": {"type": "data", "thickness": 1.1, "width": 3, "color": "@model-sol", "array": [0.19174], "arrayminus": [0.14835]}, "hovertemplate": "<b>%{customdata[0]}</b><br><b>%{y:.1%}</b> resolved (95% CI %{customdata[1]})<br><b>$%{x:.3f}</b> standardized cost per rollout (95% CI %{customdata[2]})<br>bootstrap frontier %{customdata[3]}<br>%{customdata[4]}<extra></extra>"}, {"type": "scatter", "meta": {"model": "sol", "harness": "codex", "frontier": false, "ids": ["tb:terminal-bench-v2-common-n30-turncap100-20260906:codex-sol"]}, "showlegend": false, "mode": "markers", "x": [0.76113], "y": [0.78889], "customdata": [["GPT-5.6 Sol \u00b7 Codex", "65.6%\u201391.1%", "$0.463\u2013$1.131", "6%", "30 unique tasks over 3 repetitions"]], "name": "sol \u00b7 Codex \u00b7 dominated", "legendgroup": "sol", "marker": {"size": [12], "color": "@model-sol", "symbol": ["triangle-up-open"], "opacity": 0.42, "line": {"color": "@model-sol", "width": 2}}, "error_y": {"type": "data", "thickness": 1.1, "width": 3, "color": "@model-sol", "array": [0.12222], "arrayminus": [0.13333]}, "error_x": {"type": "data", "thickness": 1.1, "width": 3, "color": "@model-sol", "array": [0.36968], "arrayminus": [0.29791]}, "hovertemplate": "<b>%{customdata[0]}</b><br><b>%{y:.1%}</b> resolved (95% CI %{customdata[1]})<br><b>$%{x:.3f}</b> standardized cost per rollout (95% CI %{customdata[2]})<br>bootstrap frontier %{customdata[3]}<br>%{customdata[4]}<extra></extra>"}, {"type": "scatter", "meta": {"model": "sol", "harness": "cc", "frontier": false, "ids": ["tb:arena-cross-r1-r3:sol:cc:sha256:0dd54cc5bfd048c3aa8edfc5af785146944b2ffffe540beb9258721caebde609"]}, "showlegend": false, "mode": "markers", "x": [1.35454], "y": [0.71111], "customdata": [["GPT-5.6 Sol \u00b7 Claude Code", "55.6%\u201385.6%", "$0.948\u2013$1.853", "1%", "30 unique tasks over 3 repetitions"]], "name": "sol \u00b7 Claude Code \u00b7 dominated", "legendgroup": "sol", "marker": {"size": [12], "color": "@model-sol", "symbol": ["diamond-open"], "opacity": 0.42, "line": {"color": "@model-sol", "width": 2}}, "error_y": {"type": "data", "thickness": 1.1, "width": 3, "color": "@model-sol", "array": [0.14444], "arrayminus": [0.15556]}, "error_x": {"type": "data", "thickness": 1.1, "width": 3, "color": "@model-sol", "array": [0.49801], "arrayminus": [0.40618]}, "hovertemplate": "<b>%{customdata[0]}</b><br><b>%{y:.1%}</b> resolved (95% CI %{customdata[1]})<br><b>$%{x:.3f}</b> standardized cost per rollout (95% CI %{customdata[2]})<br>bootstrap frontier %{customdata[3]}<br>%{customdata[4]}<extra></extra>"}, {"type": "scatter", "meta": {"model": "kimi", "harness": "pi", "frontier": false, "ids": ["tb:terminal-bench-v2-common-n30-turncap100-20260906:pi-kimi-k3"]}, "showlegend": false, "mode": "markers", "x": [0.38318], "y": [0.73333], "customdata": [["Kimi K3 \u00b7 Pi", "60.0%\u201385.6%", "$0.239\u2013$0.543", "13%", "30 unique tasks over 3 repetitions"]], "name": "kimi \u00b7 Pi \u00b7 dominated", "legendgroup": "kimi", "marker": {"size": [12], "color": "@model-kimi", "symbol": ["square-open"], "opacity": 0.42, "line": {"color": "@model-kimi", "width": 2}}, "error_y": {"type": "data", "thickness": 1.1, "width": 3, "color": "@model-kimi", "array": [0.12222], "arrayminus": [0.13333]}, "error_x": {"type": "data", "thickness": 1.1, "width": 3, "color": "@model-kimi", "array": [0.16008], "arrayminus": [0.14448]}, "hovertemplate": "<b>%{customdata[0]}</b><br><b>%{y:.1%}</b> resolved (95% CI %{customdata[1]})<br><b>$%{x:.3f}</b> standardized cost per rollout (95% CI %{customdata[2]})<br>bootstrap frontier %{customdata[3]}<br>%{customdata[4]}<extra></extra>"}, {"type": "scatter", "meta": {"model": "kimi", "harness": "codex", "frontier": false, "ids": ["tb:arena-cross-r1-r3:kimi:codex:sha256:0dd54cc5bfd048c3aa8edfc5af785146944b2ffffe540beb9258721caebde609"]}, "showlegend": false, "mode": "markers", "x": [0.44997], "y": [0.7], "customdata": [["Kimi K3 \u00b7 Codex", "55.6%\u201383.3%", "$0.283\u2013$0.657", "0%", "30 unique tasks over 3 repetitions"]], "name": "kimi \u00b7 Codex \u00b7 dominated", "legendgroup": "kimi", "marker": {"size": [12], "color": "@model-kimi", "symbol": ["triangle-up-open"], "opacity": 0.42, "line": {"color": "@model-kimi", "width": 2}}, "error_y": {"type": "data", "thickness": 1.1, "width": 3, "color": "@model-kimi", "array": [0.13333], "arrayminus": [0.14444]}, "error_x": {"type": "data", "thickness": 1.1, "width": 3, "color": "@model-kimi", "array": [0.20698], "arrayminus": [0.16659]}, "hovertemplate": "<b>%{customdata[0]}</b><br><b>%{y:.1%}</b> resolved (95% CI %{customdata[1]})<br><b>$%{x:.3f}</b> standardized cost per rollout (95% CI %{customdata[2]})<br>bootstrap frontier %{customdata[3]}<br>%{customdata[4]}<extra></extra>"}, {"type": "scatter", "meta": {"model": "kimi", "harness": "cc", "frontier": false, "ids": ["tb:arena-cross-r1-r3:kimi:cc:sha256:0dd54cc5bfd048c3aa8edfc5af785146944b2ffffe540beb9258721caebde609"]}, "showlegend": false, "mode": "markers", "x": [0.52131], "y": [0.66667], "customdata": [["Kimi K3 \u00b7 Claude Code", "52.2%\u201380.0%", "$0.356\u2013$0.707", "0%", "30 unique tasks over 3 repetitions"]], "name": "kimi \u00b7 Claude Code \u00b7 dominated", "legendgroup": "kimi", "marker": {"size": [12], "color": "@model-kimi", "symbol": ["diamond-open"], "opacity": 0.42, "line": {"color": "@model-kimi", "width": 2}}, "error_y": {"type": "data", "thickness": 1.1, "width": 3, "color": "@model-kimi", "array": [0.13333], "arrayminus": [0.14444]}, "error_x": {"type": "data", "thickness": 1.1, "width": 3, "color": "@model-kimi", "array": [0.18589], "arrayminus": [0.16534]}, "hovertemplate": "<b>%{customdata[0]}</b><br><b>%{y:.1%}</b> resolved (95% CI %{customdata[1]})<br><b>$%{x:.3f}</b> standardized cost per rollout (95% CI %{customdata[2]})<br>bootstrap frontier %{customdata[3]}<br>%{customdata[4]}<extra></extra>"}], "layout": {"height": 380, "margin": {"l": 54, "r": 16, "t": 26, "b": 44}, "paper_bgcolor": "rgba(0,0,0,0)", "plot_bgcolor": "rgba(0,0,0,0)", "font": {"family": "system-ui, -apple-system, \"Segoe UI\", sans-serif", "size": 12, "color": "@ink2"}, "hoverlabel": {"bgcolor": "@surface", "bordercolor": "@axis", "font": {"color": "@ink", "size": 12}}, "showlegend": false, "legend": {"orientation": "h", "yanchor": "bottom", "y": 1.02, "x": 0, "font": {"color": "@ink2", "size": 11}}, "xaxis": {"gridcolor": "@grid", "zerolinecolor": "@axis", "linecolor": "@axis", "ticks": "", "tickfont": {"color": "@muted", "size": 11}, "automargin": true, "title": {"text": "cost per rollout (USD, log scale)", "font": {"color": "@muted", "size": 12}}, "type": "log", "tickprefix": "$"}, "yaxis": {"gridcolor": "@grid", "zerolinecolor": "@axis", "linecolor": "@axis", "ticks": "", "tickfont": {"color": "@muted", "size": 11}, "automargin": true, "title": {"text": "success rate", "font": {"color": "@muted", "size": 12}}, "tickformat": ".0%", "rangemode": "tozero"}}}, "table": {"columns": ["system", "resolved", "resolved 95% CI", "standardized cost/rollout", "standardized cost 95% CI", "standardized cost/solve", "frontier bootstrap", "caveats", "Pareto", "dominated by"], "rows": [["GPT-5.6 Luna \u00b7 Pi", "76.7%", "62.2%\u201390.0%", "$0.045", "$0.026\u2013$0.069", "$0.059", "100%", "\u2014", {"kind": "status", "value": "frontier", "tone": "good"}, "\u2014"], ["GPT-5.6 Sol \u00b7 Pi", "83.3%", "71.1%\u201394.4%", "$0.421", "$0.272\u2013$0.612", "$0.505", "94%", "\u2014", {"kind": "status", "value": "frontier", "tone": "good"}, "\u2014"], ["GPT-5.6 Luna \u00b7 Codex", "72.2%", "57.8%\u201385.6%", "$0.064", "$0.039\u2013$0.093", "$0.089", "17%", "\u2014", {"kind": "status", "value": "dominated", "tone": "muted"}, "GPT-5.6 Luna \u00b7 Pi"], ["GPT-5.6 Luna \u00b7 Claude Code", "70.0%", "55.6%\u201383.3%", "$0.098", "$0.068\u2013$0.133", "$0.141", "8%", "\u2014", {"kind": "status", "value": "dominated", "tone": "muted"}, "GPT-5.6 Luna \u00b7 Codex, GPT-5.6 Luna \u00b7 Pi"], ["Claude Haiku 4.5 \u00b7 Codex", "31.1%", "15.6%\u201347.8%", "$0.214", "$0.143\u2013$0.293", "$0.687", "0%", "\u2014", {"kind": "status", "value": "dominated", "tone": "muted"}, "GPT-5.6 Luna \u00b7 Claude Code, GPT-5.6 Luna \u00b7 Codex, GPT-5.6 Luna \u00b7 Pi"], ["Claude Haiku 4.5 \u00b7 Pi", "47.8%", "32.2%\u201363.3%", "$0.250", "$0.160\u2013$0.355", "$0.522", "0%", "\u2014", {"kind": "status", "value": "dominated", "tone": "muted"}, "GPT-5.6 Luna \u00b7 Claude Code, GPT-5.6 Luna \u00b7 Codex, GPT-5.6 Luna \u00b7 Pi"], ["Claude Haiku 4.5 \u00b7 Claude Code", "41.1%", "27.8%\u201355.6%", "$0.263", "$0.162\u2013$0.377", "$0.639", "0%", "4 Claude Code --max-turns 100 cap hits (4 unsolved)", {"kind": "status", "value": "dominated", "tone": "muted"}, "GPT-5.6 Luna \u00b7 Claude Code, GPT-5.6 Luna \u00b7 Codex, Claude Haiku 4.5 \u00b7 Pi, GPT-5.6 Luna \u00b7 Pi"], ["Kimi K3 \u00b7 Pi", "73.3%", "60.0%\u201385.6%", "$0.383", "$0.239\u2013$0.543", "$0.523", "13%", "\u2014", {"kind": "status", "value": "dominated", "tone": "muted"}, "GPT-5.6 Luna \u00b7 Pi"], ["Kimi K3 \u00b7 Codex", "70.0%", "55.6%\u201383.3%", "$0.450", "$0.283\u2013$0.657", "$0.643", "0%", "\u2014", {"kind": "status", "value": "dominated", "tone": "muted"}, "GPT-5.6 Luna \u00b7 Claude Code, GPT-5.6 Luna \u00b7 Codex, GPT-5.6 Luna \u00b7 Pi, GPT-5.6 Sol \u00b7 Pi, Kimi K3 \u00b7 Pi"], ["Kimi K3 \u00b7 Claude Code", "66.7%", "52.2%\u201380.0%", "$0.521", "$0.356\u2013$0.707", "$0.782", "0%", "\u2014", {"kind": "status", "value": "dominated", "tone": "muted"}, "Kimi K3 \u00b7 Codex, GPT-5.6 Luna \u00b7 Claude Code, GPT-5.6 Luna \u00b7 Codex, GPT-5.6 Luna \u00b7 Pi, GPT-5.6 Sol \u00b7 Pi, Kimi K3 \u00b7 Pi"], ["Claude Sonnet 4.6 \u00b7 Codex", "63.3%", "46.7%\u201378.9%", "$0.552", "$0.380\u2013$0.741", "$0.872", "0%", "\u2014", {"kind": "status", "value": "dominated", "tone": "muted"}, "Kimi K3 \u00b7 Claude Code, Kimi K3 \u00b7 Codex, GPT-5.6 Luna \u00b7 Claude Code, GPT-5.6 Luna \u00b7 Codex, GPT-5.6 Luna \u00b7 Pi, GPT-5.6 Sol \u00b7 Pi, Kimi K3 \u00b7 Pi"], ["Claude Sonnet 4.6 \u00b7 Pi", "65.6%", "48.9%\u201382.2%", "$0.614", "$0.391\u2013$0.869", "$0.937", "1%", "\u2014", {"kind": "status", "value": "dominated", "tone": "muted"}, "Kimi K3 \u00b7 Claude Code, Kimi K3 \u00b7 Codex, GPT-5.6 Luna \u00b7 Claude Code, GPT-5.6 Luna \u00b7 Codex, GPT-5.6 Luna \u00b7 Pi, GPT-5.6 Sol \u00b7 Pi, Kimi K3 \u00b7 Pi"], ["Claude Sonnet 4.6 \u00b7 Claude Code", "62.2%", "46.7%\u201376.7%", "$0.669", "$0.447\u2013$0.917", "$1.075", "0%", "\u2014", {"kind": "status", "value": "dominated", "tone": "muted"}, "Kimi K3 \u00b7 Claude Code, Kimi K3 \u00b7 Codex, GPT-5.6 Luna \u00b7 Claude Code, Claude Sonnet 4.6 \u00b7 Codex, GPT-5.6 Luna \u00b7 Codex, Claude Sonnet 4.6 \u00b7 Pi, GPT-5.6 Luna \u00b7 Pi, GPT-5.6 Sol \u00b7 Pi, Kimi K3 \u00b7 Pi"], ["Claude Opus 4.8 \u00b7 Pi", "72.2%", "58.9%\u201384.4%", "$0.758", "$0.384\u2013$1.245", "$1.049", "1%", "\u2014", {"kind": "status", "value": "dominated", "tone": "muted"}, "GPT-5.6 Luna \u00b7 Codex, GPT-5.6 Luna \u00b7 Pi, GPT-5.6 Sol \u00b7 Pi, Kimi K3 \u00b7 Pi"], ["GPT-5.6 Sol \u00b7 Codex", "78.9%", "65.6%\u201391.1%", "$0.761", "$0.463\u2013$1.131", "$0.965", "6%", "\u2014", {"kind": "status", "value": "dominated", "tone": "muted"}, "GPT-5.6 Sol \u00b7 Pi"], ["Claude Opus 4.8 \u00b7 Codex", "72.2%", "56.7%\u201386.7%", "$0.848", "$0.508\u2013$1.269", "$1.174", "1%", "\u2014", {"kind": "status", "value": "dominated", "tone": "muted"}, "GPT-5.6 Luna \u00b7 Codex, GPT-5.6 Sol \u00b7 Codex, Claude Opus 4.8 \u00b7 Pi, GPT-5.6 Luna \u00b7 Pi, GPT-5.6 Sol \u00b7 Pi, Kimi K3 \u00b7 Pi"], ["Claude Opus 4.8 \u00b7 Claude Code", "68.9%", "52.2%\u201384.4%", "$0.899", "$0.590\u2013$1.277", "$1.305", "0%", "\u2014", {"kind": "status", "value": "dominated", "tone": "muted"}, "Kimi K3 \u00b7 Codex, GPT-5.6 Luna \u00b7 Claude Code, Claude Opus 4.8 \u00b7 Codex, GPT-5.6 Luna \u00b7 Codex, GPT-5.6 Sol \u00b7 Codex, Claude Opus 4.8 \u00b7 Pi, GPT-5.6 Luna \u00b7 Pi, GPT-5.6 Sol \u00b7 Pi, Kimi K3 \u00b7 Pi"], ["Claude Fable 5 \u00b7 Codex", "72.2%", "56.7%\u201386.7%", "$0.976", "$0.632\u2013$1.406", "$1.351", "1%", "\u2014", {"kind": "status", "value": "dominated", "tone": "muted"}, "Claude Opus 4.8 \u00b7 Codex, GPT-5.6 Luna \u00b7 Codex, GPT-5.6 Sol \u00b7 Codex, Claude Opus 4.8 \u00b7 Pi, GPT-5.6 Luna \u00b7 Pi, GPT-5.6 Sol \u00b7 Pi, Kimi K3 \u00b7 Pi"], ["Claude Fable 5 \u00b7 Pi", "71.1%", "54.4%\u201385.6%", "$1.079", "$0.571\u2013$1.749", "$1.517", "0%", "\u2014", {"kind": "status", "value": "dominated", "tone": "muted"}, "Claude Fable 5 \u00b7 Codex, Claude Opus 4.8 \u00b7 Codex, GPT-5.6 Luna \u00b7 Codex, GPT-5.6 Sol \u00b7 Codex, Claude Opus 4.8 \u00b7 Pi, GPT-5.6 Luna \u00b7 Pi, GPT-5.6 Sol \u00b7 Pi, Kimi K3 \u00b7 Pi"], ["GPT-5.6 Sol \u00b7 Claude Code", "71.1%", "55.6%\u201385.6%", "$1.355", "$0.948\u2013$1.853", "$1.905", "1%", "\u2014", {"kind": "status", "value": "dominated", "tone": "muted"}, "Claude Fable 5 \u00b7 Codex, Claude Opus 4.8 \u00b7 Codex, GPT-5.6 Luna \u00b7 Codex, GPT-5.6 Sol \u00b7 Codex, Claude Opus 4.8 \u00b7 Pi, GPT-5.6 Luna \u00b7 Pi, GPT-5.6 Sol \u00b7 Pi, Kimi K3 \u00b7 Pi"], ["Claude Fable 5 \u00b7 Claude Code", "75.6%", "61.1%\u201388.9%", "$1.554", "$0.993\u2013$2.235", "$2.057", "1%", "\u2014", {"kind": "status", "value": "dominated", "tone": "muted"}, "GPT-5.6 Sol \u00b7 Codex, GPT-5.6 Luna \u00b7 Pi, GPT-5.6 Sol \u00b7 Pi"]], "note": "every row: 30 unique tasks over 3 repetitions"}}