{"title": "SWE-bench Lite \u2014 Impact of harnesses on the same model", "subtitle": "", "harness": {"label": "SWE-bench Lite", "baseline": {"key": "pi", "label": "Pi"}, "second": [{"key": "cc", "label": "Claude Code", "symbol": "diamond"}, {"key": "codex", "label": "Codex", "symbol": "triangle-up"}], "models": [{"key": "fable", "label": "Claude Fable 5", "color": "@model-fable", "paired": true}, {"key": "opus", "label": "Claude Opus 4.8", "color": "@model-opus", "paired": true}, {"key": "sonnet", "label": "Claude Sonnet 4.6", "color": "@model-sonnet", "paired": true}, {"key": "haiku", "label": "Claude Haiku 4.5", "color": "@model-haiku", "paired": true}, {"key": "sol", "label": "GPT-5.6 Sol", "color": "@model-sol", "paired": true}, {"key": "luna", "label": "GPT-5.6 Luna", "color": "@model-luna", "paired": true}, {"key": "kimi", "label": "Kimi K3", "color": "@model-kimi", "paired": true}], "rows": [{"kind": "pair", "id": "swe:all-arena-v14:fable:cc:sha256:f48cd74a7b628be67703bf1a6ad11bf3e59af79e91d6f92f69dd547f6246c7c7 vs swe:all-arena-v14:fable:pi:sha256:f48cd74a7b628be67703bf1a6ad11bf3e59af79e91d6f92f69dd547f6246c7c7", "model": "fable", "model_label": "Claude Fable 5", "color": "@model-fable", "harness": "cc", "harness_label": "Claude Code", "symbol": "diamond", "baseline": "pi", "baseline_label": "Pi", "a_label": "Claude Fable 5 \u00b7 Claude Code", "b_label": "Claude Fable 5 \u00b7 Pi", "cohort": "All-Arena reprofile r1-r3", "matched": true, "reasons": [], "descriptive": false, "evidence": "cost survives Holm", "caveats": [], "tasks": 30, "rate": {"mean": 0.011111, "ci": [-0.033333, 0.066667], "p": 0.764, "p_holm": 1.0, "clear": false, "covers_zero": true, "wins_a": 1, "wins_b": 1, "same": 28, "mass_a": 0.022222, "mass_b": 0.011111, "breakdown": [{"task": "astropy__astropy-7746", "a": [3, 3], "b": [1, 3], "delta": 0.666667}, {"task": "django__django-11019", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "django__django-11564", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "django__django-11620", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "django__django-11630", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "django__django-11815", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "django__django-12308", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "django__django-12497", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "django__django-12747", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "django__django-14608", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "django__django-15400", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "django__django-15819", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "matplotlib__matplotlib-24334", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "matplotlib__matplotlib-25433", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "psf__requests-1963", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "pylint-dev__pylint-5859", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "pytest-dev__pytest-5103", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "pytest-dev__pytest-5495", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "scikit-learn__scikit-learn-13241", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "scikit-learn__scikit-learn-14983", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "sphinx-doc__sphinx-10325", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "sphinx-doc__sphinx-8435", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "sympy__sympy-11400", "a": [1, 3], "b": [2, 3], "delta": -0.333333}, {"task": "sympy__sympy-14024", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "sympy__sympy-14817", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "sympy__sympy-15345", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "sympy__sympy-16792", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "sympy__sympy-20154", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "sympy__sympy-20590", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "sympy__sympy-23117", "a": [3, 3], "b": [3, 3], "delta": 0.0}]}, "cost": {"ratio": 1.9969, "ratio_ci": [1.763, 2.3329], "diff": 0.663606, "diff_ci": [0.558397, 0.770843], "clear": true, "costlier": 29, "n": 30, "median": 2.2389, "q1": 1.8529, "q3": 2.8098, "a_mean": 1.3293, "b_mean": 0.665695, "ratios": [{"task": "astropy__astropy-7746", "ratio": 1.9752, "a": 0.519517, "b": 0.263017}, {"task": "django__django-11019", "ratio": 1.5524, "a": 3.773404, "b": 2.430683}, {"task": "django__django-11564", "ratio": 1.9861, "a": 2.557073, "b": 1.287505}, {"task": "django__django-11620", "ratio": 2.2508, "a": 1.403359, "b": 0.623505}, {"task": "django__django-11630", "ratio": 2.825, "a": 1.179456, "b": 0.417503}, {"task": "django__django-11815", "ratio": 3.0652, "a": 1.294251, "b": 0.422244}, {"task": "django__django-12308", "ratio": 5.4824, "a": 0.909633, "b": 0.16592}, {"task": "django__django-12497", "ratio": 2.7367, "a": 0.662608, "b": 0.242122}, {"task": "django__django-12747", "ratio": 3.4621, "a": 1.502545, "b": 0.434001}, {"task": "django__django-14608", "ratio": 2.4661, "a": 1.253907, "b": 0.508462}, {"task": "django__django-15400", "ratio": 4.5567, "a": 0.934897, "b": 0.205171}, {"task": "django__django-15819", "ratio": 1.9766, "a": 1.037908, "b": 0.525093}, {"task": "matplotlib__matplotlib-24334", "ratio": 1.6397, "a": 0.909114, "b": 0.554438}, {"task": "matplotlib__matplotlib-25433", "ratio": 2.7641, "a": 1.84879, "b": 0.668859}, {"task": "psf__requests-1963", "ratio": 3.0258, "a": 1.22549, "b": 0.405011}, {"task": "pylint-dev__pylint-5859", "ratio": 3.1448, "a": 0.954612, "b": 0.30355}, {"task": "pytest-dev__pytest-5103", "ratio": 1.3273, "a": 2.803778, "b": 2.112378}, {"task": "pytest-dev__pytest-5495", "ratio": 1.8742, "a": 0.927302, "b": 0.49476}, {"task": "scikit-learn__scikit-learn-13241", "ratio": 2.4398, "a": 1.114892, "b": 0.456958}, {"task": "scikit-learn__scikit-learn-14983", "ratio": 2.1177, "a": 0.555798, "b": 0.262449}, {"task": "sphinx-doc__sphinx-10325", "ratio": 1.4394, "a": 1.880182, "b": 1.306258}, {"task": "sphinx-doc__sphinx-8435", "ratio": 1.666, "a": 1.280404, "b": 0.768539}, {"task": "sympy__sympy-11400", "ratio": 1.8458, "a": 1.037002, "b": 0.561813}, {"task": "sympy__sympy-14024", "ratio": 1.4928, "a": 1.821566, "b": 1.220238}, {"task": "sympy__sympy-14817", "ratio": 2.5743, "a": 0.694026, "b": 0.269596}, {"task": "sympy__sympy-15345", "ratio": 2.3418, "a": 0.951344, "b": 0.406253}, {"task": "sympy__sympy-16792", "ratio": 2.2271, "a": 1.509286, "b": 0.677692}, {"task": "sympy__sympy-20154", "ratio": 2.1291, "a": 1.663188, "b": 0.781172}, {"task": "sympy__sympy-20590", "ratio": 4.3939, "a": 0.713472, "b": 0.162379}, {"task": "sympy__sympy-23117", "ratio": 0.9293, "a": 0.960205, "b": 1.03327}]}}, {"kind": "pair", "id": "swe:all-arena-v14:fable:codex:sha256:f48cd74a7b628be67703bf1a6ad11bf3e59af79e91d6f92f69dd547f6246c7c7 vs swe:all-arena-v14:fable:pi:sha256:f48cd74a7b628be67703bf1a6ad11bf3e59af79e91d6f92f69dd547f6246c7c7", "model": "fable", "model_label": "Claude Fable 5", "color": "@model-fable", "harness": "codex", "harness_label": "Codex", "symbol": "triangle-up", "baseline": "pi", "baseline_label": "Pi", "a_label": "Claude Fable 5 \u00b7 Codex", "b_label": "Claude Fable 5 \u00b7 Pi", "cohort": "All-Arena reprofile r1-r3", "matched": true, "reasons": [], "descriptive": false, "evidence": "cost survives Holm", "caveats": [], "tasks": 30, "rate": {"mean": 0.0, "ci": [-0.055556, 0.055556], "p": 1.0, "p_holm": 1.0, "clear": false, "covers_zero": true, "wins_a": 1, "wins_b": 2, "same": 27, "mass_a": 0.022222, "mass_b": 0.022222, "breakdown": [{"task": "astropy__astropy-7746", "a": [3, 3], "b": [1, 3], "delta": 0.666667}, {"task": "django__django-11019", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "django__django-11564", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "django__django-11620", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "django__django-11630", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "django__django-11815", "a": [2, 3], "b": [3, 3], "delta": -0.333333}, {"task": "django__django-12308", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "django__django-12497", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "django__django-12747", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "django__django-14608", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "django__django-15400", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "django__django-15819", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "matplotlib__matplotlib-24334", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "matplotlib__matplotlib-25433", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "psf__requests-1963", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "pylint-dev__pylint-5859", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "pytest-dev__pytest-5103", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "pytest-dev__pytest-5495", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "scikit-learn__scikit-learn-13241", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "scikit-learn__scikit-learn-14983", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "sphinx-doc__sphinx-10325", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "sphinx-doc__sphinx-8435", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "sympy__sympy-11400", "a": [1, 3], "b": [2, 3], "delta": -0.333333}, {"task": "sympy__sympy-14024", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "sympy__sympy-14817", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "sympy__sympy-15345", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "sympy__sympy-16792", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "sympy__sympy-20154", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "sympy__sympy-20590", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "sympy__sympy-23117", "a": [3, 3], "b": [3, 3], "delta": 0.0}]}, "cost": {"ratio": 1.3377, "ratio_ci": [1.2192, 1.4965], "diff": 0.224772, "diff_ci": [0.154158, 0.301092], "clear": true, "costlier": 28, "n": 30, "median": 1.3439, "q1": 1.2157, "q3": 1.6567, "a_mean": 0.890467, "b_mean": 0.665695, "ratios": [{"task": "astropy__astropy-7746", "ratio": 1.2064, "a": 0.317315, "b": 0.263017}, {"task": "django__django-11019", "ratio": 1.2594, "a": 3.061146, "b": 2.430683}, {"task": "django__django-11564", "ratio": 1.251, "a": 1.610676, "b": 1.287505}, {"task": "django__django-11620", "ratio": 1.0992, "a": 0.685387, "b": 0.623505}, {"task": "django__django-11630", "ratio": 1.6887, "a": 0.705055, "b": 0.417503}, {"task": "django__django-11815", "ratio": 1.6612, "a": 0.701436, "b": 0.422244}, {"task": "django__django-12308", "ratio": 2.5399, "a": 0.421417, "b": 0.16592}, {"task": "django__django-12497", "ratio": 1.4706, "a": 0.356068, "b": 0.242122}, {"task": "django__django-12747", "ratio": 1.7769, "a": 0.771189, "b": 0.434001}, {"task": "django__django-14608", "ratio": 1.5166, "a": 0.771144, "b": 0.508462}, {"task": "django__django-15400", "ratio": 2.6244, "a": 0.538442, "b": 0.205171}, {"task": "django__django-15819", "ratio": 1.4274, "a": 0.749509, "b": 0.525093}, {"task": "matplotlib__matplotlib-24334", "ratio": 1.3278, "a": 0.736165, "b": 0.554438}, {"task": "matplotlib__matplotlib-25433", "ratio": 2.3881, "a": 1.597304, "b": 0.668859}, {"task": "psf__requests-1963", "ratio": 1.2501, "a": 0.5063, "b": 0.405011}, {"task": "pylint-dev__pylint-5859", "ratio": 2.5739, "a": 0.781318, "b": 0.30355}, {"task": "pytest-dev__pytest-5103", "ratio": 0.9847, "a": 2.080111, "b": 2.112378}, {"task": "pytest-dev__pytest-5495", "ratio": 1.2837, "a": 0.635124, "b": 0.49476}, {"task": "scikit-learn__scikit-learn-13241", "ratio": 1.1, "a": 0.502643, "b": 0.456958}, {"task": "scikit-learn__scikit-learn-14983", "ratio": 1.1202, "a": 0.294002, "b": 0.262449}, {"task": "sphinx-doc__sphinx-10325", "ratio": 1.2437, "a": 1.62464, "b": 1.306258}, {"task": "sphinx-doc__sphinx-8435", "ratio": 1.6432, "a": 1.262853, "b": 0.768539}, {"task": "sympy__sympy-11400", "ratio": 1.3253, "a": 0.744556, "b": 0.561813}, {"task": "sympy__sympy-14024", "ratio": 1.013, "a": 1.236095, "b": 1.220238}, {"task": "sympy__sympy-14817", "ratio": 1.492, "a": 0.402243, "b": 0.269596}, {"task": "sympy__sympy-15345", "ratio": 1.3599, "a": 0.552447, "b": 0.406253}, {"task": "sympy__sympy-16792", "ratio": 1.3862, "a": 0.9394, "b": 0.677692}, {"task": "sympy__sympy-20154", "ratio": 1.0871, "a": 0.849233, "b": 0.781172}, {"task": "sympy__sympy-20590", "ratio": 1.7574, "a": 0.285369, "b": 0.162379}, {"task": "sympy__sympy-23117", "ratio": 0.9634, "a": 0.995411, "b": 1.03327}]}}, {"kind": "pair", "id": "swe:all-arena-v14:opus:cc:sha256:f48cd74a7b628be67703bf1a6ad11bf3e59af79e91d6f92f69dd547f6246c7c7 vs swe:all-arena-v14:opus:pi:sha256:f48cd74a7b628be67703bf1a6ad11bf3e59af79e91d6f92f69dd547f6246c7c7", "model": "opus", "model_label": "Claude Opus 4.8", "color": "@model-opus", "harness": "cc", "harness_label": "Claude Code", "symbol": "diamond", "baseline": "pi", "baseline_label": "Pi", "a_label": "Claude Opus 4.8 \u00b7 Claude Code", "b_label": "Claude Opus 4.8 \u00b7 Pi", "cohort": "All-Arena reprofile r1-r3", "matched": true, "reasons": [], "descriptive": false, "evidence": "cost survives Holm", "caveats": [], "tasks": 30, "rate": {"mean": 0.044444, "ci": [0.011111, 0.088889], "p": 0.0216, "p_holm": 0.43200000000000005, "clear": false, "covers_zero": false, "wins_a": 4, "wins_b": 0, "same": 26, "mass_a": 0.044444, "mass_b": 0.0, "breakdown": [{"task": "astropy__astropy-7746", "a": [1, 3], "b": [1, 3], "delta": 0.0}, {"task": "django__django-11019", "a": [1, 3], "b": [0, 3], "delta": 0.333333}, {"task": "django__django-11564", "a": [1, 3], "b": [1, 3], "delta": 0.0}, {"task": "django__django-11620", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "django__django-11630", "a": [3, 3], "b": [2, 3], "delta": 0.333333}, {"task": "django__django-11815", "a": [3, 3], "b": [2, 3], "delta": 0.333333}, {"task": "django__django-12308", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "django__django-12497", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "django__django-12747", "a": [3, 3], "b": [2, 3], "delta": 0.333333}, {"task": "django__django-14608", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "django__django-15400", "a": [2, 3], "b": [2, 3], "delta": 0.0}, {"task": "django__django-15819", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "matplotlib__matplotlib-24334", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "matplotlib__matplotlib-25433", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "psf__requests-1963", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "pylint-dev__pylint-5859", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "pytest-dev__pytest-5103", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "pytest-dev__pytest-5495", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "scikit-learn__scikit-learn-13241", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "scikit-learn__scikit-learn-14983", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "sphinx-doc__sphinx-10325", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "sphinx-doc__sphinx-8435", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "sympy__sympy-11400", "a": [0, 3], "b": [0, 3], "delta": 0.0}, {"task": "sympy__sympy-14024", "a": [1, 3], "b": [1, 3], "delta": 0.0}, {"task": "sympy__sympy-14817", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "sympy__sympy-15345", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "sympy__sympy-16792", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "sympy__sympy-20154", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "sympy__sympy-20590", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "sympy__sympy-23117", "a": [3, 3], "b": [3, 3], "delta": 0.0}]}, "cost": {"ratio": 2.0647, "ratio_ci": [1.5194, 2.9766], "diff": 0.503158, "diff_ci": [0.313066, 0.669473], "clear": true, "costlier": 29, "n": 30, "median": 2.9844, "q1": 2.2644, "q3": 4.816, "a_mean": 0.975761, "b_mean": 0.472603, "ratios": [{"task": "astropy__astropy-7746", "ratio": 6.1717, "a": 0.768432, "b": 0.124509}, {"task": "django__django-11019", "ratio": 1.6568, "a": 3.231764, "b": 1.950564}, {"task": "django__django-11564", "ratio": 2.3342, "a": 2.398488, "b": 1.02753}, {"task": "django__django-11620", "ratio": 2.8665, "a": 1.596228, "b": 0.556859}, {"task": "django__django-11630", "ratio": 0.363, "a": 0.774775, "b": 2.134651}, {"task": "django__django-11815", "ratio": 14.0592, "a": 0.60242, "b": 0.042849}, {"task": "django__django-12308", "ratio": 2.9598, "a": 0.45802, "b": 0.154748}, {"task": "django__django-12497", "ratio": 4.7895, "a": 0.421688, "b": 0.088044}, {"task": "django__django-12747", "ratio": 4.8248, "a": 1.097361, "b": 0.227444}, {"task": "django__django-14608", "ratio": 1.5488, "a": 0.772053, "b": 0.49847}, {"task": "django__django-15400", "ratio": 5.7903, "a": 0.540319, "b": 0.093315}, {"task": "django__django-15819", "ratio": 3.156, "a": 0.918207, "b": 0.29094}, {"task": "matplotlib__matplotlib-24334", "ratio": 4.7898, "a": 0.581328, "b": 0.121367}, {"task": "matplotlib__matplotlib-25433", "ratio": 2.1144, "a": 1.042059, "b": 0.492829}, {"task": "psf__requests-1963", "ratio": 3.9003, "a": 0.448829, "b": 0.115076}, {"task": "pylint-dev__pylint-5859", "ratio": 6.9239, "a": 0.689997, "b": 0.099655}, {"task": "pytest-dev__pytest-5103", "ratio": 1.0291, "a": 2.668538, "b": 2.592979}, {"task": "pytest-dev__pytest-5495", "ratio": 2.5392, "a": 0.5893, "b": 0.232082}, {"task": "scikit-learn__scikit-learn-13241", "ratio": 5.1275, "a": 0.682497, "b": 0.133106}, {"task": "scikit-learn__scikit-learn-14983", "ratio": 2.2411, "a": 0.333256, "b": 0.148701}, {"task": "sphinx-doc__sphinx-10325", "ratio": 3.1996, "a": 1.675262, "b": 0.523579}, {"task": "sphinx-doc__sphinx-8435", "ratio": 2.2175, "a": 0.896375, "b": 0.404226}, {"task": "sympy__sympy-11400", "ratio": 2.4409, "a": 0.368708, "b": 0.151056}, {"task": "sympy__sympy-14024", "ratio": 3.0091, "a": 2.172055, "b": 0.721839}, {"task": "sympy__sympy-14817", "ratio": 5.2744, "a": 0.652366, "b": 0.123685}, {"task": "sympy__sympy-15345", "ratio": 9.9393, "a": 0.645096, "b": 0.064904}, {"task": "sympy__sympy-16792", "ratio": 2.4169, "a": 0.649648, "b": 0.268797}, {"task": "sympy__sympy-20154", "ratio": 1.2117, "a": 0.647965, "b": 0.53477}, {"task": "sympy__sympy-20590", "ratio": 2.8616, "a": 0.331613, "b": 0.115884}, {"task": "sympy__sympy-23117", "ratio": 4.3041, "a": 0.618172, "b": 0.143625}]}}, {"kind": "pair", "id": "swe:all-arena-v14:opus:codex:sha256:f48cd74a7b628be67703bf1a6ad11bf3e59af79e91d6f92f69dd547f6246c7c7 vs swe:all-arena-v14:opus:pi:sha256:f48cd74a7b628be67703bf1a6ad11bf3e59af79e91d6f92f69dd547f6246c7c7", "model": "opus", "model_label": "Claude Opus 4.8", "color": "@model-opus", "harness": "codex", "harness_label": "Codex", "symbol": "triangle-up", "baseline": "pi", "baseline_label": "Pi", "a_label": "Claude Opus 4.8 \u00b7 Codex", "b_label": "Claude Opus 4.8 \u00b7 Pi", "cohort": "All-Arena reprofile r1-r3", "matched": true, "reasons": [], "descriptive": false, "evidence": "rate survives Holm", "caveats": [], "tasks": 30, "rate": {"mean": 0.066667, "ci": [0.022222, 0.111111], "p": 0.002, "p_holm": 0.042, "clear": true, "covers_zero": false, "wins_a": 6, "wins_b": 0, "same": 24, "mass_a": 0.066667, "mass_b": 0.0, "breakdown": [{"task": "astropy__astropy-7746", "a": [2, 3], "b": [1, 3], "delta": 0.333333}, {"task": "django__django-11019", "a": [0, 3], "b": [0, 3], "delta": 0.0}, {"task": "django__django-11564", "a": [2, 3], "b": [1, 3], "delta": 0.333333}, {"task": "django__django-11620", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "django__django-11630", "a": [3, 3], "b": [2, 3], "delta": 0.333333}, {"task": "django__django-11815", "a": [3, 3], "b": [2, 3], "delta": 0.333333}, {"task": "django__django-12308", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "django__django-12497", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "django__django-12747", "a": [3, 3], "b": [2, 3], "delta": 0.333333}, {"task": "django__django-14608", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "django__django-15400", "a": [3, 3], "b": [2, 3], "delta": 0.333333}, {"task": "django__django-15819", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "matplotlib__matplotlib-24334", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "matplotlib__matplotlib-25433", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "psf__requests-1963", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "pylint-dev__pylint-5859", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "pytest-dev__pytest-5103", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "pytest-dev__pytest-5495", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "scikit-learn__scikit-learn-13241", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "scikit-learn__scikit-learn-14983", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "sphinx-doc__sphinx-10325", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "sphinx-doc__sphinx-8435", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "sympy__sympy-11400", "a": [0, 3], "b": [0, 3], "delta": 0.0}, {"task": "sympy__sympy-14024", "a": [1, 3], "b": [1, 3], "delta": 0.0}, {"task": "sympy__sympy-14817", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "sympy__sympy-15345", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "sympy__sympy-16792", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "sympy__sympy-20154", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "sympy__sympy-20590", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "sympy__sympy-23117", "a": [3, 3], "b": [3, 3], "delta": 0.0}]}, "cost": {"ratio": 1.4679, "ratio_ci": [1.1197, 2.017], "diff": 0.221132, "diff_ci": [0.069017, 0.338002], "clear": false, "costlier": 28, "n": 30, "median": 2.0871, "q1": 1.5221, "q3": 2.8433, "a_mean": 0.693734, "b_mean": 0.472603, "ratios": [{"task": "astropy__astropy-7746", "ratio": 2.7183, "a": 0.338459, "b": 0.124509}, {"task": "django__django-11019", "ratio": 1.3662, "a": 2.664919, "b": 1.950564}, {"task": "django__django-11564", "ratio": 1.4964, "a": 1.537634, "b": 1.02753}, {"task": "django__django-11620", "ratio": 1.4315, "a": 0.797126, "b": 0.556859}, {"task": "django__django-11630", "ratio": 0.3088, "a": 0.65912, "b": 2.134651}, {"task": "django__django-11815", "ratio": 9.7745, "a": 0.418826, "b": 0.042849}, {"task": "django__django-12308", "ratio": 2.3014, "a": 0.356135, "b": 0.154748}, {"task": "django__django-12497", "ratio": 2.5247, "a": 0.222281, "b": 0.088044}, {"task": "django__django-12747", "ratio": 2.885, "a": 0.656172, "b": 0.227444}, {"task": "django__django-14608", "ratio": 1.3206, "a": 0.658277, "b": 0.49847}, {"task": "django__django-15400", "ratio": 2.8911, "a": 0.269788, "b": 0.093315}, {"task": "django__django-15819", "ratio": 1.6855, "a": 0.490394, "b": 0.29094}, {"task": "matplotlib__matplotlib-24334", "ratio": 3.2776, "a": 0.397793, "b": 0.121367}, {"task": "matplotlib__matplotlib-25433", "ratio": 2.9565, "a": 1.45705, "b": 0.492829}, {"task": "psf__requests-1963", "ratio": 2.7105, "a": 0.311918, "b": 0.115076}, {"task": "pylint-dev__pylint-5859", "ratio": 5.3718, "a": 0.535326, "b": 0.099655}, {"task": "pytest-dev__pytest-5103", "ratio": 1.0157, "a": 2.633665, "b": 2.592979}, {"task": "pytest-dev__pytest-5495", "ratio": 2.1271, "a": 0.493669, "b": 0.232082}, {"task": "scikit-learn__scikit-learn-13241", "ratio": 1.7621, "a": 0.234541, "b": 0.133106}, {"task": "scikit-learn__scikit-learn-14983", "ratio": 2.047, "a": 0.304386, "b": 0.148701}, {"task": "sphinx-doc__sphinx-10325", "ratio": 1.9256, "a": 1.008181, "b": 0.523579}, {"task": "sphinx-doc__sphinx-8435", "ratio": 1.9692, "a": 0.79601, "b": 0.404226}, {"task": "sympy__sympy-11400", "ratio": 2.2285, "a": 0.336632, "b": 0.151056}, {"task": "sympy__sympy-14024", "ratio": 1.5206, "a": 1.097645, "b": 0.721839}, {"task": "sympy__sympy-14817", "ratio": 2.6119, "a": 0.323053, "b": 0.123685}, {"task": "sympy__sympy-15345", "ratio": 4.605, "a": 0.298882, "b": 0.064904}, {"task": "sympy__sympy-16792", "ratio": 1.5267, "a": 0.410372, "b": 0.268797}, {"task": "sympy__sympy-20154", "ratio": 0.727, "a": 0.388753, "b": 0.53477}, {"task": "sympy__sympy-20590", "ratio": 1.8802, "a": 0.217891, "b": 0.115884}, {"task": "sympy__sympy-23117", "ratio": 3.4613, "a": 0.497131, "b": 0.143625}]}}, {"kind": "pair", "id": "swe:all-arena-v14:sonnet:cc:sha256:f48cd74a7b628be67703bf1a6ad11bf3e59af79e91d6f92f69dd547f6246c7c7 vs swe:all-arena-v14:sonnet:pi:sha256:f48cd74a7b628be67703bf1a6ad11bf3e59af79e91d6f92f69dd547f6246c7c7", "model": "sonnet", "model_label": "Claude Sonnet 4.6", "color": "@model-sonnet", "harness": "cc", "harness_label": "Claude Code", "symbol": "diamond", "baseline": "pi", "baseline_label": "Pi", "a_label": "Claude Sonnet 4.6 \u00b7 Claude Code", "b_label": "Claude Sonnet 4.6 \u00b7 Pi", "cohort": "All-Arena reprofile r1-r3", "matched": true, "reasons": [], "descriptive": false, "evidence": "not clear after Holm", "caveats": [], "tasks": 30, "rate": {"mean": 0.022222, "ci": [-0.044444, 0.1], "p": 0.6916, "p_holm": 1.0, "clear": false, "covers_zero": true, "wins_a": 3, "wins_b": 3, "same": 24, "mass_a": 0.055556, "mass_b": 0.033333, "breakdown": [{"task": "astropy__astropy-7746", "a": [0, 3], "b": [0, 3], "delta": 0.0}, {"task": "django__django-11019", "a": [0, 3], "b": [0, 3], "delta": 0.0}, {"task": "django__django-11564", "a": [0, 3], "b": [0, 3], "delta": 0.0}, {"task": "django__django-11620", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "django__django-11630", "a": [0, 3], "b": [0, 3], "delta": 0.0}, {"task": "django__django-11815", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "django__django-12308", "a": [1, 3], "b": [2, 3], "delta": -0.333333}, {"task": "django__django-12497", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "django__django-12747", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "django__django-14608", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "django__django-15400", "a": [0, 3], "b": [0, 3], "delta": 0.0}, {"task": "django__django-15819", "a": [0, 3], "b": [0, 3], "delta": 0.0}, {"task": "matplotlib__matplotlib-24334", "a": [1, 3], "b": [0, 3], "delta": 0.333333}, {"task": "matplotlib__matplotlib-25433", "a": [2, 3], "b": [2, 3], "delta": 0.0}, {"task": "psf__requests-1963", "a": [2, 3], "b": [3, 3], "delta": -0.333333}, {"task": "pylint-dev__pylint-5859", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "pytest-dev__pytest-5103", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "pytest-dev__pytest-5495", "a": [2, 3], "b": [0, 3], "delta": 0.666667}, {"task": "scikit-learn__scikit-learn-13241", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "scikit-learn__scikit-learn-14983", "a": [2, 3], "b": [3, 3], "delta": -0.333333}, {"task": "sphinx-doc__sphinx-10325", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "sphinx-doc__sphinx-8435", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "sympy__sympy-11400", "a": [0, 3], "b": [0, 3], "delta": 0.0}, {"task": "sympy__sympy-14024", "a": [2, 3], "b": [0, 3], "delta": 0.666667}, {"task": "sympy__sympy-14817", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "sympy__sympy-15345", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "sympy__sympy-16792", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "sympy__sympy-20154", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "sympy__sympy-20590", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "sympy__sympy-23117", "a": [3, 3], "b": [3, 3], "delta": 0.0}]}, "cost": {"ratio": 0.9851, "ratio_ci": [0.8622, 1.1754], "diff": -0.01014, "diff_ci": [-0.113844, 0.094139], "clear": false, "costlier": 18, "n": 30, "median": 1.1289, "q1": 0.8413, "q3": 1.4292, "a_mean": 0.669348, "b_mean": 0.679488, "ratios": [{"task": "astropy__astropy-7746", "ratio": 1.1413, "a": 0.233385, "b": 0.204493}, {"task": "django__django-11019", "ratio": 0.8206, "a": 1.512953, "b": 1.843746}, {"task": "django__django-11564", "ratio": 0.6233, "a": 1.219751, "b": 1.956886}, {"task": "django__django-11620", "ratio": 0.7464, "a": 0.843366, "b": 1.129891}, {"task": "django__django-11630", "ratio": 1.4502, "a": 0.621649, "b": 0.428671}, {"task": "django__django-11815", "ratio": 1.3046, "a": 0.471919, "b": 0.361723}, {"task": "django__django-12308", "ratio": 1.0021, "a": 0.480745, "b": 0.479743}, {"task": "django__django-12497", "ratio": 1.7538, "a": 0.260532, "b": 0.148553}, {"task": "django__django-12747", "ratio": 0.9257, "a": 0.514114, "b": 0.555369}, {"task": "django__django-14608", "ratio": 1.0569, "a": 0.629344, "b": 0.595469}, {"task": "django__django-15400", "ratio": 0.6837, "a": 0.137385, "b": 0.200947}, {"task": "django__django-15819", "ratio": 0.9777, "a": 0.480739, "b": 0.49168}, {"task": "matplotlib__matplotlib-24334", "ratio": 1.1528, "a": 0.460846, "b": 0.399756}, {"task": "matplotlib__matplotlib-25433", "ratio": 1.3187, "a": 1.509038, "b": 1.144367}, {"task": "psf__requests-1963", "ratio": 0.6612, "a": 0.272237, "b": 0.411738}, {"task": "pylint-dev__pylint-5859", "ratio": 0.9486, "a": 0.313533, "b": 0.330538}, {"task": "pytest-dev__pytest-5103", "ratio": 0.9034, "a": 1.841363, "b": 2.038293}, {"task": "pytest-dev__pytest-5495", "ratio": 3.57, "a": 0.930529, "b": 0.260652}, {"task": "scikit-learn__scikit-learn-13241", "ratio": 1.577, "a": 0.307517, "b": 0.195006}, {"task": "scikit-learn__scikit-learn-14983", "ratio": 1.5944, "a": 0.300356, "b": 0.188382}, {"task": "sphinx-doc__sphinx-10325", "ratio": 1.8744, "a": 1.187941, "b": 0.633783}, {"task": "sphinx-doc__sphinx-8435", "ratio": 0.7434, "a": 1.361957, "b": 1.832188}, {"task": "sympy__sympy-11400", "ratio": 1.969, "a": 0.358355, "b": 0.181995}, {"task": "sympy__sympy-14024", "ratio": 0.7929, "a": 1.850445, "b": 2.33372}, {"task": "sympy__sympy-14817", "ratio": 1.3662, "a": 0.340719, "b": 0.249392}, {"task": "sympy__sympy-15345", "ratio": 1.3537, "a": 0.468037, "b": 0.345739}, {"task": "sympy__sympy-16792", "ratio": 0.5004, "a": 0.411339, "b": 0.822084}, {"task": "sympy__sympy-20154", "ratio": 1.1165, "a": 0.403163, "b": 0.361102}, {"task": "sympy__sympy-20590", "ratio": 1.5657, "a": 0.206377, "b": 0.13181}, {"task": "sympy__sympy-23117", "ratio": 1.1881, "a": 0.150813, "b": 0.126934}]}}, {"kind": "pair", "id": "swe:all-arena-v14:sonnet:codex:sha256:f48cd74a7b628be67703bf1a6ad11bf3e59af79e91d6f92f69dd547f6246c7c7 vs swe:all-arena-v14:sonnet:pi:sha256:f48cd74a7b628be67703bf1a6ad11bf3e59af79e91d6f92f69dd547f6246c7c7", "model": "sonnet", "model_label": "Claude Sonnet 4.6", "color": "@model-sonnet", "harness": "codex", "harness_label": "Codex", "symbol": "triangle-up", "baseline": "pi", "baseline_label": "Pi", "a_label": "Claude Sonnet 4.6 \u00b7 Codex", "b_label": "Claude Sonnet 4.6 \u00b7 Pi", "cohort": "All-Arena reprofile r1-r3", "matched": true, "reasons": [], "descriptive": false, "evidence": "not clear after Holm", "caveats": [], "tasks": 30, "rate": {"mean": 0.044444, "ci": [-0.033333, 0.133333], "p": 0.3684, "p_holm": 1.0, "clear": false, "covers_zero": true, "wins_a": 3, "wins_b": 2, "same": 25, "mass_a": 0.066667, "mass_b": 0.022222, "breakdown": [{"task": "astropy__astropy-7746", "a": [0, 3], "b": [0, 3], "delta": 0.0}, {"task": "django__django-11019", "a": [0, 3], "b": [0, 3], "delta": 0.0}, {"task": "django__django-11564", "a": [0, 3], "b": [0, 3], "delta": 0.0}, {"task": "django__django-11620", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "django__django-11630", "a": [0, 3], "b": [0, 3], "delta": 0.0}, {"task": "django__django-11815", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "django__django-12308", "a": [2, 3], "b": [2, 3], "delta": 0.0}, {"task": "django__django-12497", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "django__django-12747", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "django__django-14608", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "django__django-15400", "a": [0, 3], "b": [0, 3], "delta": 0.0}, {"task": "django__django-15819", "a": [0, 3], "b": [0, 3], "delta": 0.0}, {"task": "matplotlib__matplotlib-24334", "a": [0, 3], "b": [0, 3], "delta": 0.0}, {"task": "matplotlib__matplotlib-25433", "a": [1, 3], "b": [2, 3], "delta": -0.333333}, {"task": "psf__requests-1963", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "pylint-dev__pylint-5859", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "pytest-dev__pytest-5103", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "pytest-dev__pytest-5495", "a": [2, 3], "b": [0, 3], "delta": 0.666667}, {"task": "scikit-learn__scikit-learn-13241", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "scikit-learn__scikit-learn-14983", "a": [2, 3], "b": [3, 3], "delta": -0.333333}, {"task": "sphinx-doc__sphinx-10325", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "sphinx-doc__sphinx-8435", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "sympy__sympy-11400", "a": [1, 3], "b": [0, 3], "delta": 0.333333}, {"task": "sympy__sympy-14024", "a": [3, 3], "b": [0, 3], "delta": 1.0}, {"task": "sympy__sympy-14817", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "sympy__sympy-15345", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "sympy__sympy-16792", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "sympy__sympy-20154", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "sympy__sympy-20590", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "sympy__sympy-23117", "a": [3, 3], "b": [3, 3], "delta": 0.0}]}, "cost": {"ratio": 1.0959, "ratio_ci": [0.8657, 1.4261], "diff": 0.065185, "diff_ci": [-0.096891, 0.301853], "clear": false, "costlier": 18, "n": 30, "median": 1.0953, "q1": 0.9197, "q3": 1.2832, "a_mean": 0.744673, "b_mean": 0.679488, "ratios": [{"task": "astropy__astropy-7746", "ratio": 1.153, "a": 0.235789, "b": 0.204493}, {"task": "django__django-11019", "ratio": 2.4946, "a": 4.59932, "b": 1.843746}, {"task": "django__django-11564", "ratio": 0.72, "a": 1.408864, "b": 1.956886}, {"task": "django__django-11620", "ratio": 0.9641, "a": 1.0893, "b": 1.129891}, {"task": "django__django-11630", "ratio": 1.2885, "a": 0.552361, "b": 0.428671}, {"task": "django__django-11815", "ratio": 1.2005, "a": 0.434232, "b": 0.361723}, {"task": "django__django-12308", "ratio": 0.701, "a": 0.336279, "b": 0.479743}, {"task": "django__django-12497", "ratio": 1.0811, "a": 0.160602, "b": 0.148553}, {"task": "django__django-12747", "ratio": 1.2201, "a": 0.677605, "b": 0.555369}, {"task": "django__django-14608", "ratio": 0.9166, "a": 0.545789, "b": 0.595469}, {"task": "django__django-15400", "ratio": 1.2616, "a": 0.253507, "b": 0.200947}, {"task": "django__django-15819", "ratio": 1.4156, "a": 0.696035, "b": 0.49168}, {"task": "matplotlib__matplotlib-24334", "ratio": 0.97, "a": 0.387744, "b": 0.399756}, {"task": "matplotlib__matplotlib-25433", "ratio": 1.0749, "a": 1.230055, "b": 1.144367}, {"task": "psf__requests-1963", "ratio": 0.4418, "a": 0.181924, "b": 0.411738}, {"task": "pylint-dev__pylint-5859", "ratio": 1.0935, "a": 0.361438, "b": 0.330538}, {"task": "pytest-dev__pytest-5103", "ratio": 0.7719, "a": 1.573455, "b": 2.038293}, {"task": "pytest-dev__pytest-5495", "ratio": 2.6569, "a": 0.692524, "b": 0.260652}, {"task": "scikit-learn__scikit-learn-13241", "ratio": 1.4423, "a": 0.281263, "b": 0.195006}, {"task": "scikit-learn__scikit-learn-14983", "ratio": 1.2673, "a": 0.23873, "b": 0.188382}, {"task": "sphinx-doc__sphinx-10325", "ratio": 0.929, "a": 0.588777, "b": 0.633783}, {"task": "sphinx-doc__sphinx-8435", "ratio": 1.0971, "a": 2.010049, "b": 1.832188}, {"task": "sympy__sympy-11400", "ratio": 3.0596, "a": 0.55684, "b": 0.181995}, {"task": "sympy__sympy-14024", "ratio": 0.6341, "a": 1.479917, "b": 2.33372}, {"task": "sympy__sympy-14817", "ratio": 1.7019, "a": 0.424443, "b": 0.249392}, {"task": "sympy__sympy-15345", "ratio": 0.9717, "a": 0.335965, "b": 0.345739}, {"task": "sympy__sympy-16792", "ratio": 0.4913, "a": 0.403869, "b": 0.822084}, {"task": "sympy__sympy-20154", "ratio": 0.7571, "a": 0.273381, "b": 0.361102}, {"task": "sympy__sympy-20590", "ratio": 1.1826, "a": 0.155876, "b": 0.13181}, {"task": "sympy__sympy-23117", "ratio": 1.3728, "a": 0.174257, "b": 0.126934}]}}, {"kind": "pair", "id": "swe:all-arena-v14:haiku:cc:sha256:f48cd74a7b628be67703bf1a6ad11bf3e59af79e91d6f92f69dd547f6246c7c7 vs swe:all-arena-v14:haiku:pi:sha256:f48cd74a7b628be67703bf1a6ad11bf3e59af79e91d6f92f69dd547f6246c7c7", "model": "haiku", "model_label": "Claude Haiku 4.5", "color": "@model-haiku", "harness": "cc", "harness_label": "Claude Code", "symbol": "diamond", "baseline": "pi", "baseline_label": "Pi", "a_label": "Claude Haiku 4.5 \u00b7 Claude Code", "b_label": "Claude Haiku 4.5 \u00b7 Pi", "cohort": "All-Arena reprofile r1-r3", "matched": true, "reasons": [], "descriptive": false, "evidence": "not clear after Holm", "caveats": [], "tasks": 30, "rate": {"mean": -0.077778, "ci": [-0.166667, 0.0], "p": 0.0492, "p_holm": 0.9348, "clear": false, "covers_zero": true, "wins_a": 2, "wins_b": 6, "same": 22, "mass_a": 0.022222, "mass_b": 0.1, "breakdown": [{"task": "astropy__astropy-7746", "a": [0, 3], "b": [0, 3], "delta": 0.0}, {"task": "django__django-11019", "a": [0, 3], "b": [0, 3], "delta": 0.0}, {"task": "django__django-11564", "a": [0, 3], "b": [0, 3], "delta": 0.0}, {"task": "django__django-11620", "a": [2, 3], "b": [3, 3], "delta": -0.333333}, {"task": "django__django-11630", "a": [0, 3], "b": [0, 3], "delta": 0.0}, {"task": "django__django-11815", "a": [1, 3], "b": [3, 3], "delta": -0.666667}, {"task": "django__django-12308", "a": [0, 3], "b": [0, 3], "delta": 0.0}, {"task": "django__django-12497", "a": [2, 3], "b": [3, 3], "delta": -0.333333}, {"task": "django__django-12747", "a": [1, 3], "b": [1, 3], "delta": 0.0}, {"task": "django__django-14608", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "django__django-15400", "a": [3, 3], "b": [2, 3], "delta": 0.333333}, {"task": "django__django-15819", "a": [0, 3], "b": [0, 3], "delta": 0.0}, {"task": "matplotlib__matplotlib-24334", "a": [0, 3], "b": [0, 3], "delta": 0.0}, {"task": "matplotlib__matplotlib-25433", "a": [0, 3], "b": [2, 3], "delta": -0.666667}, {"task": "psf__requests-1963", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "pylint-dev__pylint-5859", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "pytest-dev__pytest-5103", "a": [2, 3], "b": [3, 3], "delta": -0.333333}, {"task": "pytest-dev__pytest-5495", "a": [0, 3], "b": [0, 3], "delta": 0.0}, {"task": "scikit-learn__scikit-learn-13241", "a": [1, 3], "b": [0, 3], "delta": 0.333333}, {"task": "scikit-learn__scikit-learn-14983", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "sphinx-doc__sphinx-10325", "a": [1, 3], "b": [3, 3], "delta": -0.666667}, {"task": "sphinx-doc__sphinx-8435", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "sympy__sympy-11400", "a": [1, 3], "b": [1, 3], "delta": 0.0}, {"task": "sympy__sympy-14024", "a": [0, 3], "b": [0, 3], "delta": 0.0}, {"task": "sympy__sympy-14817", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "sympy__sympy-15345", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "sympy__sympy-16792", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "sympy__sympy-20154", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "sympy__sympy-20590", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "sympy__sympy-23117", "a": [3, 3], "b": [3, 3], "delta": 0.0}]}, "cost": {"ratio": 1.1408, "ratio_ci": [0.9466, 1.3469], "diff": 0.052593, "diff_ci": [-0.0214, 0.123724], "clear": false, "costlier": 20, "n": 30, "median": 1.2692, "q1": 0.9198, "q3": 1.6004, "a_mean": 0.426164, "b_mean": 0.37357, "ratios": [{"task": "astropy__astropy-7746", "ratio": 0.9131, "a": 0.241967, "b": 0.264983}, {"task": "django__django-11019", "ratio": 1.1014, "a": 0.917796, "b": 0.833311}, {"task": "django__django-11564", "ratio": 0.034, "a": 0.017523, "b": 0.515119}, {"task": "django__django-11620", "ratio": 1.2773, "a": 0.490213, "b": 0.383781}, {"task": "django__django-11630", "ratio": 2.0069, "a": 0.182251, "b": 0.090812}, {"task": "django__django-11815", "ratio": 1.1085, "a": 0.400838, "b": 0.361611}, {"task": "django__django-12308", "ratio": 1.3223, "a": 0.54786, "b": 0.414329}, {"task": "django__django-12497", "ratio": 1.261, "a": 0.337517, "b": 0.26765}, {"task": "django__django-12747", "ratio": 0.8696, "a": 0.493225, "b": 0.567187}, {"task": "django__django-14608", "ratio": 2.2001, "a": 0.902759, "b": 0.410334}, {"task": "django__django-15400", "ratio": 1.2516, "a": 0.290662, "b": 0.232239}, {"task": "django__django-15819", "ratio": 1.6231, "a": 0.642228, "b": 0.39568}, {"task": "matplotlib__matplotlib-24334", "ratio": 1.496, "a": 0.475875, "b": 0.318099}, {"task": "matplotlib__matplotlib-25433", "ratio": 1.5553, "a": 0.764832, "b": 0.491744}, {"task": "psf__requests-1963", "ratio": 0.8764, "a": 0.39126, "b": 0.446445}, {"task": "pylint-dev__pylint-5859", "ratio": 1.9273, "a": 0.333312, "b": 0.172944}, {"task": "pytest-dev__pytest-5103", "ratio": 0.9398, "a": 0.649925, "b": 0.69156}, {"task": "pytest-dev__pytest-5495", "ratio": 0.7479, "a": 0.218389, "b": 0.292004}, {"task": "scikit-learn__scikit-learn-13241", "ratio": 0.2795, "a": 0.071773, "b": 0.25681}, {"task": "scikit-learn__scikit-learn-14983", "ratio": 1.7637, "a": 0.394712, "b": 0.223799}, {"task": "sphinx-doc__sphinx-10325", "ratio": 0.4761, "a": 0.225755, "b": 0.474158}, {"task": "sphinx-doc__sphinx-8435", "ratio": 1.4097, "a": 0.770825, "b": 0.546789}, {"task": "sympy__sympy-11400", "ratio": 0.9708, "a": 0.442088, "b": 0.455395}, {"task": "sympy__sympy-14024", "ratio": 0.4123, "a": 0.266776, "b": 0.647106}, {"task": "sympy__sympy-14817", "ratio": 1.6154, "a": 0.320967, "b": 0.198693}, {"task": "sympy__sympy-15345", "ratio": 3.6097, "a": 0.560705, "b": 0.155332}, {"task": "sympy__sympy-16792", "ratio": 1.0448, "a": 0.690593, "b": 0.660999}, {"task": "sympy__sympy-20154", "ratio": 1.4171, "a": 0.264668, "b": 0.186766}, {"task": "sympy__sympy-20590", "ratio": 3.2475, "a": 0.250318, "b": 0.077079}, {"task": "sympy__sympy-23117", "ratio": 1.3037, "a": 0.227305, "b": 0.174353}]}}, {"kind": "pair", "id": "swe:all-arena-v14:haiku:codex:sha256:f48cd74a7b628be67703bf1a6ad11bf3e59af79e91d6f92f69dd547f6246c7c7 vs swe:all-arena-v14:haiku:pi:sha256:f48cd74a7b628be67703bf1a6ad11bf3e59af79e91d6f92f69dd547f6246c7c7", "model": "haiku", "model_label": "Claude Haiku 4.5", "color": "@model-haiku", "harness": "codex", "harness_label": "Codex", "symbol": "triangle-up", "baseline": "pi", "baseline_label": "Pi", "a_label": "Claude Haiku 4.5 \u00b7 Codex", "b_label": "Claude Haiku 4.5 \u00b7 Pi", "cohort": "All-Arena reprofile r1-r3", "matched": true, "reasons": [], "descriptive": false, "evidence": "not clear after Holm", "caveats": [], "tasks": 30, "rate": {"mean": -0.022222, "ci": [-0.111111, 0.078056], "p": 0.6252, "p_holm": 1.0, "clear": false, "covers_zero": true, "wins_a": 2, "wins_b": 4, "same": 24, "mass_a": 0.044444, "mass_b": 0.066667, "breakdown": [{"task": "astropy__astropy-7746", "a": [0, 3], "b": [0, 3], "delta": 0.0}, {"task": "django__django-11019", "a": [0, 3], "b": [0, 3], "delta": 0.0}, {"task": "django__django-11564", "a": [0, 3], "b": [0, 3], "delta": 0.0}, {"task": "django__django-11620", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "django__django-11630", "a": [0, 3], "b": [0, 3], "delta": 0.0}, {"task": "django__django-11815", "a": [1, 3], "b": [3, 3], "delta": -0.666667}, {"task": "django__django-12308", "a": [0, 3], "b": [0, 3], "delta": 0.0}, {"task": "django__django-12497", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "django__django-12747", "a": [0, 3], "b": [1, 3], "delta": -0.333333}, {"task": "django__django-14608", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "django__django-15400", "a": [2, 3], "b": [2, 3], "delta": 0.0}, {"task": "django__django-15819", "a": [0, 3], "b": [0, 3], "delta": 0.0}, {"task": "matplotlib__matplotlib-24334", "a": [0, 3], "b": [0, 3], "delta": 0.0}, {"task": "matplotlib__matplotlib-25433", "a": [0, 3], "b": [2, 3], "delta": -0.666667}, {"task": "psf__requests-1963", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "pylint-dev__pylint-5859", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "pytest-dev__pytest-5103", "a": [2, 3], "b": [3, 3], "delta": -0.333333}, {"task": "pytest-dev__pytest-5495", "a": [0, 3], "b": [0, 3], "delta": 0.0}, {"task": "scikit-learn__scikit-learn-13241", "a": [3, 3], "b": [0, 3], "delta": 1.0}, {"task": "scikit-learn__scikit-learn-14983", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "sphinx-doc__sphinx-10325", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "sphinx-doc__sphinx-8435", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "sympy__sympy-11400", "a": [2, 3], "b": [1, 3], "delta": 0.333333}, {"task": "sympy__sympy-14024", "a": [0, 3], "b": [0, 3], "delta": 0.0}, {"task": "sympy__sympy-14817", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "sympy__sympy-15345", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "sympy__sympy-16792", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "sympy__sympy-20154", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "sympy__sympy-20590", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "sympy__sympy-23117", "a": [3, 3], "b": [3, 3], "delta": 0.0}]}, "cost": {"ratio": 1.049, "ratio_ci": [0.8663, 1.2661], "diff": 0.018317, "diff_ci": [-0.050857, 0.100701], "clear": false, "costlier": 13, "n": 30, "median": 0.9529, "q1": 0.752, "q3": 1.2464, "a_mean": 0.391888, "b_mean": 0.37357, "ratios": [{"task": "astropy__astropy-7746", "ratio": 0.8339, "a": 0.220975, "b": 0.264983}, {"task": "django__django-11019", "ratio": 1.0054, "a": 0.837795, "b": 0.833311}, {"task": "django__django-11564", "ratio": 0.8076, "a": 0.416004, "b": 0.515119}, {"task": "django__django-11620", "ratio": 1.021, "a": 0.391832, "b": 0.383781}, {"task": "django__django-11630", "ratio": 0.9456, "a": 0.08587, "b": 0.090812}, {"task": "django__django-11815", "ratio": 1.2751, "a": 0.461094, "b": 0.361611}, {"task": "django__django-12308", "ratio": 0.7399, "a": 0.306565, "b": 0.414329}, {"task": "django__django-12497", "ratio": 0.6369, "a": 0.170458, "b": 0.26765}, {"task": "django__django-12747", "ratio": 0.7044, "a": 0.399501, "b": 0.567187}, {"task": "django__django-14608", "ratio": 0.982, "a": 0.40293, "b": 0.410334}, {"task": "django__django-15400", "ratio": 1.2534, "a": 0.291082, "b": 0.232239}, {"task": "django__django-15819", "ratio": 1.3479, "a": 0.533331, "b": 0.39568}, {"task": "matplotlib__matplotlib-24334", "ratio": 0.9123, "a": 0.290191, "b": 0.318099}, {"task": "matplotlib__matplotlib-25433", "ratio": 1.3566, "a": 0.667117, "b": 0.491744}, {"task": "psf__requests-1963", "ratio": 0.56, "a": 0.250022, "b": 0.446445}, {"task": "pylint-dev__pylint-5859", "ratio": 1.2255, "a": 0.211945, "b": 0.172944}, {"task": "pytest-dev__pytest-5103", "ratio": 0.6093, "a": 0.421371, "b": 0.69156}, {"task": "pytest-dev__pytest-5495", "ratio": 1.0118, "a": 0.295442, "b": 0.292004}, {"task": "scikit-learn__scikit-learn-13241", "ratio": 0.8112, "a": 0.208329, "b": 0.25681}, {"task": "scikit-learn__scikit-learn-14983", "ratio": 0.9602, "a": 0.214887, "b": 0.223799}, {"task": "sphinx-doc__sphinx-10325", "ratio": 0.9133, "a": 0.433032, "b": 0.474158}, {"task": "sphinx-doc__sphinx-8435", "ratio": 2.248, "a": 1.229179, "b": 0.546789}, {"task": "sympy__sympy-11400", "ratio": 0.7004, "a": 0.318977, "b": 0.455395}, {"task": "sympy__sympy-14024", "ratio": 2.1978, "a": 1.422238, "b": 0.647106}, {"task": "sympy__sympy-14817", "ratio": 1.1061, "a": 0.219782, "b": 0.198693}, {"task": "sympy__sympy-15345", "ratio": 1.7783, "a": 0.276226, "b": 0.155332}, {"task": "sympy__sympy-16792", "ratio": 0.5724, "a": 0.378355, "b": 0.660999}, {"task": "sympy__sympy-20154", "ratio": 0.7881, "a": 0.147187, "b": 0.186766}, {"task": "sympy__sympy-20590", "ratio": 1.661, "a": 0.128029, "b": 0.077079}, {"task": "sympy__sympy-23117", "ratio": 0.7277, "a": 0.126882, "b": 0.174353}]}}, {"kind": "pair", "id": "swe:all-arena-v14:sol:cc:sha256:f48cd74a7b628be67703bf1a6ad11bf3e59af79e91d6f92f69dd547f6246c7c7 vs swe:all-arena-v14:sol:pi:sha256:f48cd74a7b628be67703bf1a6ad11bf3e59af79e91d6f92f69dd547f6246c7c7", "model": "sol", "model_label": "GPT-5.6 Sol", "color": "@model-sol", "harness": "cc", "harness_label": "Claude Code", "symbol": "diamond", "baseline": "pi", "baseline_label": "Pi", "a_label": "GPT-5.6 Sol \u00b7 Claude Code", "b_label": "GPT-5.6 Sol \u00b7 Pi", "cohort": "All-Arena reprofile r1-r3", "matched": true, "reasons": [], "descriptive": false, "evidence": "cost survives Holm", "caveats": [], "tasks": 30, "rate": {"mean": 0.033333, "ci": [-0.055556, 0.133333], "p": 0.5764, "p_holm": 1.0, "clear": false, "covers_zero": true, "wins_a": 3, "wins_b": 2, "same": 25, "mass_a": 0.066667, "mass_b": 0.033333, "breakdown": [{"task": "astropy__astropy-7746", "a": [0, 3], "b": [0, 3], "delta": 0.0}, {"task": "django__django-11019", "a": [2, 3], "b": [0, 3], "delta": 0.666667}, {"task": "django__django-11564", "a": [2, 3], "b": [2, 3], "delta": 0.0}, {"task": "django__django-11620", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "django__django-11630", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "django__django-11815", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "django__django-12308", "a": [0, 3], "b": [0, 3], "delta": 0.0}, {"task": "django__django-12497", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "django__django-12747", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "django__django-14608", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "django__django-15400", "a": [0, 3], "b": [0, 3], "delta": 0.0}, {"task": "django__django-15819", "a": [0, 3], "b": [0, 3], "delta": 0.0}, {"task": "matplotlib__matplotlib-24334", "a": [1, 3], "b": [3, 3], "delta": -0.666667}, {"task": "matplotlib__matplotlib-25433", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "psf__requests-1963", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "pylint-dev__pylint-5859", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "pytest-dev__pytest-5103", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "pytest-dev__pytest-5495", "a": [3, 3], "b": [2, 3], "delta": 0.333333}, {"task": "scikit-learn__scikit-learn-13241", "a": [3, 3], "b": [0, 3], "delta": 1.0}, {"task": "scikit-learn__scikit-learn-14983", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "sphinx-doc__sphinx-10325", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "sphinx-doc__sphinx-8435", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "sympy__sympy-11400", "a": [0, 3], "b": [0, 3], "delta": 0.0}, {"task": "sympy__sympy-14024", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "sympy__sympy-14817", "a": [2, 3], "b": [3, 3], "delta": -0.333333}, {"task": "sympy__sympy-15345", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "sympy__sympy-16792", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "sympy__sympy-20154", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "sympy__sympy-20590", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "sympy__sympy-23117", "a": [3, 3], "b": [3, 3], "delta": 0.0}]}, "cost": {"ratio": 3.4954, "ratio_ci": [2.8123, 4.4617], "diff": 1.099615, "diff_ci": [0.811354, 1.569021], "clear": true, "costlier": 30, "n": 30, "median": 3.4989, "q1": 2.7732, "q3": 4.7142, "a_mean": 1.540276, "b_mean": 0.440661, "ratios": [{"task": "astropy__astropy-7746", "ratio": 3.3768, "a": 1.230053, "b": 0.364263}, {"task": "django__django-11019", "ratio": 1.8991, "a": 1.679548, "b": 0.884409}, {"task": "django__django-11564", "ratio": 2.266, "a": 2.888033, "b": 1.274517}, {"task": "django__django-11620", "ratio": 3.785, "a": 1.106945, "b": 0.292455}, {"task": "django__django-11630", "ratio": 2.9531, "a": 1.267255, "b": 0.429133}, {"task": "django__django-11815", "ratio": 5.2151, "a": 1.169126, "b": 0.224181}, {"task": "django__django-12308", "ratio": 5.9296, "a": 0.975356, "b": 0.164489}, {"task": "django__django-12497", "ratio": 4.4495, "a": 0.592308, "b": 0.133118}, {"task": "django__django-12747", "ratio": 3.2872, "a": 1.052966, "b": 0.320325}, {"task": "django__django-14608", "ratio": 2.714, "a": 1.074184, "b": 0.395791}, {"task": "django__django-15400", "ratio": 3.6157, "a": 0.803143, "b": 0.222127}, {"task": "django__django-15819", "ratio": 3.5099, "a": 1.565383, "b": 0.445986}, {"task": "matplotlib__matplotlib-24334", "ratio": 6.0075, "a": 1.319916, "b": 0.219711}, {"task": "matplotlib__matplotlib-25433", "ratio": 4.0878, "a": 2.071013, "b": 0.506628}, {"task": "psf__requests-1963", "ratio": 3.4473, "a": 0.885793, "b": 0.256954}, {"task": "pylint-dev__pylint-5859", "ratio": 5.4914, "a": 1.145309, "b": 0.208566}, {"task": "pytest-dev__pytest-5103", "ratio": 8.0284, "a": 8.196011, "b": 1.020872}, {"task": "pytest-dev__pytest-5495", "ratio": 2.9509, "a": 1.447557, "b": 0.490555}, {"task": "scikit-learn__scikit-learn-13241", "ratio": 33.0624, "a": 0.95353, "b": 0.02884}, {"task": "scikit-learn__scikit-learn-14983", "ratio": 2.5373, "a": 0.961882, "b": 0.379102}, {"task": "sphinx-doc__sphinx-10325", "ratio": 1.9507, "a": 1.625215, "b": 0.833155}, {"task": "sphinx-doc__sphinx-8435", "ratio": 2.1092, "a": 1.506126, "b": 0.714069}, {"task": "sympy__sympy-11400", "ratio": 5.227, "a": 1.305918, "b": 0.249839}, {"task": "sympy__sympy-14024", "ratio": 2.6326, "a": 2.202997, "b": 0.836807}, {"task": "sympy__sympy-14817", "ratio": 1.7131, "a": 1.407364, "b": 0.821526}, {"task": "sympy__sympy-15345", "ratio": 4.8024, "a": 0.96467, "b": 0.200871}, {"task": "sympy__sympy-16792", "ratio": 3.4879, "a": 1.955372, "b": 0.560619}, {"task": "sympy__sympy-20154", "ratio": 3.335, "a": 0.928136, "b": 0.278299}, {"task": "sympy__sympy-20590", "ratio": 4.1217, "a": 1.010793, "b": 0.245237}, {"task": "sympy__sympy-23117", "ratio": 4.2152, "a": 0.916379, "b": 0.217399}]}}, {"kind": "pair", "id": "swe:all-arena-v14:sol:codex:sha256:f48cd74a7b628be67703bf1a6ad11bf3e59af79e91d6f92f69dd547f6246c7c7 vs swe:all-arena-v14:sol:pi:sha256:f48cd74a7b628be67703bf1a6ad11bf3e59af79e91d6f92f69dd547f6246c7c7", "model": "sol", "model_label": "GPT-5.6 Sol", "color": "@model-sol", "harness": "codex", "harness_label": "Codex", "symbol": "triangle-up", "baseline": "pi", "baseline_label": "Pi", "a_label": "GPT-5.6 Sol \u00b7 Codex", "b_label": "GPT-5.6 Sol \u00b7 Pi", "cohort": "All-Arena reprofile r1-r3", "matched": true, "reasons": [], "descriptive": false, "evidence": "cost survives Holm", "caveats": [], "tasks": 30, "rate": {"mean": -0.011111, "ci": [-0.077778, 0.044444], "p": 0.8036, "p_holm": 1.0, "clear": false, "covers_zero": true, "wins_a": 2, "wins_b": 2, "same": 26, "mass_a": 0.022222, "mass_b": 0.033333, "breakdown": [{"task": "astropy__astropy-7746", "a": [0, 3], "b": [0, 3], "delta": 0.0}, {"task": "django__django-11019", "a": [0, 3], "b": [0, 3], "delta": 0.0}, {"task": "django__django-11564", "a": [0, 3], "b": [2, 3], "delta": -0.666667}, {"task": "django__django-11620", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "django__django-11630", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "django__django-11815", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "django__django-12308", "a": [0, 3], "b": [0, 3], "delta": 0.0}, {"task": "django__django-12497", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "django__django-12747", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "django__django-14608", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "django__django-15400", "a": [0, 3], "b": [0, 3], "delta": 0.0}, {"task": "django__django-15819", "a": [1, 3], "b": [0, 3], "delta": 0.333333}, {"task": "matplotlib__matplotlib-24334", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "matplotlib__matplotlib-25433", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "psf__requests-1963", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "pylint-dev__pylint-5859", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "pytest-dev__pytest-5103", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "pytest-dev__pytest-5495", "a": [3, 3], "b": [2, 3], "delta": 0.333333}, {"task": "scikit-learn__scikit-learn-13241", "a": [0, 3], "b": [0, 3], "delta": 0.0}, {"task": "scikit-learn__scikit-learn-14983", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "sphinx-doc__sphinx-10325", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "sphinx-doc__sphinx-8435", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "sympy__sympy-11400", "a": [0, 3], "b": [0, 3], "delta": 0.0}, {"task": "sympy__sympy-14024", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "sympy__sympy-14817", "a": [2, 3], "b": [3, 3], "delta": -0.333333}, {"task": "sympy__sympy-15345", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "sympy__sympy-16792", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "sympy__sympy-20154", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "sympy__sympy-20590", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "sympy__sympy-23117", "a": [3, 3], "b": [3, 3], "delta": 0.0}]}, "cost": {"ratio": 1.2737, "ratio_ci": [1.16, 1.4078], "diff": 0.120604, "diff_ci": [0.071106, 0.179045], "clear": true, "costlier": 27, "n": 30, "median": 1.2864, "q1": 1.1467, "q3": 1.5631, "a_mean": 0.561265, "b_mean": 0.440661, "ratios": [{"task": "astropy__astropy-7746", "ratio": 1.123, "a": 0.409077, "b": 0.364263}, {"task": "django__django-11019", "ratio": 0.928, "a": 0.82077, "b": 0.884409}, {"task": "django__django-11564", "ratio": 1.0936, "a": 1.393775, "b": 1.274517}, {"task": "django__django-11620", "ratio": 1.1679, "a": 0.341546, "b": 0.292455}, {"task": "django__django-11630", "ratio": 0.8754, "a": 0.375653, "b": 0.429133}, {"task": "django__django-11815", "ratio": 2.0892, "a": 0.468354, "b": 0.224181}, {"task": "django__django-12308", "ratio": 1.3217, "a": 0.217398, "b": 0.164489}, {"task": "django__django-12497", "ratio": 1.1464, "a": 0.152612, "b": 0.133118}, {"task": "django__django-12747", "ratio": 1.6988, "a": 0.54416, "b": 0.320325}, {"task": "django__django-14608", "ratio": 1.2343, "a": 0.488541, "b": 0.395791}, {"task": "django__django-15400", "ratio": 1.1525, "a": 0.256006, "b": 0.222127}, {"task": "django__django-15819", "ratio": 1.4292, "a": 0.637381, "b": 0.445986}, {"task": "matplotlib__matplotlib-24334", "ratio": 1.8238, "a": 0.400715, "b": 0.219711}, {"task": "matplotlib__matplotlib-25433", "ratio": 1.1592, "a": 0.587278, "b": 0.506628}, {"task": "psf__requests-1963", "ratio": 1.2792, "a": 0.328691, "b": 0.256954}, {"task": "pylint-dev__pylint-5859", "ratio": 1.2935, "a": 0.269783, "b": 0.208566}, {"task": "pytest-dev__pytest-5103", "ratio": 1.7042, "a": 1.739779, "b": 1.020872}, {"task": "pytest-dev__pytest-5495", "ratio": 1.6077, "a": 0.788666, "b": 0.490555}, {"task": "scikit-learn__scikit-learn-13241", "ratio": 1.9369, "a": 0.05586, "b": 0.02884}, {"task": "scikit-learn__scikit-learn-14983", "ratio": 1.3621, "a": 0.516368, "b": 0.379102}, {"task": "sphinx-doc__sphinx-10325", "ratio": 1.0251, "a": 0.854103, "b": 0.833155}, {"task": "sphinx-doc__sphinx-8435", "ratio": 1.175, "a": 0.839043, "b": 0.714069}, {"task": "sympy__sympy-11400", "ratio": 2.1845, "a": 0.54578, "b": 0.249839}, {"task": "sympy__sympy-14024", "ratio": 1.0138, "a": 0.848324, "b": 0.836807}, {"task": "sympy__sympy-14817", "ratio": 1.3688, "a": 1.124492, "b": 0.821526}, {"task": "sympy__sympy-15345", "ratio": 1.3942, "a": 0.280052, "b": 0.200871}, {"task": "sympy__sympy-16792", "ratio": 0.8517, "a": 0.477499, "b": 0.560619}, {"task": "sympy__sympy-20154", "ratio": 1.1475, "a": 0.319337, "b": 0.278299}, {"task": "sympy__sympy-20590", "ratio": 1.8545, "a": 0.454795, "b": 0.245237}, {"task": "sympy__sympy-23117", "ratio": 1.3897, "a": 0.302125, "b": 0.217399}]}}, {"kind": "pair", "id": "swe:all-arena-v14:luna:cc:sha256:f48cd74a7b628be67703bf1a6ad11bf3e59af79e91d6f92f69dd547f6246c7c7 vs swe:all-arena-v14:luna:pi:sha256:f48cd74a7b628be67703bf1a6ad11bf3e59af79e91d6f92f69dd547f6246c7c7", "model": "luna", "model_label": "GPT-5.6 Luna", "color": "@model-luna", "harness": "cc", "harness_label": "Claude Code", "symbol": "diamond", "baseline": "pi", "baseline_label": "Pi", "a_label": "GPT-5.6 Luna \u00b7 Claude Code", "b_label": "GPT-5.6 Luna \u00b7 Pi", "cohort": "All-Arena reprofile r1-r3", "matched": true, "reasons": [], "descriptive": false, "evidence": "cost survives Holm", "caveats": [], "tasks": 30, "rate": {"mean": 0.022222, "ci": [-0.033333, 0.1], "p": 0.734, "p_holm": 1.0, "clear": false, "covers_zero": true, "wins_a": 1, "wins_b": 1, "same": 28, "mass_a": 0.033333, "mass_b": 0.011111, "breakdown": [{"task": "astropy__astropy-7746", "a": [0, 3], "b": [0, 3], "delta": 0.0}, {"task": "django__django-11019", "a": [0, 3], "b": [0, 3], "delta": 0.0}, {"task": "django__django-11564", "a": [0, 3], "b": [0, 3], "delta": 0.0}, {"task": "django__django-11620", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "django__django-11630", "a": [0, 3], "b": [0, 3], "delta": 0.0}, {"task": "django__django-11815", "a": [2, 3], "b": [2, 3], "delta": 0.0}, {"task": "django__django-12308", "a": [0, 3], "b": [0, 3], "delta": 0.0}, {"task": "django__django-12497", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "django__django-12747", "a": [0, 3], "b": [0, 3], "delta": 0.0}, {"task": "django__django-14608", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "django__django-15400", "a": [0, 3], "b": [0, 3], "delta": 0.0}, {"task": "django__django-15819", "a": [0, 3], "b": [0, 3], "delta": 0.0}, {"task": "matplotlib__matplotlib-24334", "a": [0, 3], "b": [0, 3], "delta": 0.0}, {"task": "matplotlib__matplotlib-25433", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "psf__requests-1963", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "pylint-dev__pylint-5859", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "pytest-dev__pytest-5103", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "pytest-dev__pytest-5495", "a": [0, 3], "b": [0, 3], "delta": 0.0}, {"task": "scikit-learn__scikit-learn-13241", "a": [0, 3], "b": [0, 3], "delta": 0.0}, {"task": "scikit-learn__scikit-learn-14983", "a": [3, 3], "b": [0, 3], "delta": 1.0}, {"task": "sphinx-doc__sphinx-10325", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "sphinx-doc__sphinx-8435", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "sympy__sympy-11400", "a": [0, 3], "b": [0, 3], "delta": 0.0}, {"task": "sympy__sympy-14024", "a": [1, 3], "b": [2, 3], "delta": -0.333333}, {"task": "sympy__sympy-14817", "a": [2, 3], "b": [2, 3], "delta": 0.0}, {"task": "sympy__sympy-15345", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "sympy__sympy-16792", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "sympy__sympy-20154", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "sympy__sympy-20590", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "sympy__sympy-23117", "a": [3, 3], "b": [3, 3], "delta": 0.0}]}, "cost": {"ratio": 5.0861, "ratio_ci": [4.4739, 5.7989], "diff": 0.122485, "diff_ci": [0.094565, 0.155369], "clear": true, "costlier": 30, "n": 30, "median": 5.2178, "q1": 3.8725, "q3": 6.6525, "a_mean": 0.15246, "b_mean": 0.029976, "ratios": [{"task": "astropy__astropy-7746", "ratio": 3.6999, "a": 0.103932, "b": 0.028091}, {"task": "django__django-11019", "ratio": 4.8491, "a": 0.227913, "b": 0.047001}, {"task": "django__django-11564", "ratio": 5.7502, "a": 0.328272, "b": 0.057088}, {"task": "django__django-11620", "ratio": 6.3462, "a": 0.189397, "b": 0.029844}, {"task": "django__django-11630", "ratio": 3.4112, "a": 0.173571, "b": 0.050883}, {"task": "django__django-11815", "ratio": 3.8922, "a": 0.088668, "b": 0.022781}, {"task": "django__django-12308", "ratio": 6.3561, "a": 0.07078, "b": 0.011136}, {"task": "django__django-12497", "ratio": 6.7513, "a": 0.04155, "b": 0.006154}, {"task": "django__django-12747", "ratio": 5.4076, "a": 0.128318, "b": 0.023729}, {"task": "django__django-14608", "ratio": 3.2525, "a": 0.101696, "b": 0.031267}, {"task": "django__django-15400", "ratio": 6.0954, "a": 0.064405, "b": 0.010566}, {"task": "django__django-15819", "ratio": 3.7133, "a": 0.083628, "b": 0.022521}, {"task": "matplotlib__matplotlib-24334", "ratio": 9.7648, "a": 0.165809, "b": 0.01698}, {"task": "matplotlib__matplotlib-25433", "ratio": 9.1763, "a": 0.277822, "b": 0.030276}, {"task": "psf__requests-1963", "ratio": 5.1909, "a": 0.091879, "b": 0.0177}, {"task": "pylint-dev__pylint-5859", "ratio": 8.0346, "a": 0.069806, "b": 0.008688}, {"task": "pytest-dev__pytest-5103", "ratio": 7.2781, "a": 0.491461, "b": 0.067526}, {"task": "pytest-dev__pytest-5495", "ratio": 4.7939, "a": 0.161126, "b": 0.033611}, {"task": "scikit-learn__scikit-learn-13241", "ratio": 9.9682, "a": 0.019968, "b": 0.002003}, {"task": "scikit-learn__scikit-learn-14983", "ratio": 3.7351, "a": 0.074696, "b": 0.019999}, {"task": "sphinx-doc__sphinx-10325", "ratio": 3.8659, "a": 0.187826, "b": 0.048585}, {"task": "sphinx-doc__sphinx-8435", "ratio": 3.102, "a": 0.222758, "b": 0.071811}, {"task": "sympy__sympy-11400", "ratio": 7.2237, "a": 0.180765, "b": 0.025024}, {"task": "sympy__sympy-14024", "ratio": 4.6897, "a": 0.297232, "b": 0.06338}, {"task": "sympy__sympy-14817", "ratio": 3.2549, "a": 0.126351, "b": 0.038819}, {"task": "sympy__sympy-15345", "ratio": 5.5146, "a": 0.080399, "b": 0.014579}, {"task": "sympy__sympy-16792", "ratio": 4.7182, "a": 0.222349, "b": 0.047126}, {"task": "sympy__sympy-20154", "ratio": 5.2447, "a": 0.096457, "b": 0.018391}, {"task": "sympy__sympy-20590", "ratio": 7.8061, "a": 0.122438, "b": 0.015685}, {"task": "sympy__sympy-23117", "ratio": 4.579, "a": 0.082535, "b": 0.018025}]}}, {"kind": "pair", "id": "swe:all-arena-v14:luna:codex:sha256:f48cd74a7b628be67703bf1a6ad11bf3e59af79e91d6f92f69dd547f6246c7c7 vs swe:all-arena-v14:luna:pi:sha256:f48cd74a7b628be67703bf1a6ad11bf3e59af79e91d6f92f69dd547f6246c7c7", "model": "luna", "model_label": "GPT-5.6 Luna", "color": "@model-luna", "harness": "codex", "harness_label": "Codex", "symbol": "triangle-up", "baseline": "pi", "baseline_label": "Pi", "a_label": "GPT-5.6 Luna \u00b7 Codex", "b_label": "GPT-5.6 Luna \u00b7 Pi", "cohort": "All-Arena reprofile r1-r3", "matched": true, "reasons": [], "descriptive": false, "evidence": "not clear after Holm", "caveats": [], "tasks": 30, "rate": {"mean": 0.022222, "ci": [-0.022222, 0.066667], "p": 0.2672, "p_holm": 1.0, "clear": false, "covers_zero": true, "wins_a": 3, "wins_b": 1, "same": 26, "mass_a": 0.033333, "mass_b": 0.011111, "breakdown": [{"task": "astropy__astropy-7746", "a": [0, 3], "b": [0, 3], "delta": 0.0}, {"task": "django__django-11019", "a": [0, 3], "b": [0, 3], "delta": 0.0}, {"task": "django__django-11564", "a": [0, 3], "b": [0, 3], "delta": 0.0}, {"task": "django__django-11620", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "django__django-11630", "a": [0, 3], "b": [0, 3], "delta": 0.0}, {"task": "django__django-11815", "a": [3, 3], "b": [2, 3], "delta": 0.333333}, {"task": "django__django-12308", "a": [0, 3], "b": [0, 3], "delta": 0.0}, {"task": "django__django-12497", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "django__django-12747", "a": [0, 3], "b": [0, 3], "delta": 0.0}, {"task": "django__django-14608", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "django__django-15400", "a": [0, 3], "b": [0, 3], "delta": 0.0}, {"task": "django__django-15819", "a": [0, 3], "b": [0, 3], "delta": 0.0}, {"task": "matplotlib__matplotlib-24334", "a": [0, 3], "b": [0, 3], "delta": 0.0}, {"task": "matplotlib__matplotlib-25433", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "psf__requests-1963", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "pylint-dev__pylint-5859", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "pytest-dev__pytest-5103", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "pytest-dev__pytest-5495", "a": [0, 3], "b": [0, 3], "delta": 0.0}, {"task": "scikit-learn__scikit-learn-13241", "a": [0, 3], "b": [0, 3], "delta": 0.0}, {"task": "scikit-learn__scikit-learn-14983", "a": [1, 3], "b": [0, 3], "delta": 0.333333}, {"task": "sphinx-doc__sphinx-10325", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "sphinx-doc__sphinx-8435", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "sympy__sympy-11400", "a": [0, 3], "b": [0, 3], "delta": 0.0}, {"task": "sympy__sympy-14024", "a": [1, 3], "b": [2, 3], "delta": -0.333333}, {"task": "sympy__sympy-14817", "a": [3, 3], "b": [2, 3], "delta": 0.333333}, {"task": "sympy__sympy-15345", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "sympy__sympy-16792", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "sympy__sympy-20154", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "sympy__sympy-20590", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "sympy__sympy-23117", "a": [3, 3], "b": [3, 3], "delta": 0.0}]}, "cost": {"ratio": 1.1838, "ratio_ci": [1.051, 1.3072], "diff": 0.00551, "diff_ci": [0.001598, 0.008966], "clear": false, "costlier": 24, "n": 30, "median": 1.2651, "q1": 1.0522, "q3": 1.4802, "a_mean": 0.035486, "b_mean": 0.029976, "ratios": [{"task": "astropy__astropy-7746", "ratio": 0.9722, "a": 0.027311, "b": 0.028091}, {"task": "django__django-11019", "ratio": 1.2061, "a": 0.05669, "b": 0.047001}, {"task": "django__django-11564", "ratio": 1.2502, "a": 0.071373, "b": 0.057088}, {"task": "django__django-11620", "ratio": 1.3067, "a": 0.038998, "b": 0.029844}, {"task": "django__django-11630", "ratio": 0.3346, "a": 0.017023, "b": 0.050883}, {"task": "django__django-11815", "ratio": 1.0401, "a": 0.023694, "b": 0.022781}, {"task": "django__django-12308", "ratio": 1.5905, "a": 0.017712, "b": 0.011136}, {"task": "django__django-12497", "ratio": 1.7866, "a": 0.010995, "b": 0.006154}, {"task": "django__django-12747", "ratio": 1.2187, "a": 0.028918, "b": 0.023729}, {"task": "django__django-14608", "ratio": 0.9481, "a": 0.029643, "b": 0.031267}, {"task": "django__django-15400", "ratio": 1.7975, "a": 0.018992, "b": 0.010566}, {"task": "django__django-15819", "ratio": 1.5066, "a": 0.03393, "b": 0.022521}, {"task": "matplotlib__matplotlib-24334", "ratio": 1.4356, "a": 0.024377, "b": 0.01698}, {"task": "matplotlib__matplotlib-25433", "ratio": 1.4338, "a": 0.043409, "b": 0.030276}, {"task": "psf__requests-1963", "ratio": 1.7937, "a": 0.031748, "b": 0.0177}, {"task": "pylint-dev__pylint-5859", "ratio": 2.0301, "a": 0.017638, "b": 0.008688}, {"task": "pytest-dev__pytest-5103", "ratio": 1.4824, "a": 0.100097, "b": 0.067526}, {"task": "pytest-dev__pytest-5495", "ratio": 1.3399, "a": 0.045034, "b": 0.033611}, {"task": "scikit-learn__scikit-learn-13241", "ratio": 1.4736, "a": 0.002952, "b": 0.002003}, {"task": "scikit-learn__scikit-learn-14983", "ratio": 1.1514, "a": 0.023027, "b": 0.019999}, {"task": "sphinx-doc__sphinx-10325", "ratio": 1.2982, "a": 0.063074, "b": 0.048585}, {"task": "sphinx-doc__sphinx-8435", "ratio": 0.9425, "a": 0.067683, "b": 0.071811}, {"task": "sympy__sympy-11400", "ratio": 1.0501, "a": 0.026277, "b": 0.025024}, {"task": "sympy__sympy-14024", "ratio": 1.1695, "a": 0.074125, "b": 0.06338}, {"task": "sympy__sympy-14817", "ratio": 1.0585, "a": 0.041091, "b": 0.038819}, {"task": "sympy__sympy-15345", "ratio": 1.2192, "a": 0.017774, "b": 0.014579}, {"task": "sympy__sympy-16792", "ratio": 0.9463, "a": 0.044595, "b": 0.047126}, {"task": "sympy__sympy-20154", "ratio": 1.2799, "a": 0.023538, "b": 0.018391}, {"task": "sympy__sympy-20590", "ratio": 1.7312, "a": 0.027153, "b": 0.015685}, {"task": "sympy__sympy-23117", "ratio": 0.8713, "a": 0.015705, "b": 0.018025}]}}, {"kind": "pair", "id": "swe:all-arena-v14:kimi:cc:sha256:f48cd74a7b628be67703bf1a6ad11bf3e59af79e91d6f92f69dd547f6246c7c7 vs swe:all-arena-v14:kimi:pi:sha256:f48cd74a7b628be67703bf1a6ad11bf3e59af79e91d6f92f69dd547f6246c7c7", "model": "kimi", "model_label": "Kimi K3", "color": "@model-kimi", "harness": "cc", "harness_label": "Claude Code", "symbol": "diamond", "baseline": "pi", "baseline_label": "Pi", "a_label": "Kimi K3 \u00b7 Claude Code", "b_label": "Kimi K3 \u00b7 Pi", "cohort": "All-Arena reprofile r1-r3", "matched": true, "reasons": [], "descriptive": false, "evidence": "cost survives Holm", "caveats": [], "tasks": 30, "rate": {"mean": 0.044444, "ci": [-0.044444, 0.144444], "p": 0.3484, "p_holm": 1.0, "clear": false, "covers_zero": true, "wins_a": 5, "wins_b": 4, "same": 21, "mass_a": 0.088889, "mass_b": 0.044444, "breakdown": [{"task": "astropy__astropy-7746", "a": [0, 3], "b": [0, 3], "delta": 0.0}, {"task": "django__django-11019", "a": [1, 3], "b": [0, 3], "delta": 0.333333}, {"task": "django__django-11564", "a": [0, 3], "b": [1, 3], "delta": -0.333333}, {"task": "django__django-11620", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "django__django-11630", "a": [0, 3], "b": [0, 3], "delta": 0.0}, {"task": "django__django-11815", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "django__django-12308", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "django__django-12497", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "django__django-12747", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "django__django-14608", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "django__django-15400", "a": [1, 3], "b": [2, 3], "delta": -0.333333}, {"task": "django__django-15819", "a": [2, 3], "b": [0, 3], "delta": 0.666667}, {"task": "matplotlib__matplotlib-24334", "a": [1, 3], "b": [2, 3], "delta": -0.333333}, {"task": "matplotlib__matplotlib-25433", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "psf__requests-1963", "a": [3, 3], "b": [2, 3], "delta": 0.333333}, {"task": "pylint-dev__pylint-5859", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "pytest-dev__pytest-5103", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "pytest-dev__pytest-5495", "a": [1, 3], "b": [0, 3], "delta": 0.333333}, {"task": "scikit-learn__scikit-learn-13241", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "scikit-learn__scikit-learn-14983", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "sphinx-doc__sphinx-10325", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "sphinx-doc__sphinx-8435", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "sympy__sympy-11400", "a": [0, 3], "b": [1, 3], "delta": -0.333333}, {"task": "sympy__sympy-14024", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "sympy__sympy-14817", "a": [3, 3], "b": [0, 3], "delta": 1.0}, {"task": "sympy__sympy-15345", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "sympy__sympy-16792", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "sympy__sympy-20154", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "sympy__sympy-20590", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "sympy__sympy-23117", "a": [3, 3], "b": [3, 3], "delta": 0.0}]}, "cost": {"ratio": 1.7234, "ratio_ci": [1.5109, 1.9652], "diff": 0.329268, "diff_ci": [0.216793, 0.468915], "clear": true, "costlier": 28, "n": 30, "median": 1.9632, "q1": 1.4838, "q3": 2.5587, "a_mean": 0.784425, "b_mean": 0.455158, "ratios": [{"task": "astropy__astropy-7746", "ratio": 2.5151, "a": 0.434417, "b": 0.172724}, {"task": "django__django-11019", "ratio": 1.2251, "a": 1.887422, "b": 1.540638}, {"task": "django__django-11564", "ratio": 1.8686, "a": 2.896218, "b": 1.549967}, {"task": "django__django-11620", "ratio": 1.9689, "a": 0.858552, "b": 0.436051}, {"task": "django__django-11630", "ratio": 1.2804, "a": 1.145942, "b": 0.894993}, {"task": "django__django-11815", "ratio": 3.3745, "a": 0.341523, "b": 0.101208}, {"task": "django__django-12308", "ratio": 2.7613, "a": 0.258845, "b": 0.093741}, {"task": "django__django-12497", "ratio": 3.3009, "a": 0.146848, "b": 0.044487}, {"task": "django__django-12747", "ratio": 1.6227, "a": 0.458332, "b": 0.282456}, {"task": "django__django-14608", "ratio": 2.1602, "a": 0.440334, "b": 0.203841}, {"task": "django__django-15400", "ratio": 1.9576, "a": 0.226667, "b": 0.115788}, {"task": "django__django-15819", "ratio": 1.5348, "a": 0.749191, "b": 0.488134}, {"task": "matplotlib__matplotlib-24334", "ratio": 2.4021, "a": 0.759992, "b": 0.316383}, {"task": "matplotlib__matplotlib-25433", "ratio": 1.4643, "a": 1.097353, "b": 0.749405}, {"task": "psf__requests-1963", "ratio": 2.6048, "a": 0.219084, "b": 0.084108}, {"task": "pylint-dev__pylint-5859", "ratio": 0.6917, "a": 0.373316, "b": 0.539689}, {"task": "pytest-dev__pytest-5103", "ratio": 2.15, "a": 2.477265, "b": 1.152201}, {"task": "pytest-dev__pytest-5495", "ratio": 1.3711, "a": 0.859016, "b": 0.626514}, {"task": "scikit-learn__scikit-learn-13241", "ratio": 2.4594, "a": 0.320339, "b": 0.13025}, {"task": "scikit-learn__scikit-learn-14983", "ratio": 2.5924, "a": 0.88356, "b": 0.340828}, {"task": "sphinx-doc__sphinx-10325", "ratio": 1.5164, "a": 0.848415, "b": 0.559477}, {"task": "sphinx-doc__sphinx-8435", "ratio": 2.1656, "a": 0.967356, "b": 0.446696}, {"task": "sympy__sympy-11400", "ratio": 1.1816, "a": 0.43553, "b": 0.368579}, {"task": "sympy__sympy-14024", "ratio": 2.5732, "a": 1.829415, "b": 0.710958}, {"task": "sympy__sympy-14817", "ratio": 1.4729, "a": 0.584453, "b": 0.396803}, {"task": "sympy__sympy-15345", "ratio": 3.8362, "a": 0.222411, "b": 0.057977}, {"task": "sympy__sympy-16792", "ratio": 0.9847, "a": 0.641593, "b": 0.651532}, {"task": "sympy__sympy-20154", "ratio": 1.5221, "a": 0.541248, "b": 0.355583}, {"task": "sympy__sympy-20590", "ratio": 3.496, "a": 0.34467, "b": 0.098589}, {"task": "sympy__sympy-23117", "ratio": 1.9532, "a": 0.28346, "b": 0.145129}]}}, {"kind": "pair", "id": "swe:all-arena-v14:kimi:codex:sha256:f48cd74a7b628be67703bf1a6ad11bf3e59af79e91d6f92f69dd547f6246c7c7 vs swe:all-arena-v14:kimi:pi:sha256:f48cd74a7b628be67703bf1a6ad11bf3e59af79e91d6f92f69dd547f6246c7c7", "model": "kimi", "model_label": "Kimi K3", "color": "@model-kimi", "harness": "codex", "harness_label": "Codex", "symbol": "triangle-up", "baseline": "pi", "baseline_label": "Pi", "a_label": "Kimi K3 \u00b7 Codex", "b_label": "Kimi K3 \u00b7 Pi", "cohort": "All-Arena reprofile r1-r3", "matched": true, "reasons": [], "descriptive": false, "evidence": "cost survives Holm", "caveats": [], "tasks": 30, "rate": {"mean": 0.022222, "ci": [-0.055556, 0.111389], "p": 0.6556, "p_holm": 1.0, "clear": false, "covers_zero": true, "wins_a": 4, "wins_b": 4, "same": 22, "mass_a": 0.066667, "mass_b": 0.044444, "breakdown": [{"task": "astropy__astropy-7746", "a": [0, 3], "b": [0, 3], "delta": 0.0}, {"task": "django__django-11019", "a": [0, 3], "b": [0, 3], "delta": 0.0}, {"task": "django__django-11564", "a": [0, 3], "b": [1, 3], "delta": -0.333333}, {"task": "django__django-11620", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "django__django-11630", "a": [0, 3], "b": [0, 3], "delta": 0.0}, {"task": "django__django-11815", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "django__django-12308", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "django__django-12497", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "django__django-12747", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "django__django-14608", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "django__django-15400", "a": [1, 3], "b": [2, 3], "delta": -0.333333}, {"task": "django__django-15819", "a": [1, 3], "b": [0, 3], "delta": 0.333333}, {"task": "matplotlib__matplotlib-24334", "a": [1, 3], "b": [2, 3], "delta": -0.333333}, {"task": "matplotlib__matplotlib-25433", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "psf__requests-1963", "a": [3, 3], "b": [2, 3], "delta": 0.333333}, {"task": "pylint-dev__pylint-5859", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "pytest-dev__pytest-5103", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "pytest-dev__pytest-5495", "a": [3, 3], "b": [0, 3], "delta": 1.0}, {"task": "scikit-learn__scikit-learn-13241", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "scikit-learn__scikit-learn-14983", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "sphinx-doc__sphinx-10325", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "sphinx-doc__sphinx-8435", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "sympy__sympy-11400", "a": [2, 3], "b": [1, 3], "delta": 0.333333}, {"task": "sympy__sympy-14024", "a": [2, 3], "b": [3, 3], "delta": -0.333333}, {"task": "sympy__sympy-14817", "a": [0, 3], "b": [0, 3], "delta": 0.0}, {"task": "sympy__sympy-15345", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "sympy__sympy-16792", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "sympy__sympy-20154", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "sympy__sympy-20590", "a": [3, 3], "b": [3, 3], "delta": 0.0}, {"task": "sympy__sympy-23117", "a": [3, 3], "b": [3, 3], "delta": 0.0}]}, "cost": {"ratio": 1.8565, "ratio_ci": [1.5831, 2.2323], "diff": 0.389846, "diff_ci": [0.291658, 0.498554], "clear": true, "costlier": 28, "n": 30, "median": 2.203, "q1": 1.803, "q3": 2.7989, "a_mean": 0.845004, "b_mean": 0.455158, "ratios": [{"task": "astropy__astropy-7746", "ratio": 4.4591, "a": 0.770197, "b": 0.172724}, {"task": "django__django-11019", "ratio": 1.3576, "a": 2.091514, "b": 1.540638}, {"task": "django__django-11564", "ratio": 1.1748, "a": 1.820933, "b": 1.549967}, {"task": "django__django-11620", "ratio": 2.0547, "a": 0.895967, "b": 0.436051}, {"task": "django__django-11630", "ratio": 0.7393, "a": 0.661701, "b": 0.894993}, {"task": "django__django-11815", "ratio": 2.183, "a": 0.220941, "b": 0.101208}, {"task": "django__django-12308", "ratio": 3.335, "a": 0.312622, "b": 0.093741}, {"task": "django__django-12497", "ratio": 2.21, "a": 0.098315, "b": 0.044487}, {"task": "django__django-12747", "ratio": 2.1337, "a": 0.602667, "b": 0.282456}, {"task": "django__django-14608", "ratio": 2.2365, "a": 0.455896, "b": 0.203841}, {"task": "django__django-15400", "ratio": 3.416, "a": 0.395529, "b": 0.115788}, {"task": "django__django-15819", "ratio": 1.7838, "a": 0.870724, "b": 0.488134}, {"task": "matplotlib__matplotlib-24334", "ratio": 2.196, "a": 0.694777, "b": 0.316383}, {"task": "matplotlib__matplotlib-25433", "ratio": 1.5272, "a": 1.144511, "b": 0.749405}, {"task": "psf__requests-1963", "ratio": 7.9458, "a": 0.668308, "b": 0.084108}, {"task": "pylint-dev__pylint-5859", "ratio": 1.1453, "a": 0.618115, "b": 0.539689}, {"task": "pytest-dev__pytest-5103", "ratio": 1.554, "a": 1.790532, "b": 1.152201}, {"task": "pytest-dev__pytest-5495", "ratio": 1.9772, "a": 1.238712, "b": 0.626514}, {"task": "scikit-learn__scikit-learn-13241", "ratio": 2.0541, "a": 0.267549, "b": 0.13025}, {"task": "scikit-learn__scikit-learn-14983", "ratio": 2.4148, "a": 0.82303, "b": 0.340828}, {"task": "sphinx-doc__sphinx-10325", "ratio": 2.2298, "a": 1.247497, "b": 0.559477}, {"task": "sphinx-doc__sphinx-8435", "ratio": 2.8111, "a": 1.255712, "b": 0.446696}, {"task": "sympy__sympy-11400", "ratio": 2.7181, "a": 1.001829, "b": 0.368579}, {"task": "sympy__sympy-14024", "ratio": 2.7109, "a": 1.927369, "b": 0.710958}, {"task": "sympy__sympy-14817", "ratio": 3.0005, "a": 1.190614, "b": 0.396803}, {"task": "sympy__sympy-15345", "ratio": 4.5198, "a": 0.262043, "b": 0.057977}, {"task": "sympy__sympy-16792", "ratio": 0.9704, "a": 0.632279, "b": 0.651532}, {"task": "sympy__sympy-20154", "ratio": 1.8605, "a": 0.661559, "b": 0.355583}, {"task": "sympy__sympy-20590", "ratio": 2.7623, "a": 0.27233, "b": 0.098589}, {"task": "sympy__sympy-23117", "ratio": 3.1443, "a": 0.456337, "b": 0.145129}]}}], "frontier_points": [{"id": "swe:all-arena-v14:fable:cc:sha256:f48cd74a7b628be67703bf1a6ad11bf3e59af79e91d6f92f69dd547f6246c7c7", "label": "cc:fable", "cohort": "All-Arena reprofile r1-r3", "qualifier": null, "sample": "30 unique tasks over 3 repetitions", "model": "fable", "model_label": "Claude Fable 5", "harness": "cc", "harness_label": "Claude Code", "x": 1.3293, "y": 0.977778, "x_ci": [1.101256, 1.601415], "y_ci": [0.933333, 1.0], "frontier": true, "color": "@model-fable", "symbol": "diamond", "bootstrap": 0.337, "single_replicate": false, "publication_scope": "swe-all-arena-v14-r1-r3", "replicates": 3}, {"id": "swe:all-arena-v14:opus:cc:sha256:f48cd74a7b628be67703bf1a6ad11bf3e59af79e91d6f92f69dd547f6246c7c7", "label": "cc:opus", "cohort": "All-Arena reprofile r1-r3", "qualifier": null, "sample": "30 unique tasks over 3 repetitions", "model": "opus", "model_label": "Claude Opus 4.8", "harness": "cc", "harness_label": "Claude Code", "x": 0.975761, "y": 0.866667, "x_ci": [0.733246, 1.262789], "y_ci": [0.755556, 0.955556], "frontier": false, "color": "@model-opus", "symbol": "diamond", "bootstrap": 0.0, "single_replicate": false, "publication_scope": "swe-all-arena-v14-r1-r3", "replicates": 3}, {"id": "swe:all-arena-v14:sonnet:cc:sha256:f48cd74a7b628be67703bf1a6ad11bf3e59af79e91d6f92f69dd547f6246c7c7", "label": "cc:sonnet", "cohort": "All-Arena reprofile r1-r3", "qualifier": null, "sample": "30 unique tasks over 3 repetitions", "model": "sonnet", "model_label": "Claude Sonnet 4.6", "harness": "cc", "harness_label": "Claude Code", "x": 0.669348, "y": 0.666667, "x_ci": [0.498678, 0.862522], "y_ci": [0.511111, 0.811111], "frontier": false, "color": "@model-sonnet", "symbol": "diamond", "bootstrap": 0.0, "single_replicate": false, "publication_scope": "swe-all-arena-v14-r1-r3", "replicates": 3}, {"id": "swe:all-arena-v14:haiku:cc:sha256:f48cd74a7b628be67703bf1a6ad11bf3e59af79e91d6f92f69dd547f6246c7c7", "label": "cc:haiku", "cohort": "All-Arena reprofile r1-r3", "qualifier": null, "sample": "30 unique tasks over 3 repetitions", "model": "haiku", "model_label": "Claude Haiku 4.5", "harness": "cc", "harness_label": "Claude Code", "x": 0.426164, "y": 0.522222, "x_ci": [0.345863, 0.507158], "y_ci": [0.366667, 0.677778], "frontier": false, "color": "@model-haiku", "symbol": "diamond", "bootstrap": 0.0125, "single_replicate": false, "publication_scope": "swe-all-arena-v14-r1-r3", "replicates": 3}, {"id": "swe:all-arena-v14:sol:cc:sha256:f48cd74a7b628be67703bf1a6ad11bf3e59af79e91d6f92f69dd547f6246c7c7", "label": "cc:sol", "cohort": "All-Arena reprofile r1-r3", "qualifier": null, "sample": "30 unique tasks over 3 repetitions", "model": "sol", "model_label": "GPT-5.6 Sol", "harness": "cc", "harness_label": "Claude Code", "x": 1.540276, "y": 0.777778, "x_ci": [1.184199, 2.09472], "y_ci": [0.633333, 0.9], "frontier": false, "color": "@model-sol", "symbol": "diamond", "bootstrap": 0.0, "single_replicate": false, "publication_scope": "swe-all-arena-v14-r1-r3", "replicates": 3}, {"id": "swe:all-arena-v14:luna:cc:sha256:f48cd74a7b628be67703bf1a6ad11bf3e59af79e91d6f92f69dd547f6246c7c7", "label": "cc:luna", "cohort": "All-Arena reprofile r1-r3", "qualifier": null, "sample": "30 unique tasks over 3 repetitions", "model": "luna", "model_label": "GPT-5.6 Luna", "harness": "cc", "harness_label": "Claude Code", "x": 0.15246, "y": 0.555556, "x_ci": [0.119702, 0.190419], "y_ci": [0.388889, 0.722222], "frontier": false, "color": "@model-luna", "symbol": "diamond", "bootstrap": 0.3845, "single_replicate": false, "publication_scope": "swe-all-arena-v14-r1-r3", "replicates": 3}, {"id": "swe:all-arena-v14:kimi:cc:sha256:f48cd74a7b628be67703bf1a6ad11bf3e59af79e91d6f92f69dd547f6246c7c7", "label": "cc:kimi", "cohort": "All-Arena reprofile r1-r3", "qualifier": null, "sample": "30 unique tasks over 3 repetitions", "model": "kimi", "model_label": "Kimi K3", "harness": "cc", "harness_label": "Claude Code", "x": 0.784425, "y": 0.766667, "x_ci": [0.566712, 1.044778], "y_ci": [0.622222, 0.888889], "frontier": false, "color": "@model-kimi", "symbol": "diamond", "bootstrap": 0.007, "single_replicate": false, "publication_scope": "swe-all-arena-v14-r1-r3", "replicates": 3}, {"id": "swe:all-arena-v14:fable:codex:sha256:f48cd74a7b628be67703bf1a6ad11bf3e59af79e91d6f92f69dd547f6246c7c7", "label": "codex:fable", "cohort": "All-Arena reprofile r1-r3", "qualifier": null, "sample": "30 unique tasks over 3 repetitions", "model": "fable", "model_label": "Claude Fable 5", "harness": "codex", "harness_label": "Codex", "x": 0.890467, "y": 0.966667, "x_ci": [0.696656, 1.126628], "y_ci": [0.911111, 1.0], "frontier": false, "color": "@model-fable", "symbol": "triangle-up", "bootstrap": 0.384, "single_replicate": false, "publication_scope": "swe-all-arena-v14-r1-r3", "replicates": 3}, {"id": "swe:all-arena-v14:opus:codex:sha256:f48cd74a7b628be67703bf1a6ad11bf3e59af79e91d6f92f69dd547f6246c7c7", "label": "codex:opus", "cohort": "All-Arena reprofile r1-r3", "qualifier": null, "sample": "30 unique tasks over 3 repetitions", "model": "opus", "model_label": "Claude Opus 4.8", "harness": "codex", "harness_label": "Codex", "x": 0.693734, "y": 0.888889, "x_ci": [0.491485, 0.946207], "y_ci": [0.777778, 0.977778], "frontier": false, "color": "@model-opus", "symbol": "triangle-up", "bootstrap": 0.2775, "single_replicate": false, "publication_scope": "swe-all-arena-v14-r1-r3", "replicates": 3}, {"id": "swe:all-arena-v14:sonnet:codex:sha256:f48cd74a7b628be67703bf1a6ad11bf3e59af79e91d6f92f69dd547f6246c7c7", "label": "codex:sonnet", "cohort": "All-Arena reprofile r1-r3", "qualifier": null, "sample": "30 unique tasks over 3 repetitions", "model": "sonnet", "model_label": "Claude Sonnet 4.6", "harness": "codex", "harness_label": "Codex", "x": 0.744673, "y": 0.688889, "x_ci": [0.485095, 1.0996], "y_ci": [0.533333, 0.833333], "frontier": false, "color": "@model-sonnet", "symbol": "triangle-up", "bootstrap": 0.0015, "single_replicate": false, "publication_scope": "swe-all-arena-v14-r1-r3", "replicates": 3}, {"id": "swe:all-arena-v14:haiku:codex:sha256:f48cd74a7b628be67703bf1a6ad11bf3e59af79e91d6f92f69dd547f6246c7c7", "label": "codex:haiku", "cohort": "All-Arena reprofile r1-r3", "qualifier": null, "sample": "30 unique tasks over 3 repetitions", "model": "haiku", "model_label": "Claude Haiku 4.5", "harness": "codex", "harness_label": "Codex", "x": 0.391888, "y": 0.577778, "x_ci": [0.295968, 0.510313], "y_ci": [0.411111, 0.744444], "frontier": false, "color": "@model-haiku", "symbol": "triangle-up", "bootstrap": 0.2645, "single_replicate": false, "publication_scope": "swe-all-arena-v14-r1-r3", "replicates": 3}, {"id": "swe:all-arena-v14:sol:codex:sha256:f48cd74a7b628be67703bf1a6ad11bf3e59af79e91d6f92f69dd547f6246c7c7", "label": "codex:sol", "cohort": "All-Arena reprofile r1-r3", "qualifier": null, "sample": "30 unique tasks over 3 repetitions", "model": "sol", "model_label": "GPT-5.6 Sol", "harness": "codex", "harness_label": "Codex", "x": 0.561265, "y": 0.733333, "x_ci": [0.442062, 0.702244], "y_ci": [0.577778, 0.877778], "frontier": false, "color": "@model-sol", "symbol": "triangle-up", "bootstrap": 0.057, "single_replicate": false, "publication_scope": "swe-all-arena-v14-r1-r3", "replicates": 3}, {"id": "swe:all-arena-v14:luna:codex:sha256:f48cd74a7b628be67703bf1a6ad11bf3e59af79e91d6f92f69dd547f6246c7c7", "label": "codex:luna", "cohort": "All-Arena reprofile r1-r3", "qualifier": null, "sample": "30 unique tasks over 3 repetitions", "model": "luna", "model_label": "GPT-5.6 Luna", "harness": "codex", "harness_label": "Codex", "x": 0.035486, "y": 0.555556, "x_ci": [0.028296, 0.043708], "y_ci": [0.377778, 0.722222], "frontier": true, "color": "@model-luna", "symbol": "triangle-up", "bootstrap": 0.7885, "single_replicate": false, "publication_scope": "swe-all-arena-v14-r1-r3", "replicates": 3}, {"id": "swe:all-arena-v14:kimi:codex:sha256:f48cd74a7b628be67703bf1a6ad11bf3e59af79e91d6f92f69dd547f6246c7c7", "label": "codex:kimi", "cohort": "All-Arena reprofile r1-r3", "qualifier": null, "sample": "30 unique tasks over 3 repetitions", "model": "kimi", "model_label": "Kimi K3", "harness": "codex", "harness_label": "Codex", "x": 0.845004, "y": 0.744444, "x_ci": [0.665122, 1.042766], "y_ci": [0.6, 0.877778], "frontier": false, "color": "@model-kimi", "symbol": "triangle-up", "bootstrap": 0.0, "single_replicate": false, "publication_scope": "swe-all-arena-v14-r1-r3", "replicates": 3}, {"id": "swe:all-arena-v14:fable:pi:sha256:f48cd74a7b628be67703bf1a6ad11bf3e59af79e91d6f92f69dd547f6246c7c7", "label": "pi:fable", "cohort": "All-Arena reprofile r1-r3", "qualifier": null, "sample": "30 unique tasks over 3 repetitions", "model": "fable", "model_label": "Claude Fable 5", "harness": "pi", "harness_label": "Pi", "x": 0.665695, "y": 0.966667, "x_ci": [0.493714, 0.875495], "y_ci": [0.911111, 1.0], "frontier": true, "color": "@model-fable", "symbol": "square", "bootstrap": 0.989, "single_replicate": false, "publication_scope": "swe-all-arena-v14-r1-r3", "replicates": 3}, {"id": "swe:all-arena-v14:opus:pi:sha256:f48cd74a7b628be67703bf1a6ad11bf3e59af79e91d6f92f69dd547f6246c7c7", "label": "pi:opus", "cohort": "All-Arena reprofile r1-r3", "qualifier": null, "sample": "30 unique tasks over 3 repetitions", "model": "opus", "model_label": "Claude Opus 4.8", "harness": "pi", "harness_label": "Pi", "x": 0.472603, "y": 0.822222, "x_ci": [0.271722, 0.725234], "y_ci": [0.711111, 0.922222], "frontier": true, "color": "@model-opus", "symbol": "square", "bootstrap": 0.861, "single_replicate": false, "publication_scope": "swe-all-arena-v14-r1-r3", "replicates": 3}, {"id": "swe:all-arena-v14:sonnet:pi:sha256:f48cd74a7b628be67703bf1a6ad11bf3e59af79e91d6f92f69dd547f6246c7c7", "label": "pi:sonnet", "cohort": "All-Arena reprofile r1-r3", "qualifier": null, "sample": "30 unique tasks over 3 repetitions", "model": "sonnet", "model_label": "Claude Sonnet 4.6", "harness": "pi", "harness_label": "Pi", "x": 0.679488, "y": 0.644444, "x_ci": [0.461425, 0.926388], "y_ci": [0.477778, 0.8], "frontier": false, "color": "@model-sonnet", "symbol": "square", "bootstrap": 0.0005, "single_replicate": false, "publication_scope": "swe-all-arena-v14-r1-r3", "replicates": 3}, {"id": "swe:all-arena-v14:haiku:pi:sha256:f48cd74a7b628be67703bf1a6ad11bf3e59af79e91d6f92f69dd547f6246c7c7", "label": "pi:haiku", "cohort": "All-Arena reprofile r1-r3", "qualifier": null, "sample": "30 unique tasks over 3 repetitions", "model": "haiku", "model_label": "Claude Haiku 4.5", "harness": "pi", "harness_label": "Pi", "x": 0.37357, "y": 0.6, "x_ci": [0.308035, 0.440719], "y_ci": [0.433333, 0.755556], "frontier": true, "color": "@model-haiku", "symbol": "square", "bootstrap": 0.5115, "single_replicate": false, "publication_scope": "swe-all-arena-v14-r1-r3", "replicates": 3}, {"id": "swe:all-arena-v14:sol:pi:sha256:f48cd74a7b628be67703bf1a6ad11bf3e59af79e91d6f92f69dd547f6246c7c7", "label": "pi:sol", "cohort": "All-Arena reprofile r1-r3", "qualifier": null, "sample": "30 unique tasks over 3 repetitions", "model": "sol", "model_label": "GPT-5.6 Sol", "harness": "pi", "harness_label": "Pi", "x": 0.440661, "y": 0.744444, "x_ci": [0.341663, 0.553393], "y_ci": [0.588889, 0.888889], "frontier": true, "color": "@model-sol", "symbol": "square", "bootstrap": 0.587, "single_replicate": false, "publication_scope": "swe-all-arena-v14-r1-r3", "replicates": 3}, {"id": "swe:all-arena-v14:luna:pi:sha256:f48cd74a7b628be67703bf1a6ad11bf3e59af79e91d6f92f69dd547f6246c7c7", "label": "pi:luna", "cohort": "All-Arena reprofile r1-r3", "qualifier": null, "sample": "30 unique tasks over 3 repetitions", "model": "luna", "model_label": "GPT-5.6 Luna", "harness": "pi", "harness_label": "Pi", "x": 0.029976, "y": 0.533333, "x_ci": [0.023563, 0.037023], "y_ci": [0.355556, 0.7], "frontier": true, "color": "@model-luna", "symbol": "square", "bootstrap": 0.9985, "single_replicate": false, "publication_scope": "swe-all-arena-v14-r1-r3", "replicates": 3}, {"id": "swe:all-arena-v14:kimi:pi:sha256:f48cd74a7b628be67703bf1a6ad11bf3e59af79e91d6f92f69dd547f6246c7c7", "label": "pi:kimi", "cohort": "All-Arena reprofile r1-r3", "qualifier": null, "sample": "30 unique tasks over 3 repetitions", "model": "kimi", "model_label": "Kimi K3", "harness": "pi", "harness_label": "Pi", "x": 0.455158, "y": 0.722222, "x_ci": [0.322859, 0.607449], "y_ci": [0.577778, 0.855556], "frontier": false, "color": "@model-kimi", "symbol": "square", "bootstrap": 0.3095, "single_replicate": false, "publication_scope": "swe-all-arena-v14-r1-r3", "replicates": 3}]}, "plotly": {"data": [], "layout": {}}}